From 525b11e4c70a512381f7d0a56b9f7c01f79c0649 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 12 Feb 2025 14:53:10 +0100 Subject: [PATCH 1/2] vendor latest c/storage To include the new fileutils.ReflinkOrCopy() function. Signed-off-by: Paul Holzinger --- go.mod | 4 +- go.sum | 8 ++-- .../github.com/containers/storage/.cirrus.yml | 2 +- vendor/github.com/containers/storage/VERSION | 2 +- .../storage/drivers/overlay/overlay.go | 37 ++++++++----------- .../github.com/containers/storage/layers.go | 23 ++++++------ .../containers/storage/pkg/archive/archive.go | 1 - .../storage/pkg/fileutils/reflink_linux.go | 20 ++++++++++ .../pkg/fileutils/reflink_unsupported.go | 15 ++++++++ .../storage/pkg/unshare/unshare_linux.go | 11 +++++- vendor/github.com/containers/storage/store.go | 6 +-- .../vbatts/tar-split/archive/tar/writer.go | 3 ++ vendor/modules.txt | 4 +- 13 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 vendor/github.com/containers/storage/pkg/fileutils/reflink_linux.go create mode 100644 vendor/github.com/containers/storage/pkg/fileutils/reflink_unsupported.go diff --git a/go.mod b/go.mod index 4148fde2a2..7e25ff5a61 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/containers/libhvee v0.9.0 github.com/containers/ocicrypt v1.2.1 github.com/containers/psgo v1.9.0 - github.com/containers/storage v1.57.1 + github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3 github.com/containers/winquit v1.1.0 github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 github.com/crc-org/crc/v2 v2.45.0 @@ -211,7 +211,7 @@ require ( github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/vbatts/tar-split v0.11.7 // indirect + github.com/vbatts/tar-split v0.12.1 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect diff --git a/go.sum b/go.sum index 15908629c7..5d39c5717e 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,8 @@ github.com/containers/ocicrypt v1.2.1 h1:0qIOTT9DoYwcKmxSt8QJt+VzMY18onl9jUXsxpV github.com/containers/ocicrypt v1.2.1/go.mod h1:aD0AAqfMp0MtwqWgHM1bUwe1anx0VazI108CRrSKINQ= github.com/containers/psgo v1.9.0 h1:eJ74jzSaCHnWt26OlKZROSyUyRcGDf+gYBdXnxrMW4g= github.com/containers/psgo v1.9.0/go.mod h1:0YoluUm43Mz2UnBIh1P+6V6NWcbpTL5uRtXyOcH0B5A= -github.com/containers/storage v1.57.1 h1:hKPoFsuBcB3qTzBxa4IFpZMRzUuL5Xhv/BE44W0XHx8= -github.com/containers/storage v1.57.1/go.mod h1:i/Hb4lu7YgFr9G0K6BMjqW0BLJO1sFsnWQwj2UoWCUM= +github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3 h1:YLjd5aplmRP98Jlrqz5+kNmbVZvpZwrZygkF96KR2Fs= +github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3/go.mod h1:zsh6czcxcdqKIz//cVU6waEJ+2Ui8OEnrwCvM/DE3iU= github.com/containers/winquit v1.1.0 h1:jArun04BNDQvt2W0Y78kh9TazN2EIEMG5Im6/JY7+pE= github.com/containers/winquit v1.1.0/go.mod h1:PsPeZlnbkmGGIToMPHF1zhWjBUkd8aHjMOr/vFcPxw8= github.com/coreos/go-oidc/v3 v3.12.0 h1:sJk+8G2qq94rDI6ehZ71Bol3oUHy63qNYmkiSjrc/Jo= @@ -514,8 +514,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/vbatts/tar-split v0.11.7 h1:ixZ93pO/GmvaZw4Vq9OwmfZK/kc2zKdPfu0B+gYqs3U= -github.com/vbatts/tar-split v0.11.7/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= +github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/vbauerster/mpb/v8 v8.9.2 h1:kb91+D643Qg040bbICYtzpjgZ9ypVO/+sjv4Jcm6si4= github.com/vbauerster/mpb/v8 v8.9.2/go.mod h1:hxS8Hz4C6ijnppDSIX6LjG8FYJSoPo9iIOcE53Zik0c= github.com/vishvananda/netlink v1.3.1-0.20250128002108-7c2350bd140f h1:G5t3qYQ3YL2zMn2kFzRYIPk1EvDvMNV9pP+w+39VtzI= diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index 2102e938df..ba00093ed0 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -23,7 +23,7 @@ env: # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - IMAGE_SUFFIX: "c20250107t132430z-f41f40d13" + IMAGE_SUFFIX: "c20250131t121915z-f41f40d13" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index b4cf7c0db5..cfeaf2d089 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.57.1 +1.58.0-dev diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 56278805f8..e4bcd8a6f4 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -36,7 +36,6 @@ import ( "github.com/containers/storage/pkg/system" "github.com/containers/storage/pkg/unshare" units "github.com/docker/go-units" - "github.com/hashicorp/go-multierror" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux" "github.com/opencontainers/selinux/go-selinux/label" @@ -419,7 +418,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) if !opts.skipMountHome { if err := mount.MakePrivate(home); err != nil { - return nil, err + return nil, fmt.Errorf("overlay: failed to make mount private: %w", err) } } @@ -1343,7 +1342,7 @@ func (d *Driver) recreateSymlinks() error { return err } // Keep looping as long as we take some corrective action in each iteration - var errs *multierror.Error + var errs error madeProgress := true iterations := 0 for madeProgress { @@ -1359,7 +1358,7 @@ func (d *Driver) recreateSymlinks() error { // Read the "link" file under each layer to get the name of the symlink data, err := os.ReadFile(path.Join(d.dir(dir.Name()), "link")) if err != nil { - errs = multierror.Append(errs, fmt.Errorf("reading name of symlink for %q: %w", dir.Name(), err)) + errs = errors.Join(errs, fmt.Errorf("reading name of symlink for %q: %w", dir.Name(), err)) continue } linkPath := path.Join(d.home, linkDir, strings.Trim(string(data), "\n")) @@ -1368,12 +1367,12 @@ func (d *Driver) recreateSymlinks() error { err = fileutils.Lexists(linkPath) if err != nil && os.IsNotExist(err) { if err := os.Symlink(path.Join("..", dir.Name(), "diff"), linkPath); err != nil { - errs = multierror.Append(errs, err) + errs = errors.Join(errs, err) continue } madeProgress = true } else if err != nil { - errs = multierror.Append(errs, err) + errs = errors.Join(errs, err) continue } } @@ -1384,7 +1383,7 @@ func (d *Driver) recreateSymlinks() error { // that each symlink we have corresponds to one. links, err := os.ReadDir(linkDirFullPath) if err != nil { - errs = multierror.Append(errs, err) + errs = errors.Join(errs, err) continue } // Go through all of the symlinks in the "l" directory @@ -1392,16 +1391,16 @@ func (d *Driver) recreateSymlinks() error { // Read the symlink's target, which should be "../$layer/diff" target, err := os.Readlink(filepath.Join(linkDirFullPath, link.Name())) if err != nil { - errs = multierror.Append(errs, err) + errs = errors.Join(errs, err) continue } targetComponents := strings.Split(target, string(os.PathSeparator)) if len(targetComponents) != 3 || targetComponents[0] != ".." || targetComponents[2] != "diff" { - errs = multierror.Append(errs, fmt.Errorf("link target of %q looks weird: %q", link, target)) + errs = errors.Join(errs, fmt.Errorf("link target of %q looks weird: %q", link, target)) // force the link to be recreated on the next pass if err := os.Remove(filepath.Join(linkDirFullPath, link.Name())); err != nil { if !os.IsNotExist(err) { - errs = multierror.Append(errs, fmt.Errorf("removing link %q: %w", link, err)) + errs = errors.Join(errs, fmt.Errorf("removing link %q: %w", link, err)) } // else don’t report any error, but also don’t set madeProgress. continue } @@ -1417,7 +1416,7 @@ func (d *Driver) recreateSymlinks() error { // NOTE: If two or more links point to the same target, we will update linkFile // with every value of link.Name(), and set madeProgress = true every time. if err := os.WriteFile(linkFile, []byte(link.Name()), 0o644); err != nil { - errs = multierror.Append(errs, fmt.Errorf("correcting link for layer %s: %w", targetID, err)) + errs = errors.Join(errs, fmt.Errorf("correcting link for layer %s: %w", targetID, err)) continue } madeProgress = true @@ -1425,14 +1424,11 @@ func (d *Driver) recreateSymlinks() error { } iterations++ if iterations >= maxIterations { - errs = multierror.Append(errs, fmt.Errorf("reached %d iterations in overlay graph driver’s recreateSymlink, giving up", iterations)) + errs = errors.Join(errs, fmt.Errorf("reached %d iterations in overlay graph driver’s recreateSymlink, giving up", iterations)) break } } - if errs != nil { - return errs.ErrorOrNil() - } - return nil + return errs } // Get creates and mounts the required file system for the given id and returns the mount path. @@ -2103,17 +2099,16 @@ func (g *overlayFileGetter) Get(path string) (io.ReadCloser, error) { return nil, fmt.Errorf("%s: %w", path, os.ErrNotExist) } -func (g *overlayFileGetter) Close() error { - var errs *multierror.Error +func (g *overlayFileGetter) Close() (errs error) { for _, f := range g.composefsMounts { if err := f.Close(); err != nil { - errs = multierror.Append(errs, err) + errs = errors.Join(errs, err) } if err := unix.Rmdir(f.Name()); err != nil { - errs = multierror.Append(errs, err) + errs = errors.Join(errs, err) } } - return errs.ErrorOrNil() + return errs } // newStagingDir creates a new staging directory and returns the path to it. diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 1f8203fbfc..69a4887bfb 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -26,7 +26,6 @@ import ( "github.com/containers/storage/pkg/system" "github.com/containers/storage/pkg/tarlog" "github.com/containers/storage/pkg/truncindex" - multierror "github.com/hashicorp/go-multierror" "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux" @@ -937,7 +936,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { // Don't return the error immediately, because deleteInternal does not saveLayers(); // Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully // deleted incomplete layers have their metadata correctly removed. - incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors, + incompleteDeletionErrors = errors.Join(incompleteDeletionErrors, fmt.Errorf("deleting layer %#v: %w", layer.ID, err)) } modifiedLocations |= layerLocation(layer) @@ -2256,33 +2255,33 @@ func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, // but they modify in-memory state. fgetter, err := r.newFileGetter(to) if err != nil { - errs := multierror.Append(nil, fmt.Errorf("creating file-getter: %w", err)) + errs := fmt.Errorf("creating file-getter: %w", err) if err := decompressor.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("closing decompressor: %w", err)) + errs = errors.Join(errs, fmt.Errorf("closing decompressor: %w", err)) } if err := tsfile.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("closing tarstream headers: %w", err)) + errs = errors.Join(errs, fmt.Errorf("closing tarstream headers: %w", err)) } - return nil, errs.ErrorOrNil() + return nil, errs } tarstream := asm.NewOutputTarStream(fgetter, metadata) rc := ioutils.NewReadCloserWrapper(tarstream, func() error { - var errs *multierror.Error + var errs error if err := decompressor.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("closing decompressor: %w", err)) + errs = errors.Join(errs, fmt.Errorf("closing decompressor: %w", err)) } if err := tsfile.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("closing tarstream headers: %w", err)) + errs = errors.Join(errs, fmt.Errorf("closing tarstream headers: %w", err)) } if err := tarstream.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("closing reconstructed tarstream: %w", err)) + errs = errors.Join(errs, fmt.Errorf("closing reconstructed tarstream: %w", err)) } if err := fgetter.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("closing file-getter: %w", err)) + errs = errors.Join(errs, fmt.Errorf("closing file-getter: %w", err)) } if errs != nil { - return errs.ErrorOrNil() + return errs } return nil }) diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 41daad8530..f17a4651fc 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -78,7 +78,6 @@ const ( windows = "windows" darwin = "darwin" freebsd = "freebsd" - linux = "linux" ) var xattrsToIgnore = map[string]interface{}{ diff --git a/vendor/github.com/containers/storage/pkg/fileutils/reflink_linux.go b/vendor/github.com/containers/storage/pkg/fileutils/reflink_linux.go new file mode 100644 index 0000000000..9f5c6c90bb --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/fileutils/reflink_linux.go @@ -0,0 +1,20 @@ +package fileutils + +import ( + "io" + "os" + + "golang.org/x/sys/unix" +) + +// ReflinkOrCopy attempts to reflink the source to the destination fd. +// If reflinking fails or is unsupported, it falls back to io.Copy(). +func ReflinkOrCopy(src, dst *os.File) error { + err := unix.IoctlFileClone(int(dst.Fd()), int(src.Fd())) + if err == nil { + return nil + } + + _, err = io.Copy(dst, src) + return err +} diff --git a/vendor/github.com/containers/storage/pkg/fileutils/reflink_unsupported.go b/vendor/github.com/containers/storage/pkg/fileutils/reflink_unsupported.go new file mode 100644 index 0000000000..c0a30e670c --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/fileutils/reflink_unsupported.go @@ -0,0 +1,15 @@ +//go:build !linux + +package fileutils + +import ( + "io" + "os" +) + +// ReflinkOrCopy attempts to reflink the source to the destination fd. +// If reflinking fails or is unsupported, it falls back to io.Copy(). +func ReflinkOrCopy(src, dst *os.File) error { + _, err := io.Copy(dst, src) + return err +} diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go index b45a6819a2..5c473cfa4c 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go @@ -98,7 +98,7 @@ func IsSetID(path string, modeid os.FileMode, capid capability.Cap) (bool, error return cap.Get(capability.EFFECTIVE, capid), nil } -func (c *Cmd) Start() error { +func (c *Cmd) Start() (retErr error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -167,6 +167,15 @@ func (c *Cmd) Start() error { return err } + // If the function fails from here, we need to make sure the + // child process is killed and properly cleaned up. + defer func() { + if retErr != nil { + _ = c.Cmd.Process.Kill() + _ = c.Cmd.Wait() + } + }() + // Close the ends of the pipes that the parent doesn't need. continueRead.Close() continueRead = nil diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index cd1cf861f6..053c31c426 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -18,6 +18,7 @@ import ( // register all of the built-in drivers _ "github.com/containers/storage/drivers/register" + "golang.org/x/sync/errgroup" drivers "github.com/containers/storage/drivers" "github.com/containers/storage/internal/dedup" @@ -30,7 +31,6 @@ import ( "github.com/containers/storage/pkg/stringutils" "github.com/containers/storage/pkg/system" "github.com/containers/storage/types" - "github.com/hashicorp/go-multierror" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux/label" "github.com/sirupsen/logrus" @@ -2744,7 +2744,7 @@ func (s *store) DeleteContainer(id string) error { } } - var wg multierror.Group + var wg errgroup.Group middleDir := s.graphDriverName + "-containers" @@ -2759,7 +2759,7 @@ func (s *store) DeleteContainer(id string) error { }) if multierr := wg.Wait(); multierr != nil { - return multierr.ErrorOrNil() + return multierr } return s.containerStore.Delete(id) }) diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go b/vendor/github.com/vbatts/tar-split/archive/tar/writer.go index e80498d03e..893eac00ae 100644 --- a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go +++ b/vendor/github.com/vbatts/tar-split/archive/tar/writer.go @@ -199,6 +199,9 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error { flag = TypeXHeader } data := buf.String() + if len(data) > maxSpecialFileSize { + return ErrFieldTooLong + } if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal { return err // Global headers return here } diff --git a/vendor/modules.txt b/vendor/modules.txt index 9c8db4acb1..9070c8a323 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -364,7 +364,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.57.1 +# github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3 ## explicit; go 1.22.0 github.com/containers/storage github.com/containers/storage/drivers @@ -1105,7 +1105,7 @@ github.com/ulikunitz/xz github.com/ulikunitz/xz/internal/hash github.com/ulikunitz/xz/internal/xlog github.com/ulikunitz/xz/lzma -# github.com/vbatts/tar-split v0.11.7 +# github.com/vbatts/tar-split v0.12.1 ## explicit; go 1.17 github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm From 3b5d7d1e644b4a2937eb21612fd3490d669f4268 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 12 Feb 2025 15:07:32 +0100 Subject: [PATCH 2/2] artifact extract: support reflink copy When the fs supports reflinks use that over a normal copy, this speeds things up a lot when big files are used. Signed-off-by: Paul Holzinger --- pkg/libartifact/store/store.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/libartifact/store/store.go b/pkg/libartifact/store/store.go index f95cbb26fa..c49916b924 100644 --- a/pkg/libartifact/store/store.go +++ b/pkg/libartifact/store/store.go @@ -411,7 +411,12 @@ func copyImageBlobToFile(ctx context.Context, imgSrc types.ImageSource, digest d } defer dest.Close() - // TODO use reflink is possible + // By default the c/image oci layout API for GetBlob() should always return a os.File in our usage here. + // And since it is a file we can try to reflink it. In case it is not we should default to the normal copy. + if file, ok := src.(*os.File); ok { + return fileutils.ReflinkOrCopy(file, dest) + } + _, err = io.Copy(dest, src) return err }