From 3925a30fa7050df35a71b3789e541347c9823b0a Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 5 Feb 2025 18:49:10 +0100 Subject: [PATCH] add podman artifact extract Add a new command to extract the blob content of the artifact store to a local path. Fixes https://issues.redhat.com/browse/RUN-2445 Signed-off-by: Paul Holzinger --- cmd/podman/artifact/extract.go | 52 +++++ .../markdown/podman-artifact-extract.1.md | 83 ++++++++ docs/source/markdown/podman-artifact.1.md | 1 + pkg/domain/entities/artifact.go | 9 + pkg/domain/entities/engine_image.go | 1 + pkg/domain/infra/abi/artifact.go | 13 ++ pkg/domain/infra/tunnel/artifact.go | 2 +- pkg/libartifact/store/store.go | 162 ++++++++++++++++ pkg/libartifact/types/config.go | 7 + test/e2e/artifact_test.go | 183 ++++++++++++++++++ test/e2e/common_test.go | 7 + 11 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 cmd/podman/artifact/extract.go create mode 100644 docs/source/markdown/podman-artifact-extract.1.md diff --git a/cmd/podman/artifact/extract.go b/cmd/podman/artifact/extract.go new file mode 100644 index 0000000000..ed17e085b3 --- /dev/null +++ b/cmd/podman/artifact/extract.go @@ -0,0 +1,52 @@ +package artifact + +import ( + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v5/cmd/podman/common" + "github.com/containers/podman/v5/cmd/podman/registry" + "github.com/containers/podman/v5/pkg/domain/entities" + "github.com/spf13/cobra" +) + +var ( + extractCmd = &cobra.Command{ + Use: "extract [options] ARTIFACT PATH", + Short: "Extract an OCI artifact to a local path", + Long: "Extract the blobs of an OCI artifact to a local file or directory", + RunE: extract, + Args: cobra.ExactArgs(2), + ValidArgsFunction: common.AutocompleteArtifactAdd, + Example: `podman artifact Extract quay.io/myimage/myartifact:latest /tmp/foobar.txt +podman artifact Extract quay.io/myimage/myartifact:latest /home/paul/mydir`, + Annotations: map[string]string{registry.EngineMode: registry.ABIMode}, + } +) + +var ( + extractOpts entities.ArtifactExtractOptions +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: extractCmd, + Parent: artifactCmd, + }) + flags := extractCmd.Flags() + + digestFlagName := "digest" + flags.StringVar(&extractOpts.Digest, digestFlagName, "", "Only extract blob with the given digest") + _ = extractCmd.RegisterFlagCompletionFunc(digestFlagName, completion.AutocompleteNone) + + titleFlagName := "title" + flags.StringVar(&extractOpts.Title, titleFlagName, "", "Only extract blob with the given title") + _ = extractCmd.RegisterFlagCompletionFunc(titleFlagName, completion.AutocompleteNone) +} + +func extract(cmd *cobra.Command, args []string) error { + err := registry.ImageEngine().ArtifactExtract(registry.Context(), args[0], args[1], &extractOpts) + if err != nil { + return err + } + + return nil +} diff --git a/docs/source/markdown/podman-artifact-extract.1.md b/docs/source/markdown/podman-artifact-extract.1.md new file mode 100644 index 0000000000..86aa6d888c --- /dev/null +++ b/docs/source/markdown/podman-artifact-extract.1.md @@ -0,0 +1,83 @@ +% podman-artifact-extract 1 + + +## WARNING: Experimental command +*This command is considered experimental and still in development. Inputs, options, and outputs are all +subject to change.* + +## NAME +podman\-artifact\-extract - Extract an OCI artifact to a local path + +## SYNOPSIS +**podman artifact extract** *artifact* *target* + +## DESCRIPTION + +Extract the blobs of an OCI artifact to a local file or directory. + +If the target path is a file or does not exist, the artifact must either consist +of one blob (layer) or if it has multiple blobs (layers) then the **--digest** or +**--title** option must be used to select only a single blob. If the file already +exists it will be overwritten. + +If the target is a directory (it must exist), all blobs will be copied to the +target directory. As the target file name the value from the `org.opencontainers.image.title` +annotation is used. If the annotation is missing, the target file name will be the +digest of the blob (with `:` replaced by `-` in the name). +If the target file already exists in the directory, it will be overwritten. + +## OPTIONS + +#### **--digest**=**digest** + +When extracting blobs from the artifact only use the one with the specified digest. +If the target is a directory then the digest is always used as file name instead even +when the title annotation exists on the blob. +Conflicts with **--title**. + +#### **--help** + +Print usage statement. + +#### **--title**=**title** + +When extracting blobs from the artifact only use the one with the specified title. +It looks for the `org.opencontainers.image.title` annotation and compares that +against the given title. +Conflicts with **--digest**. + +## EXAMPLES + +Extract an artifact with a single blob + +``` +$ podman artifact extract quay.io/artifact/foobar1:test /tmp/myfile +``` + +Extract an artifact with multiple blobs + +``` +$ podman artifact extract quay.io/artifact/foobar2:test /tmp/mydir +$ ls /tmp/mydir +CONTRIBUTING.md README.md +``` + +Extract only a single blob from an artifact with multiple blobs + +``` +$ podman artifact extract --title README.md quay.io/artifact/foobar2:test /tmp/mydir +$ ls /tmp/mydir +README.md +``` +Or using the digest instead of the title +``` +$ podman artifact extract --digest sha256:c0594e012b17fd9e6548355ceb571a79613f7bb988d7d883f112513601ac6e9a quay.io/artifact/foobar2:test /tmp/mydir +$ ls /tmp/mydir +README.md +``` + +## SEE ALSO +**[podman(1)](podman.1.md)**, **[podman-artifact(1)](podman-artifact.1.md)** + +## HISTORY +Feb 2025, Originally compiled by Paul Holzinger diff --git a/docs/source/markdown/podman-artifact.1.md b/docs/source/markdown/podman-artifact.1.md index 8597464edb..a6b16ab199 100644 --- a/docs/source/markdown/podman-artifact.1.md +++ b/docs/source/markdown/podman-artifact.1.md @@ -22,6 +22,7 @@ from its local "artifact store". | Command | Man Page | Description | |---------|------------------------------------------------------------|--------------------------------------------------------------| | add | [podman-artifact-add(1)](podman-artifact-add.1.md) | Add an OCI artifact to the local store | +| extract | [podman-artifact-extract(1)](podman-artifact-extract.1.md) | Extract an OCI artifact to a local path | | inspect | [podman-artifact-inspect(1)](podman-artifact-inspect.1.md) | Inspect an OCI artifact | | ls | [podman-artifact-ls(1)](podman-artifact-ls.1.md) | List OCI artifacts in local store | | pull | [podman-artifact-pull(1)](podman-artifact-pull.1.md) | Pulls an artifact from a registry and stores it locally | diff --git a/pkg/domain/entities/artifact.go b/pkg/domain/entities/artifact.go index 5103401e8e..e7a110fe51 100644 --- a/pkg/domain/entities/artifact.go +++ b/pkg/domain/entities/artifact.go @@ -14,6 +14,15 @@ type ArtifactAddOptions struct { ArtifactType string } +type ArtifactExtractOptions struct { + // Title annotation value to extract only a single blob matching that name. + // Conflicts with Digest. Optional. + Title string + // Digest of the blob to extract. + // Conflicts with Title. Optional. + Digest string +} + type ArtifactInspectOptions struct { Remote bool } diff --git a/pkg/domain/entities/engine_image.go b/pkg/domain/entities/engine_image.go index d93883b68b..796911b108 100644 --- a/pkg/domain/entities/engine_image.go +++ b/pkg/domain/entities/engine_image.go @@ -10,6 +10,7 @@ import ( type ImageEngine interface { //nolint:interfacebloat ArtifactAdd(ctx context.Context, name string, paths []string, opts *ArtifactAddOptions) (*ArtifactAddReport, error) + ArtifactExtract(ctx context.Context, name string, target string, opts *ArtifactExtractOptions) error ArtifactInspect(ctx context.Context, name string, opts ArtifactInspectOptions) (*ArtifactInspectReport, error) ArtifactList(ctx context.Context, opts ArtifactListOptions) ([]*ArtifactListReport, error) ArtifactPull(ctx context.Context, name string, opts ArtifactPullOptions) (*ArtifactPullReport, error) diff --git a/pkg/domain/infra/abi/artifact.go b/pkg/domain/infra/abi/artifact.go index fe1c3c12c8..76ea1f01ae 100644 --- a/pkg/domain/infra/abi/artifact.go +++ b/pkg/domain/infra/abi/artifact.go @@ -172,3 +172,16 @@ func (ir *ImageEngine) ArtifactAdd(ctx context.Context, name string, paths []str ArtifactDigest: artifactDigest, }, nil } + +func (ir *ImageEngine) ArtifactExtract(ctx context.Context, name string, target string, opts *entities.ArtifactExtractOptions) error { + artStore, err := store.NewArtifactStore(getDefaultArtifactStore(ir), ir.Libpod.SystemContext()) + if err != nil { + return err + } + extractOpt := &types.ExtractOptions{ + Digest: opts.Digest, + Title: opts.Title, + } + + return artStore.Extract(ctx, name, target, extractOpt) +} diff --git a/pkg/domain/infra/tunnel/artifact.go b/pkg/domain/infra/tunnel/artifact.go index c404da95f0..c55a328fe6 100644 --- a/pkg/domain/infra/tunnel/artifact.go +++ b/pkg/domain/infra/tunnel/artifact.go @@ -9,7 +9,7 @@ import ( // TODO For now, no remote support has been added. We need the API to firm up first. -func ArtifactAdd(ctx context.Context, path, name string, opts entities.ArtifactAddOptions) error { +func (ir *ImageEngine) ArtifactExtract(ctx context.Context, name string, target string, opts *entities.ArtifactExtractOptions) error { return fmt.Errorf("not implemented") } diff --git a/pkg/libartifact/store/store.go b/pkg/libartifact/store/store.go index 8a0364c247..f95cbb26fa 100644 --- a/pkg/libartifact/store/store.go +++ b/pkg/libartifact/store/store.go @@ -8,10 +8,12 @@ import ( "errors" "fmt" "io" + "io/fs" "maps" "net/http" "os" "path/filepath" + "strings" "github.com/containers/common/libimage" "github.com/containers/image/v5/manifest" @@ -254,6 +256,166 @@ func (as ArtifactStore) Add(ctx context.Context, dest string, paths []string, op return &artifactManifestDigest, nil } +// Inspect an artifact in a local store +func (as ArtifactStore) Extract(ctx context.Context, nameOrDigest string, target string, options *libartTypes.ExtractOptions) error { + if len(options.Digest) > 0 && len(options.Title) > 0 { + return errors.New("cannot specify both digest and title") + } + if len(nameOrDigest) == 0 { + return ErrEmptyArtifactName + } + + artifacts, err := as.getArtifacts(ctx, nil) + if err != nil { + return err + } + + arty, nameIsDigest, err := artifacts.GetByNameOrDigest(nameOrDigest) + if err != nil { + return err + } + name := nameOrDigest + if nameIsDigest { + name = arty.Name + } + + if len(arty.Manifest.Layers) == 0 { + return fmt.Errorf("the artifact has no blobs, nothing to extract") + } + + ir, err := layout.NewReference(as.storePath, name) + if err != nil { + return err + } + imgSrc, err := ir.NewImageSource(ctx, as.SystemContext) + if err != nil { + return err + } + defer imgSrc.Close() + + // check if dest is a dir to know if we can copy more than one blob + destIsFile := true + stat, err := os.Stat(target) + if err == nil { + destIsFile = !stat.IsDir() + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } + + if destIsFile { + var digest digest.Digest + if len(arty.Manifest.Layers) > 1 { + if len(options.Digest) == 0 && len(options.Title) == 0 { + return fmt.Errorf("the artifact consists of several blobs and the target %q is not a directory and neither digest or title was specified to only copy a single blob", target) + } + digest, err = findDigest(arty, options) + if err != nil { + return err + } + } else { + digest = arty.Manifest.Layers[0].Digest + } + + return copyImageBlobToFile(ctx, imgSrc, digest, target) + } + + if len(options.Digest) > 0 || len(options.Title) > 0 { + digest, err := findDigest(arty, options) + if err != nil { + return err + } + // In case the digest is set we always use it as target name + // so we do not have to get the actual title annotation form the blob. + // Passing options.Title is enough because we know it is empty when digest + // is set as we only allow either one. + filename, err := generateArtifactBlobName(options.Title, digest) + if err != nil { + return err + } + return copyImageBlobToFile(ctx, imgSrc, digest, filepath.Join(target, filename)) + } + + for _, l := range arty.Manifest.Layers { + title := l.Annotations[specV1.AnnotationTitle] + filename, err := generateArtifactBlobName(title, l.Digest) + if err != nil { + return err + } + err = copyImageBlobToFile(ctx, imgSrc, l.Digest, filepath.Join(target, filename)) + if err != nil { + return err + } + } + + return nil +} + +func generateArtifactBlobName(title string, digest digest.Digest) (string, error) { + filename := title + if len(filename) == 0 { + // No filename given, use the digest. But because ":" is not a valid path char + // on all platforms replace it with "-". + filename = strings.ReplaceAll(digest.String(), ":", "-") + } + + // Important: A potentially malicious artifact could contain a title name with "/" + // and could try via relative paths such as "../" try to overwrite files on the host + // the user did not intend. As there is no use for directories in this path we + // disallow all of them and not try to "make it safe" via securejoin or others. + // We must use os.IsPathSeparator() as on Windows it checks both "\\" and "/". + for i := 0; i < len(filename); i++ { + if os.IsPathSeparator(filename[i]) { + return "", fmt.Errorf("invalid name: %q cannot contain %c", filename, filename[i]) + } + } + return filename, nil +} + +func findDigest(arty *libartifact.Artifact, options *libartTypes.ExtractOptions) (digest.Digest, error) { + var digest digest.Digest + for _, l := range arty.Manifest.Layers { + if options.Digest == l.Digest.String() { + if len(digest.String()) > 0 { + return digest, fmt.Errorf("more than one match for the digest %q", options.Digest) + } + digest = l.Digest + } + if len(options.Title) > 0 { + if val, ok := l.Annotations[specV1.AnnotationTitle]; ok && + val == options.Title { + if len(digest.String()) > 0 { + return digest, fmt.Errorf("more than one match for the title %q", options.Title) + } + digest = l.Digest + } + } + } + if len(digest.String()) == 0 { + if len(options.Title) > 0 { + return digest, fmt.Errorf("no blob with the title %q", options.Title) + } + return digest, fmt.Errorf("no blob with the digest %q", options.Digest) + } + return digest, nil +} + +func copyImageBlobToFile(ctx context.Context, imgSrc types.ImageSource, digest digest.Digest, target string) error { + src, _, err := imgSrc.GetBlob(ctx, types.BlobInfo{Digest: digest}, nil) + if err != nil { + return fmt.Errorf("failed to get artifact file: %w", err) + } + defer src.Close() + dest, err := os.Create(target) + if err != nil { + return fmt.Errorf("failed to create target file: %w", err) + } + defer dest.Close() + + // TODO use reflink is possible + _, err = io.Copy(dest, src) + return err +} + // readIndex is currently unused but I want to keep this around until // the artifact code is more mature. func (as ArtifactStore) readIndex() (*specV1.Index, error) { //nolint:unused diff --git a/pkg/libartifact/types/config.go b/pkg/libartifact/types/config.go index c458e646a6..932f75d855 100644 --- a/pkg/libartifact/types/config.go +++ b/pkg/libartifact/types/config.go @@ -9,3 +9,10 @@ type AddOptions struct { Annotations map[string]string `json:"annotations,omitempty"` ArtifactType string `json:",omitempty"` } + +type ExtractOptions struct { + // Title annotation value to extract only a single blob matching that name. Optional. + Title string + // Digest of the blob to extract. Optional. + Digest string +} diff --git a/test/e2e/artifact_test.go b/test/e2e/artifact_test.go index c3cfe504eb..4cd146eb97 100644 --- a/test/e2e/artifact_test.go +++ b/test/e2e/artifact_test.go @@ -5,6 +5,9 @@ package integration import ( "encoding/json" "fmt" + "os" + "path/filepath" + "strings" "github.com/containers/podman/v5/pkg/libartifact" . "github.com/containers/podman/v5/test/utils" @@ -13,6 +16,17 @@ import ( . "github.com/onsi/gomega/gexec" ) +const ( + //nolint:revive,stylecheck + ARTIFACT_SINGLE = "quay.io/libpod/testartifact:20250206-single" + //nolint:revive,stylecheck + ARTIFACT_MULTI = "quay.io/libpod/testartifact:20250206-multi" + //nolint:revive,stylecheck + ARTIFACT_MULTI_NO_TITLE = "quay.io/libpod/testartifact:20250206-multi-no-title" + //nolint:revive,stylecheck + ARTIFACT_EVIL = "quay.io/libpod/testartifact:20250206-evil" +) + var _ = Describe("Podman artifact", func() { BeforeEach(func() { SkipIfRemote("artifacts are not supported on the remote client yet due to being in development still") @@ -196,4 +210,173 @@ var _ = Describe("Podman artifact", func() { podmanTest.PodmanExitCleanly([]string{"artifact", "inspect", artifactDigest[:12]}...) }) + + It("podman artifact extract single", func() { + podmanTest.PodmanExitCleanly("artifact", "pull", ARTIFACT_SINGLE) + + const ( + artifactContent = "mRuO9ykak1Q2j\n" + artifactDigest = "sha256:e9510923578af3632946ecf5ae479c1b5f08b47464e707b5cbab9819272a9752" + artifactTitle = "testfile" + ) + + path := filepath.Join(podmanTest.TempDir, "testfile") + // Extract to non existing file + podmanTest.PodmanExitCleanly("artifact", "extract", ARTIFACT_SINGLE, path) + Expect(readFileToString(path)).To(Equal(artifactContent)) + + // Extract to existing file will overwrite file + path = filepath.Join(podmanTest.TempDir, "abcd") + f, err := os.Create(path) + Expect(err).ToNot(HaveOccurred()) + f.Close() + podmanTest.PodmanExitCleanly("artifact", "extract", ARTIFACT_SINGLE, path) + Expect(readFileToString(path)).To(Equal(artifactContent)) + + tests := []struct { + name string + filename string + extraArgs []string + }{ + { + name: "extract to dir", + filename: artifactTitle, + }, + { + name: "extract to dir by digest", + filename: digestToFilename(artifactDigest), + extraArgs: []string{"--digest", artifactDigest}, + }, + { + name: "extract to dir by title", + filename: artifactTitle, + extraArgs: []string{"--title", artifactTitle}, + }, + } + + for _, tt := range tests { + By(tt.name) + dir := makeTempDirInDir(podmanTest.TempDir) + args := append([]string{"artifact", "extract"}, tt.extraArgs...) + args = append(args, ARTIFACT_SINGLE, dir) + podmanTest.PodmanExitCleanly(args...) + Expect(readFileToString(filepath.Join(dir, tt.filename))).To(Equal(artifactContent)) + } + + // invalid digest + session := podmanTest.Podman([]string{"artifact", "extract", "--digest", "blah", ARTIFACT_SINGLE, podmanTest.TempDir}) + session.WaitWithDefaultTimeout() + Expect(session).To(ExitWithError(125, `no blob with the digest "blah"`)) + + // invalid title + session = podmanTest.Podman([]string{"artifact", "extract", "--title", "abcd", ARTIFACT_SINGLE, podmanTest.TempDir}) + session.WaitWithDefaultTimeout() + Expect(session).To(ExitWithError(125, `no blob with the title "abcd"`)) + }) + + It("podman artifact extract multi", func() { + podmanTest.PodmanExitCleanly("artifact", "pull", ARTIFACT_MULTI) + podmanTest.PodmanExitCleanly("artifact", "pull", ARTIFACT_MULTI_NO_TITLE) + + const ( + artifactContent1 = "xuHWedtC0ADST\n" + artifactDigest1 = "sha256:8257bba28b9d19ac353c4b713b470860278857767935ef7e139afd596cb1bb2d" + artifactTitle1 = "test1" + artifactContent2 = "tAyZczFlgFsi4\n" + artifactDigest2 = "sha256:63700c54129c6daaafe3a20850079f82d6d658d69de73d6158d81f920c6fbdd7" + artifactTitle2 = "test2" + ) + + type expect struct { + filename string + content string + } + tests := []struct { + name string + image string + extraArgs []string + expect []expect + }{ + { + name: "extract multi blob to dir", + image: ARTIFACT_MULTI, + expect: []expect{ + {filename: artifactTitle1, content: artifactContent1}, + {filename: artifactTitle2, content: artifactContent2}, + }, + }, + { + name: "extract multi blob to dir without title", + image: ARTIFACT_MULTI_NO_TITLE, + expect: []expect{ + {filename: digestToFilename(artifactDigest1), content: artifactContent1}, + {filename: digestToFilename(artifactDigest2), content: artifactContent2}, + }, + }, + { + name: "extract multi blob to dir with --title", + image: ARTIFACT_MULTI, + extraArgs: []string{"--title", artifactTitle1}, + expect: []expect{ + {filename: artifactTitle1, content: artifactContent1}, + }, + }, + { + name: "extract multi blob to dir with --digest", + image: ARTIFACT_MULTI, + extraArgs: []string{"--digest", artifactDigest2}, + expect: []expect{ + {filename: digestToFilename(artifactDigest2), content: artifactContent2}, + }, + }, + } + + for _, tt := range tests { + By(tt.name) + dir := makeTempDirInDir(podmanTest.TempDir) + args := append([]string{"artifact", "extract"}, tt.extraArgs...) + args = append(args, tt.image, dir) + podmanTest.PodmanExitCleanly(args...) + files, err := os.ReadDir(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(files).To(HaveLen(len(tt.expect))) + for _, expect := range tt.expect { + Expect(readFileToString(filepath.Join(dir, expect.filename))).To(Equal(expect.content)) + } + } + }) + + It("podman artifact extract evil", func() { + path := filepath.Join(podmanTest.TempDir, "testfile") + podmanTest.PodmanExitCleanly("artifact", "pull", ARTIFACT_EVIL) + + const ( + artifactContent = "RM5eA27F9psa2\n" + artifactDigest = "sha256:4c29da41ff27fcbf273653bcfba58ed69efa4aefec7b6c486262711cb1dfd050" + ) + + // Extract to file is fine as we are not using the malicious title + podmanTest.PodmanExitCleanly("artifact", "extract", ARTIFACT_EVIL, path) + Expect(readFileToString(path)).To(Equal(artifactContent)) + + // This must fail for security reasons we do not allow a title with / + session := podmanTest.Podman([]string{"artifact", "extract", ARTIFACT_EVIL, podmanTest.TempDir}) + session.WaitWithDefaultTimeout() + Expect(session).To(ExitWithError(125, `invalid name: "../../../../tmp/evil" cannot contain /`)) + + // Extracting by digest should be fine too + podmanTest.PodmanExitCleanly("artifact", "extract", "--digest", artifactDigest, ARTIFACT_EVIL, podmanTest.TempDir) + Expect(readFileToString(filepath.Join(podmanTest.TempDir, digestToFilename(artifactDigest)))).To(Equal(artifactContent)) + }) }) + +func digestToFilename(digest string) string { + return strings.ReplaceAll(digest, ":", "-") +} + +func readFileToString(path string) string { + GinkgoHelper() + b, err := os.ReadFile(path) + Expect(err).ToNot(HaveOccurred()) + return string(b) +} diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 9417653839..2d6ce7bd7a 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -1610,3 +1610,10 @@ func createArtifactFile(numBytes int64) (string, error) { } return outFile, nil } + +func makeTempDirInDir(dir string) string { + GinkgoHelper() + path, err := os.MkdirTemp(dir, "podman-test") + Expect(err).ToNot(HaveOccurred()) + return path +}