From 9ed3b61f023b2b4a941f0c57d26ff8a9d621e632 Mon Sep 17 00:00:00 2001 From: "Casper S. Jensen" Date: Fri, 7 Feb 2020 00:05:50 +0100 Subject: [PATCH] Add push command (#302) * Add push command Co-authored-by: Yiran Wang --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 1 + bin/makisu/cmd/build.go | 6 +- bin/makisu/cmd/push.go | 265 ++++++++++++++++++ bin/makisu/cmd/root.go | 1 + bin/makisu/cmd/utils.go | 11 +- go.sum | 1 + lib/builder/build_stage.go | 2 +- lib/docker/cli/image.go | 18 +- .../image/distribution_manifest_test.go | 31 +- lib/docker/image/export_manifest_test.go | 8 +- lib/storage/image_store.go | 32 +++ lib/tario/untar.go | 143 ++++++++++ test/python/test_build.py | 52 ++-- test/python/test_push.py | 25 ++ test/python/utils.py | 60 +++- .../json | 0 .../layer.tar | Bin 18 files changed, 579 insertions(+), 77 deletions(-) delete mode 100644 .DS_Store create mode 100644 bin/makisu/cmd/push.go create mode 100644 lib/tario/untar.go create mode 100644 test/python/test_push.py rename testdata/files/busybox/{4ac76077f2c741c856a2419dfdb0804b18e48d2e1a9ce9c6a3f0605a2078caba => 393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b}/json (100%) rename testdata/files/busybox/{4ac76077f2c741c856a2419dfdb0804b18e48d2e1a9ce9c6a3f0605a2078caba => 393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b}/layer.tar (100%) diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 4abe6ef7924e2029f6741da260ee5eac580db3c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%TC-d6g}>QA)RjM0+nT!-GJ0}Hz}Q2AeAoQ2RbjRhztQ{s9={(|DnIoKkJ?k zfOD@+WjiU$s;YcP_D!5~uYHakdjh~5-Ynh$I{9ed$gcIl1g*kFBB zH}hHD%sS}QkQ7J?{A&uxyF0`bYyOr9-(T4j-m|=YP}89gFhhk4?=2V9&j>Zb7!Bq) zXU=P!F`~vYj4^MbnAd$VIN;c7TuPsv4Bw*+85Y{BzXx%2=03vZ9MvuJu9%nO3ihlg z&ObcPLwsf41L7MjLKe!r54*+H!pcW&;}eeX5fgmIDJBtGxcTTD z$7CkD_o^=QX>yO-h^)?^bR)+*-q&Ft!ZC>1QYB<|W2HH#0rDkIIPMRe*<*T9Fw)PE zZ4j}`^|Hhj)vNSs#k&hxm)Q44x#9e6!codB2bs1RGiQ#9v)sF`&8??HMs!77v`UqG zh$DvBxOZXh<8sL*IF5SCdK!&Zc>DM*VjBMHj;6xPM-K5#HIcP`9cIL{QVyW zrMsj+QsBQ*z+~go@gXU>v$aD`&f0?I3yYZGTI)50l{=30LXP4+7BxOA [flags] ", + Use: "build -t= [flags] ", DisableFlagsInUseLine: true, - Short: "Build docker image, optionally push to registries and/or load into docker daemon", + Short: "Build docker image, optionally push to registries and/or load into docker daemon", }, } buildCmd.Args = func(cmd *cobra.Command, args []string) error { @@ -151,7 +151,7 @@ func (cmd *buildCmd) processFlags() error { return fmt.Errorf("invalid commit option: %s", cmd.commit) } - if err := cmd.initRegistryConfig(); err != nil { + if err := initRegistryConfig(cmd.registryConfig); err != nil { return fmt.Errorf("failed to initialize registry configuration: %s", err) } diff --git a/bin/makisu/cmd/push.go b/bin/makisu/cmd/push.go new file mode 100644 index 00000000..e9da82f3 --- /dev/null +++ b/bin/makisu/cmd/push.go @@ -0,0 +1,265 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/uber/makisu/lib/docker/image" + "github.com/uber/makisu/lib/log" + "github.com/uber/makisu/lib/registry" + "github.com/uber/makisu/lib/storage" + "github.com/uber/makisu/lib/tario" + "github.com/uber/makisu/lib/utils" + + "github.com/spf13/cobra" +) + +type pushCmd struct { + *cobra.Command + + tag string + + pushRegistries []string + replicas []string + registryConfig string +} + +func getPushCmd() *pushCmd { + pushCmd := &pushCmd{ + Command: &cobra.Command{ + Use: "push -t= [flags] ", + DisableFlagsInUseLine: true, + Short: "Push docker image to registries", + }, + } + pushCmd.Args = func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("Requires image tar path as argument") + } + return nil + } + pushCmd.Run = func(cmd *cobra.Command, args []string) { + if err := pushCmd.processFlags(); err != nil { + log.Errorf("failed to process flags: %s", err) + os.Exit(1) + } + + if err := pushCmd.Push(args[0]); err != nil { + log.Error(err) + os.Exit(1) + } + } + + pushCmd.PersistentFlags().StringVarP(&pushCmd.tag, "tag", "t", "", "Image tag (required)") + + pushCmd.PersistentFlags().StringArrayVar(&pushCmd.pushRegistries, "push", nil, "Registry to push image to") + pushCmd.PersistentFlags().StringArrayVar(&pushCmd.replicas, "replica", nil, "Push targets with alternative full image names \"/:\"") + pushCmd.PersistentFlags().StringVar(&pushCmd.registryConfig, "registry-config", "", "Set build-time variables") + + pushCmd.MarkFlagRequired("tag") + pushCmd.Flags().SortFlags = false + pushCmd.PersistentFlags().SortFlags = false + + return pushCmd +} + +func (cmd *pushCmd) processFlags() error { + if err := initRegistryConfig(cmd.registryConfig); err != nil { + return fmt.Errorf("failed to initialize registry configuration: %s", err) + } + + return nil +} + +// Push image tar to docker registries. +func (cmd *pushCmd) Push(imageTarPath string) error { + log.Infof("Starting Makisu push (version=%s)", utils.BuildHash) + + imageName, err := cmd.getTargetImageName() + if err != nil { + return err + } + + // TODO: make configurable? + store, err := storage.NewImageStore("/tmp/makisu-storage") + if err != nil { + return fmt.Errorf("unable to create internal store: %s", err) + } + + if err := cmd.loadImageTarIntoStore(store, imageName, cmd.replicas, imageTarPath); err != nil { + return fmt.Errorf("unable to import image: %s", err) + } + + // Push image to registries that were specified in the --push flag. + for _, registry := range cmd.pushRegistries { + target := imageName.WithRegistry(registry) + if err := cmd.pushImage(store, target); err != nil { + return fmt.Errorf("failed to push image: %s", err) + } + } + for _, replica := range cmd.replicas { + target := image.MustParseName(replica) + if err := cmd.pushImage(store, target); err != nil { + return fmt.Errorf("failed to push image: %s", err) + } + } + + log.Infof("Finished pushing %s", imageName.ShortName()) + return nil +} + +func (cmd *pushCmd) getTargetImageName() (image.Name, error) { + if cmd.tag == "" { + msg := "please specify a target image name: push -t= [flags] " + return image.Name{}, errors.New(msg) + } + + return image.MustParseName(cmd.tag), nil +} + +func (cmd *pushCmd) loadImageTarIntoStore( + store *storage.ImageStore, imageName image.Name, replicas []string, imageTarPath string) error { + + if err := cmd.importTar(store, imageName, replicas, imageTarPath); err != nil { + return fmt.Errorf("import image tar: %s", err) + } + + return nil +} + +func (cmd *pushCmd) pushImage(store *storage.ImageStore, imageName image.Name) error { + registryClient := registry.New(store, imageName.GetRegistry(), imageName.GetRepository()) + if err := registryClient.Push(imageName.GetTag()); err != nil { + return fmt.Errorf("failed to push image: %s", err) + } + log.Infof("Successfully pushed %s to %s", imageName, imageName.GetRegistry()) + return nil +} + +// importTar imports an image, as a tar, to the image store. +func (cmd *pushCmd) importTar( + store *storage.ImageStore, imageName image.Name, replicas []string, tarPath string) error { + + repo, tag := imageName.GetRepository(), imageName.GetTag() + + // Extract tar into temporary directory. + dir := filepath.Join(store.SandboxDir, repo, tag) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("create unpack directory: %s", err) + } + defer os.RemoveAll(dir) + + reader, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("open tar file: %s", err) + } + defer reader.Close() + + if err := tario.Untar(reader, dir); err != nil { + return fmt.Errorf("unpack tar: %s", err) + } + + // Read manifest. + exportManifestPath := filepath.Join(dir, "manifest.json") + exportManifestData, err := ioutil.ReadFile(exportManifestPath) + if err != nil { + return fmt.Errorf("read export manifest: %s", err) + } + + var exportManifests []image.ExportManifest + if err := json.Unmarshal(exportManifestData, &exportManifests); err != nil { + return fmt.Errorf("unmarshal export manifest: %s", err) + } + + for _, exportManifest := range exportManifests { + // Import extracted dir content into image store -- {sha}.json. + configPath := filepath.Join(dir, exportManifest.Config.String()) + + configInfo, err := os.Stat(configPath) + if err != nil { + return fmt.Errorf("lookup config file info: %s", err) + } + + configReader, err := os.Open(configPath) + if err != nil { + return fmt.Errorf("open config json: %s", err) + } + defer configReader.Close() + configDigest, err := image.NewDigester().FromReader(configReader) + + if err := store.Layers.LinkStoreFileFrom( + configDigest.Hex(), configPath); err != nil && !os.IsExist(err) { + + return fmt.Errorf("commit config to store: %s", err) + } + + // Import extracted dir content into image store -- {sha}/layer.tar. + var layers []image.Descriptor + for _, layer := range exportManifest.Layers { + layerPath := path.Join(dir, layer.String()) + + layerInfo, err := os.Stat(layerPath) + if err != nil { + return fmt.Errorf("lookup layer file info: %s", err) + } + + layerReader, err := os.Open(layerPath) + if err != nil { + return fmt.Errorf("open layer tar: %s", err) + } + defer layerReader.Close() + layerDigest, err := image.NewDigester().FromReader(layerReader) + + if err := store.Layers.LinkStoreFileFrom( + layerDigest.Hex(), layerPath); err != nil && !os.IsExist(err) { + + return fmt.Errorf("commit layer to store: %s", err) + } + + layers = append(layers, image.Descriptor{ + MediaType: image.MediaTypeLayer, + Size: layerInfo.Size(), + Digest: layerDigest, + }) + } + + // Import extracted dir content into image store -- manifest.json. + distManifest := image.DistributionManifest{ + SchemaVersion: 2, + MediaType: image.MediaTypeManifest, + Config: image.Descriptor{ + MediaType: image.MediaTypeConfig, + Size: configInfo.Size(), + Digest: configDigest, + }, + Layers: layers, + } + store.SaveManifest(distManifest, imageName) + + for _, replica := range replicas { + parsed := image.MustParseName(replica) + store.SaveManifest(distManifest, parsed) + } + } + + return nil +} diff --git a/bin/makisu/cmd/root.go b/bin/makisu/cmd/root.go index 606bf36b..66e3504d 100644 --- a/bin/makisu/cmd/root.go +++ b/bin/makisu/cmd/root.go @@ -75,6 +75,7 @@ func Execute() { rootCmd.AddCommand(getBuildCmd().Command) rootCmd.AddCommand(getVersionCmd()) rootCmd.AddCommand(getPullCmd().Command) + rootCmd.AddCommand(getPushCmd().Command) if err := rootCmd.Execute(); err != nil { log.Error(err) os.Exit(1) diff --git a/bin/makisu/cmd/utils.go b/bin/makisu/cmd/utils.go index cdd2103c..b3589cd7 100644 --- a/bin/makisu/cmd/utils.go +++ b/bin/makisu/cmd/utils.go @@ -16,6 +16,7 @@ package cmd import ( ctx "context" + "errors" "fmt" "io/ioutil" "net/http" @@ -37,12 +38,12 @@ import ( "github.com/uber/makisu/lib/utils/stringset" ) -func (cmd *buildCmd) initRegistryConfig() error { - if cmd.registryConfig == "" { +func initRegistryConfig(registryConfig string) error { + if registryConfig == "" { return nil } - cmd.registryConfig = os.ExpandEnv(cmd.registryConfig) - if err := registry.UpdateGlobalConfig(cmd.registryConfig); err != nil { + registryConfig = os.ExpandEnv(registryConfig) + if err := registry.UpdateGlobalConfig(registryConfig); err != nil { return fmt.Errorf("init registry config: %s", err) } return nil @@ -89,7 +90,7 @@ func (cmd *buildCmd) getDockerfile(contextDir string) ([]*dockerfile.Stage, erro func (cmd *buildCmd) getTargetImageName() (image.Name, error) { if cmd.tag == "" { msg := "please specify a target image name: makisu build -t=(/): ./" - return image.Name{}, fmt.Errorf(msg) + return image.Name{}, errors.New(msg) } // Parse the target's image name into its components. diff --git a/go.sum b/go.sum index b33126d0..62c85f5c 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,7 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/lib/builder/build_stage.go b/lib/builder/build_stage.go index 440f9dbb..3c830acd 100644 --- a/lib/builder/build_stage.go +++ b/lib/builder/build_stage.go @@ -279,7 +279,7 @@ func (stage *buildStage) saveManifest( } manifestPath := manifestFile.Name() - // Remove temp file after hard-linked to manifest store + // Remove temp file after hard-linked to manifest store. defer os.Remove(manifestPath) if err := ioutil.WriteFile(manifestPath, manifestJSON, 0755); err != nil { diff --git a/lib/docker/cli/image.go b/lib/docker/cli/image.go index 9abeac98..d6228395 100644 --- a/lib/docker/cli/image.go +++ b/lib/docker/cli/image.go @@ -29,13 +29,7 @@ import ( "github.com/uber/makisu/lib/stream" ) -// ImageTarer contains a Tar function that returns a reader to the resulting -// tar file. -type ImageTarer interface { - Tar(registry, repo, tag string) (io.Reader, error) -} - -// DefaultImageTarer is the default implementation of the ImageTarer interface. +// DefaultImageTarer exports/imports images from an ImageStore. type DefaultImageTarer struct { store *storage.ImageStore } @@ -48,8 +42,8 @@ func NewDefaultImageTarer(store *storage.ImageStore) DefaultImageTarer { } } -// CreateTarReadCloser creates a new tar from the inputs and returns a reader -// that automatically closes on EOF. +// CreateTarReadCloser exports an image from the image store as a tar, and +// returns a reader for the tar that automatically closes on EOF. func (tarer DefaultImageTarer) CreateTarReadCloser(imageName image.Name) (io.Reader, error) { dir, err := tarer.createTarDir(imageName) if err != nil { @@ -75,8 +69,8 @@ func (tarer DefaultImageTarer) CreateTarReadCloser(imageName image.Name) (io.Rea return reader, nil } -// CreateTarReader creates a new tar from the inputs and returns a simple reader -// to that file. +// CreateTarReader exports an image from the image store as a tar, and returns a +// reader for the tar. func (tarer DefaultImageTarer) CreateTarReader(imageName image.Name) (io.Reader, error) { dir, err := tarer.createTarDir(imageName) if err != nil { @@ -104,8 +98,8 @@ func (tarer DefaultImageTarer) createTarDir(imageName image.Name) (string, error return "", err } - repo, tag := imageName.GetRepository(), imageName.GetTag() // Create tmp file for target tar. + repo, tag := imageName.GetRepository(), imageName.GetTag() dir := filepath.Join(tarer.store.SandboxDir, repo, tag) if err := os.MkdirAll(dir, 0755); err != nil { return "", err diff --git a/lib/docker/image/distribution_manifest_test.go b/lib/docker/image/distribution_manifest_test.go index d20ebd1c..37a3af21 100644 --- a/lib/docker/image/distribution_manifest_test.go +++ b/lib/docker/image/distribution_manifest_test.go @@ -20,25 +20,28 @@ import ( "github.com/stretchr/testify/require" ) -const testManifest = `{ - "schemaVersion": 2, - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "config": { - "mediaType": "application/vnd.docker.container.image.v1+json", - "size": 1503, - "digest": "sha256:79f4bda919894b2fe9a66f403337bdc0c547ac95183ec034a3a37869e17ee72e" +const busyboxDistManifest = `{ + "schemaVersion":2, + "mediaType":"application/vnd.docker.distribution.manifest.v2+json", + "config":{ + "mediaType":"application/vnd.docker.container.image.v1+json", + "size":1346, + "digest":"411a417c1f6ef5b93fac71c92276013f45762dde0bb36a80a6148ca114d1b0fa" }, - "layers": [ + "layers":[ { - "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "size": 54252125, - "digest": "sha256:d660b1f15b9bfb8142f50b518156f2d364d9642fe05854538b060498e2f7928d" + "mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip", + "size":1308672, + "digest":"393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b" } ] }` func TestUnmarshalDistributionManifest(t *testing.T) { - manifest, _, err := UnmarshalDistributionManifest(MediaTypeManifest, []byte(testManifest)) - require.NoError(t, err) - require.Equal(t, 1, len(manifest.GetLayerDigests())) + require := require.New(t) + + manifest, _, err := UnmarshalDistributionManifest( + MediaTypeManifest, []byte(busyboxDistManifest)) + require.NoError(err) + require.Equal(1, len(manifest.GetLayerDigests())) } diff --git a/lib/docker/image/export_manifest_test.go b/lib/docker/image/export_manifest_test.go index e3dcc2a9..8089c43d 100644 --- a/lib/docker/image/export_manifest_test.go +++ b/lib/docker/image/export_manifest_test.go @@ -21,12 +21,12 @@ import ( ) func TestExportManifest(t *testing.T) { - manifest, _, err := UnmarshalDistributionManifest(MediaTypeManifest, []byte(testManifest)) + manifest, _, err := UnmarshalDistributionManifest(MediaTypeManifest, []byte(busyboxDistManifest)) require.NoError(t, err) expManifest := NewExportManifestFromDistribution(Name{}, manifest) require.Equal(t, 1, len(expManifest.Layers)) layer := expManifest.Layers[0] - require.Equal(t, "d660b1f15b9bfb8142f50b518156f2d364d9642fe05854538b060498e2f7928d", layer.ID()) - require.Equal(t, "d660b1f15b9bfb8142f50b518156f2d364d9642fe05854538b060498e2f7928d/layer.tar", layer.String()) - require.Equal(t, "79f4bda919894b2fe9a66f403337bdc0c547ac95183ec034a3a37869e17ee72e", expManifest.Config.ID()) + require.Equal(t, "393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b", layer.ID()) + require.Equal(t, "393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b/layer.tar", layer.String()) + require.Equal(t, "411a417c1f6ef5b93fac71c92276013f45762dde0bb36a80a6148ca114d1b0fa", expManifest.Config.ID()) } diff --git a/lib/storage/image_store.go b/lib/storage/image_store.go index ac130f56..70bc2d59 100644 --- a/lib/storage/image_store.go +++ b/lib/storage/image_store.go @@ -15,10 +15,13 @@ package storage import ( + "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" + + "github.com/uber/makisu/lib/docker/image" ) // ImageStore contains a manifeststore, a layertarstore, and a sandbox dir. @@ -65,3 +68,32 @@ func CleanupSandbox(rootDir string) error { } return nil } + +func (store *ImageStore) SaveManifest( + distManifest image.DistributionManifest, imageName image.Name) error { + + distManifestJSON, err := json.Marshal(distManifest) + if err != nil { + return fmt.Errorf("marshal manifest to JSON: %s", err) + } + + distManifestFile, err := ioutil.TempFile(store.SandboxDir, "") + if err != nil { + return fmt.Errorf("create tmp manifest file: %s", err) + } + if _, err := distManifestFile.Write(distManifestJSON); err != nil { + return fmt.Errorf("write manifest file: %s", err) + } + if err := distManifestFile.Close(); err != nil { + return fmt.Errorf("close manifest file: %s", err) + } + + distManifestPath := distManifestFile.Name() + if err := store.Manifests.LinkStoreFileFrom( + imageName.GetRepository(), imageName.GetTag(), distManifestPath); err != nil && !os.IsExist(err) { + + return fmt.Errorf("commit replica manifest to store: %s", err) + } + + return nil +} diff --git a/lib/tario/untar.go b/lib/tario/untar.go new file mode 100644 index 00000000..8d6bdb1e --- /dev/null +++ b/lib/tario/untar.go @@ -0,0 +1,143 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tario + +import ( + "archive/tar" + "fmt" + "io" + "log" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +// Note: This is copied from https://github.com/golang/build/blob/master/internal/untar/untar.go +// Removed logic about gzip. + +// Untar reads the tar file from r and writes it into dir. +func Untar(r io.Reader, dir string) error { + return untar(r, dir) +} + +func untar(r io.Reader, dir string) (err error) { + t0 := time.Now() + nFiles := 0 + madeDir := map[string]bool{} + defer func() { + td := time.Since(t0) + if err == nil { + log.Printf("extracted tarball into %s: %d files, %d dirs (%v)", dir, nFiles, len(madeDir), td) + } else { + log.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err) + } + }() + tr := tar.NewReader(r) + loggedChtimesError := false + for { + f, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + log.Printf("tar reading error: %v", err) + return fmt.Errorf("tar error: %v", err) + } + if !validRelPath(f.Name) { + return fmt.Errorf("tar contained invalid name error %q", f.Name) + } + rel := filepath.FromSlash(f.Name) + abs := filepath.Join(dir, rel) + + fi := f.FileInfo() + mode := fi.Mode() + switch { + case mode.IsRegular(): + // Make the directory. This is redundant because it should + // already be made by a directory entry in the tar + // beforehand. Thus, don't check for errors; the next + // write will fail with the same error. + dir := filepath.Dir(abs) + if !madeDir[dir] { + if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { + return err + } + madeDir[dir] = true + } + wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) + if err != nil { + return err + } + n, err := io.Copy(wf, tr) + if closeErr := wf.Close(); closeErr != nil && err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("error writing to %s: %v", abs, err) + } + if n != f.Size { + return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size) + } + modTime := f.ModTime + if modTime.After(t0) { + // Clamp modtimes at system time. See + // golang.org/issue/19062 when clock on + // buildlet was behind the gitmirror server + // doing the git-archive. + modTime = t0 + } + if !modTime.IsZero() { + if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError { + // benign error. Gerrit doesn't even set the + // modtime in these, and we don't end up relying + // on it anywhere (the gomote push command relies + // on digests only), so this is a little pointless + // for now. + log.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err) + loggedChtimesError = true // once is enough + } + } + nFiles++ + case mode.IsDir(): + if err := os.MkdirAll(abs, 0755); err != nil { + return err + } + madeDir[abs] = true + default: + return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode) + } + } + return nil +} + +func validRelativeDir(dir string) bool { + if strings.Contains(dir, `\`) || path.IsAbs(dir) { + return false + } + dir = path.Clean(dir) + if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." { + return false + } + return true +} + +func validRelPath(p string) bool { + if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { + return false + } + return true +} diff --git a/test/python/test_build.py b/test/python/test_build.py index 6fa386d8..39adeb72 100644 --- a/test/python/test_build.py +++ b/test/python/test_build.py @@ -1,5 +1,4 @@ import os -import random import subprocess import tempfile import utils @@ -7,13 +6,9 @@ import image -def new_image_name(): - return "makisu-test:{}".format(random.randint(0, 1000000)) - - def test_build_simple(registry1, registry2, storage_dir): - new_image = new_image_name() - replica_image = new_image_name() + new_image = utils.new_image_name() + replica_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/simple') @@ -28,7 +23,7 @@ def test_build_simple(registry1, registry2, storage_dir): def test_build_symlink(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/symlink') @@ -39,7 +34,7 @@ def test_build_symlink(registry1, storage_dir): def test_build_copy_glob(registry1, storage_dir, cache_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/copy-glob') @@ -51,7 +46,7 @@ def test_build_copy_glob(registry1, storage_dir, cache_dir): def test_build_copy_from(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/copy-from') @@ -60,8 +55,9 @@ def test_build_copy_from(registry1, storage_dir): code, err = utils.docker_run_image(registry1.addr, new_image) assert code == 0, err + def test_build_copy_add_chown(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/copy-add-chown') @@ -70,17 +66,19 @@ def test_build_copy_add_chown(registry1, storage_dir): code, err = utils.docker_run_image(registry1.addr, new_image) assert code == 0, err + def test_build_copy_archive(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/copy-archive') utils.makisu_build_image( - new_image, context_dir, storage_dir, registry=registry1.addr) + new_image, context_dir, storage_dir, registry=registry1.addr) code, err = utils.docker_run_image(registry1.addr, new_image) assert code == 0, err + def test_build_arg_and_env(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/arg-and-env') @@ -98,7 +96,7 @@ def test_build_arg_and_env(registry1, storage_dir): def test_user_change(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/user-change') @@ -119,7 +117,7 @@ def test_user_change(registry1, storage_dir): # This is necessary in the k8s environment, because the default token is mounted # under `/var/run/secrets/kubernetes/serviceaccount`. def test_build_with_readonly_var_run_mnt(registry1, storage_dir, cache_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/simple') @@ -134,7 +132,7 @@ def test_build_with_readonly_var_run_mnt(registry1, storage_dir, cache_dir): def test_build_go_from_scratch(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/go-from-scratch') @@ -145,8 +143,8 @@ def test_build_go_from_scratch(registry1, storage_dir): def test_build_with_distributed_cache(registry1, storage_dir, cache_dir, tmpdir): - new_image1 = new_image_name() - new_image2 = new_image_name() + new_image1 = utils.new_image_name() + new_image2 = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/mount') test_file = tmpdir.join("f1") @@ -190,8 +188,8 @@ def test_build_with_distributed_cache(registry1, storage_dir, cache_dir, tmpdir) def test_build_with_local_cache(registry1, storage_dir, cache_dir, tmpdir): - new_image1 = new_image_name() - new_image2 = new_image_name() + new_image1 = utils.new_image_name() + new_image2 = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/mount') test_file = tmpdir.join("f1") @@ -217,7 +215,7 @@ def test_build_with_local_cache(registry1, storage_dir, cache_dir, tmpdir): def test_build_go_with_debian_package(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/go-with-debian-package') @@ -247,11 +245,12 @@ def test_build_go_with_debian_package(registry1, storage_dir): assert list(l2.get_tar_headers())[0].uname != "root" assert list(l2.get_tar_headers())[0].gname != "root" -# A little bit of crazyness on this test but I didn't want to add more complex methods -# to test the `--preserve-root` flag. + +# A little bit of crazyness on this test but I didn't want to add more complex +# methods to test the `--preserve-root` flag. # See the [Dockerfile](../testdata/build-context/preserve-root/Dockerfile) def test_build_with_preserve_root(registry1, storage_dir): - new_image = new_image_name() + new_image = utils.new_image_name() context_dir = os.path.join( os.getcwd(), 'testdata/build-context/preserve-root') @@ -263,7 +262,8 @@ def test_build_with_preserve_root(registry1, storage_dir): "BASE_IMAGE={}/{}".format(registry1.addr, utils.get_base_image()), ] utils.makisu_build_image( - new_image, context_dir, storage_dir, registry=registry1.addr, docker_args=docker_build_args, load=True) + new_image, context_dir, storage_dir, registry=registry1.addr, + docker_args=docker_build_args, load=True) code, err = utils.docker_run_image(registry1.addr, new_image) del os.environ["MAKISU_ALPINE"] diff --git a/test/python/test_push.py b/test/python/test_push.py new file mode 100644 index 00000000..93af2bdd --- /dev/null +++ b/test/python/test_push.py @@ -0,0 +1,25 @@ +import os +import subprocess +import tempfile +import utils + +import image + + +def test_push_simple(registry1, registry2, storage_dir, tmpdir): + new_image = utils.new_image_name() + replica_image = utils.new_image_name() + + _, image_tar_path = tempfile.mkstemp(dir='/tmp') # TODO: prevent leaking if test failed. + utils.docker_save_image('busybox:latest', image_tar_path) + + utils.makisu_push_image( + new_image, image_tar_path, + registry=registry1.addr, + replicas=[registry2.addr + '/' + replica_image], + registry_cfg={"*": {"*": {"security": {"tls": {"client": {"disabled": True}}}}}}) + code, err = utils.docker_run_image(registry1.addr, new_image) + assert code == 0, err + code, err = utils.docker_run_image(registry2.addr, replica_image) + assert code == 0, err + diff --git a/test/python/utils.py b/test/python/utils.py index 443570f8..7d8546b5 100644 --- a/test/python/utils.py +++ b/test/python/utils.py @@ -1,10 +1,15 @@ import json import os.path +import random import requests import string import subprocess +def new_image_name(): + return "makisu-test:{}".format(random.randint(0, 1000000)) + + def docker_image_exists(image): output = subprocess.check_output([ 'docker', 'images', image, '--format', '"{{.Repository}}:{{.Tag}}"' @@ -48,6 +53,16 @@ def docker_delete_image(image): assert not docker_image_exists(image) +def docker_run_image(registry, image): + if registry: + image = '{}/{}'.format(registry, image) + proc = subprocess.Popen( + ["docker", "run", "-i", "--rm", image], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, err = proc.communicate() + return proc.returncode, err + + def registry_image_exists(image, registry): repotag = string.split(image, ':') assert len(repotag) >= 2 @@ -92,8 +107,9 @@ def makisu_run_cmd(volumes, args): def makisu_build_image( - new_image, context_dir, storage_dir, cache_dir=None, volumes=None, - docker_args=None, load=False, registry=None, replicas=None, registry_cfg=None): + new_image_tag, context_dir, storage_dir, cache_dir=None, volumes=None, + docker_args=None, load=False, registry=None, replicas=None, + registry_cfg=None): volumes = volumes or {} volumes[storage_dir] = storage_dir # Sandbox and file store @@ -105,7 +121,7 @@ def makisu_build_image( args = [ 'build', - '-t', '{}'.format(new_image), + '-t', '{}'.format(new_image_tag), '--storage', storage_dir, '--modifyfs=true', '--commit=explicit', @@ -114,7 +130,7 @@ def makisu_build_image( args.extend(['--build-arg', docker_arg]) if registry: - args.extend([ '--push', registry]) + args.extend(['--push', registry]) if replicas: for replica in replicas: @@ -135,14 +151,34 @@ def makisu_build_image( assert exit_code == 0 if registry: - assert registry_image_exists(new_image, registry) + assert registry_image_exists(new_image_tag, registry) -def docker_run_image(registry, image): +def makisu_push_image( + new_image_tag, image_tar_path, registry=None, replicas=None, + registry_cfg=None): + + volumes = {image_tar_path: image_tar_path} # Mount image tar. + + args = [ + 'push', + '-t', '{}'.format(new_image_tag), + ] + if registry: - image = '{}/{}'.format(registry, image) - proc = subprocess.Popen( - ["docker", "run", "-i", "--rm", image], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, err = proc.communicate() - return proc.returncode, err + args.extend(['--push', registry]) + + if replicas: + for replica in replicas: + args.extend(['--replica', replica]) + + if registry_cfg is not None: + args.extend(['--registry-config', json.dumps(registry_cfg)]) + + args.append(image_tar_path) + + exit_code = makisu_run_cmd(volumes, args) + assert exit_code == 0 + + if registry: + assert registry_image_exists(new_image_tag, registry) diff --git a/testdata/files/busybox/4ac76077f2c741c856a2419dfdb0804b18e48d2e1a9ce9c6a3f0605a2078caba/json b/testdata/files/busybox/393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b/json similarity index 100% rename from testdata/files/busybox/4ac76077f2c741c856a2419dfdb0804b18e48d2e1a9ce9c6a3f0605a2078caba/json rename to testdata/files/busybox/393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b/json diff --git a/testdata/files/busybox/4ac76077f2c741c856a2419dfdb0804b18e48d2e1a9ce9c6a3f0605a2078caba/layer.tar b/testdata/files/busybox/393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b/layer.tar similarity index 100% rename from testdata/files/busybox/4ac76077f2c741c856a2419dfdb0804b18e48d2e1a9ce9c6a3f0605a2078caba/layer.tar rename to testdata/files/busybox/393ccd5c4dd90344c9d725125e13f636ce0087c62f5ca89050faaacbb9e3ed5b/layer.tar