From 81435aadcbeadfa1d1a556718f0ade47233179c6 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 12 Dec 2023 12:18:20 -0500 Subject: [PATCH] mkcw: populate the rootfs using an overlay When using the working container's rootfs to populate a plaintext disk image with mkfs, instead of writing .krun_config.json to the rootfs and then removing it afterward (since we don't want it to show up if the same working container is later committed to non confidential-workload image), mount an overlay filesystem using a temporary directory as the upper and the rootfs as the lower, create the .krun_config.json file in the overlay filesystem, and use the overlay filesystem as the source directory for mkfs. Add the necessary stubs to allow pkg/overlay to at least compile on non-Linux systems. Change the naming scheme for a test so that the path names it uses for temporary directories don't include "," or "=", which can confuse the kernel. Creating confidential workload images will now only be possible on Linux systems, but we exec'd out to sevctl to read platform certificates, and that requires kernel support with vendor firmware, so I don't know that anyone will actually be impacted by the change. Teach pkg/overlay.MountWithOptions() to accept `nil` as a pointer to a struct parameter that is otherwise optional. Signed-off-by: Nalin Dahyabhai --- convertcw.go | 1 + convertcw_test.go | 2 +- define/types.go | 2 +- image.go | 17 ++++++ internal/mkcw/archive.go | 85 ++++++++++++++++++++++++++++-- pkg/overlay/overlay.go | 4 +- pkg/overlay/overlay_freebsd.go | 3 ++ pkg/overlay/overlay_linux.go | 3 ++ pkg/overlay/overlay_unsupported.go | 20 +++++++ tests/mkcw.bats | 11 ++-- 10 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 pkg/overlay/overlay_unsupported.go diff --git a/convertcw.go b/convertcw.go index 85576f425ab..ede4f24c86b 100644 --- a/convertcw.go +++ b/convertcw.go @@ -171,6 +171,7 @@ func CWConvertImage(ctx context.Context, systemContext *types.SystemContext, sto Slop: options.Slop, FirmwareLibrary: options.FirmwareLibrary, Logger: logger, + GraphOptions: store.GraphOptions(), } rc, workloadConfig, err := mkcw.Archive(sourceDir, &source.OCIv1, archiveOptions) if err != nil { diff --git a/convertcw_test.go b/convertcw_test.go index 7e5263935e0..c70c18991bf 100644 --- a/convertcw_test.go +++ b/convertcw_test.go @@ -72,7 +72,7 @@ func TestCWConvertImage(t *testing.T) { for _, status := range []int{http.StatusOK, http.StatusInternalServerError} { for _, ignoreChainRetrievalErrors := range []bool{false, true} { for _, ignoreAttestationErrors := range []bool{false, true} { - t.Run(fmt.Sprintf("status=%d,ignoreChainRetrievalErrors=%v,ignoreAttestationErrors=%v", status, ignoreChainRetrievalErrors, ignoreAttestationErrors), func(t *testing.T) { + t.Run(fmt.Sprintf("status~%d~ignoreChainRetrievalErrors~%v~ignoreAttestationErrors~%v", status, ignoreChainRetrievalErrors, ignoreAttestationErrors), func(t *testing.T) { // create a per-test Store object storeOptions := storage.StoreOptions{ GraphRoot: t.TempDir(), diff --git a/define/types.go b/define/types.go index c3e77ed8ad6..132592e54d5 100644 --- a/define/types.go +++ b/define/types.go @@ -121,7 +121,7 @@ type ConfidentialWorkloadOptions struct { AttestationURL string CPUs int Memory int - TempDir string + TempDir string // used for the temporary plaintext copy of the disk image TeeType TeeType IgnoreAttestationErrors bool WorkloadID string diff --git a/image.go b/image.go index 7318e04bdac..163699e0540 100644 --- a/image.go +++ b/image.go @@ -171,6 +171,22 @@ func (i *containerImageRef) extractConfidentialWorkloadFS(options ConfidentialWo if err := json.Unmarshal(i.oconfig, &image); err != nil { return nil, fmt.Errorf("recreating OCI configuration for %q: %w", i.containerID, err) } + if options.TempDir == "" { + cdir, err := i.store.ContainerDirectory(i.containerID) + if err != nil { + return nil, fmt.Errorf("getting the per-container data directory for %q: %w", i.containerID, err) + } + tempdir, err := os.MkdirTemp(cdir, "buildah-rootfs") + if err != nil { + return nil, fmt.Errorf("creating a temporary data directory to hold a rootfs image for %q: %w", i.containerID, err) + } + defer func() { + if err := os.RemoveAll(tempdir); err != nil { + logrus.Warnf("removing temporary directory %q: %v", tempdir, err) + } + }() + options.TempDir = tempdir + } mountPoint, err := i.store.Mount(i.containerID, i.mountLabel) if err != nil { return nil, fmt.Errorf("mounting container %q: %w", i.containerID, err) @@ -186,6 +202,7 @@ func (i *containerImageRef) extractConfidentialWorkloadFS(options ConfidentialWo DiskEncryptionPassphrase: options.DiskEncryptionPassphrase, Slop: options.Slop, FirmwareLibrary: options.FirmwareLibrary, + GraphOptions: i.store.GraphOptions(), } rc, _, err := mkcw.Archive(mountPoint, &image, archiveOptions) if err != nil { diff --git a/internal/mkcw/archive.go b/internal/mkcw/archive.go index a0677e42650..6caea17df33 100644 --- a/internal/mkcw/archive.go +++ b/internal/mkcw/archive.go @@ -17,7 +17,12 @@ import ( "strings" "time" + "github.com/containers/buildah/internal/tmpdir" + "github.com/containers/buildah/pkg/overlay" "github.com/containers/luksy" + "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/mount" + "github.com/containers/storage/pkg/system" "github.com/docker/docker/pkg/ioutils" "github.com/docker/go-units" digest "github.com/opencontainers/go-digest" @@ -48,6 +53,7 @@ type ArchiveOptions struct { DiskEncryptionPassphrase string FirmwareLibrary string Logger *logrus.Logger + GraphOptions []string // passed in from a storage Store, probably } type chainRetrievalError struct { @@ -107,6 +113,9 @@ func Archive(path string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadC Memory: memory, AttestationURL: options.AttestationURL, } + if options.TempDir == "" { + options.TempDir = tmpdir.GetTempDir() + } // Do things which are specific to the type of TEE we're building for. var chainBytes []byte @@ -165,6 +174,77 @@ func Archive(path string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadC workloadConfig.TeeData = string(encodedTeeData) } + // We're going to want to add some content to the rootfs, so set up an + // overlay that uses it as a lower layer so that we can write to it. + st, err := system.Stat(path) + if err != nil { + return nil, WorkloadConfig{}, fmt.Errorf("reading information about the container root filesystem: %w", err) + } + // Create a temporary directory to hold all of this. Use tmpdir.GetTempDir() + // instead of the passed-in location, which a crafty caller might have put in an + // overlay filesystem in storage because there tends to be more room there than + // in, say, /var/tmp, and the plaintext disk image, which we put in the passed-in + // location, can get quite large. + rootfsParentDir, err := os.MkdirTemp(tmpdir.GetTempDir(), "buildah-rootfs") + if err != nil { + return nil, WorkloadConfig{}, fmt.Errorf("setting up parent for container root filesystem: %w", err) + } + defer func() { + if err := os.RemoveAll(rootfsParentDir); err != nil { + logger.Warnf("cleaning up parent for container root filesystem: %v", err) + } + }() + // Create a mountpoint for the new overlay, which we'll use as the rootfs. + rootfsDir := filepath.Join(rootfsParentDir, "rootfs") + if err := idtools.MkdirAndChown(rootfsDir, fs.FileMode(st.Mode()), idtools.IDPair{UID: int(st.UID()), GID: int(st.GID())}); err != nil { + return nil, WorkloadConfig{}, fmt.Errorf("creating mount target for container root filesystem: %w", err) + } + defer func() { + if err := os.Remove(rootfsDir); err != nil { + logger.Warnf("removing mount target for container root filesystem: %v", err) + } + }() + // Create a directory to hold all of the overlay package's working state. + tempDir := filepath.Join(rootfsParentDir, "tmp") + if err = os.Mkdir(tempDir, 0o700); err != nil { + return nil, WorkloadConfig{}, err + } + // Create some working state in there. + overlayTempDir, err := overlay.TempDir(tempDir, int(st.UID()), int(st.GID())) + if err != nil { + return nil, WorkloadConfig{}, fmt.Errorf("setting up mount of container root filesystem: %w", err) + } + defer func() { + if err := overlay.RemoveTemp(overlayTempDir); err != nil { + logger.Warnf("cleaning up mount of container root filesystem: %v", err) + } + }() + // Create a mount point using that working state. + rootfsMount, err := overlay.Mount(overlayTempDir, path, rootfsDir, 0, 0, options.GraphOptions) + if err != nil { + return nil, WorkloadConfig{}, fmt.Errorf("setting up support for overlay of container root filesystem: %w", err) + } + defer func() { + if err := overlay.Unmount(overlayTempDir); err != nil { + logger.Warnf("unmounting support for overlay of container root filesystem: %v", err) + } + }() + // Follow through on the overlay or bind mount, whatever the overlay package decided + // to leave to us to do. + rootfsMountOptions := strings.Join(rootfsMount.Options, ",") + logrus.Debugf("mounting %q to %q as %q with options %v", rootfsMount.Source, rootfsMount.Destination, rootfsMount.Type, rootfsMountOptions) + if err := mount.Mount(rootfsMount.Source, rootfsMount.Destination, rootfsMount.Type, rootfsMountOptions); err != nil { + return nil, WorkloadConfig{}, fmt.Errorf("mounting overlay of container root filesystem: %w", err) + } + defer func() { + logrus.Debugf("unmounting %q", rootfsMount.Destination) + if err := mount.Unmount(rootfsMount.Destination); err != nil { + logger.Warnf("unmounting overlay of container root filesystem: %v", err) + } + }() + // Pretend that we didn't have to do any of the preceding. + path = rootfsDir + // Write part of the config blob where the krun init process will be // looking for it. The oci2cw tool used `buildah inspect` output, but // init is just looking for fields that have the right names in any @@ -178,11 +258,6 @@ func Archive(path string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadC if err := ioutils.AtomicWriteFile(krunConfigPath, krunConfigBytes, 0o600); err != nil { return nil, WorkloadConfig{}, fmt.Errorf("saving krun config: %w", err) } - defer func() { - if err := os.Remove(krunConfigPath); err != nil { - logger.Warnf("removing krun configuration file: %v", err) - } - }() // Encode the workload config, in case it fails for any reason. cleanedUpWorkloadConfig := workloadConfig diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go index e416ecd780e..bbcc8eac695 100644 --- a/pkg/overlay/overlay.go +++ b/pkg/overlay/overlay.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "strings" + "syscall" "errors" @@ -14,7 +15,6 @@ import ( "github.com/containers/storage/pkg/unshare" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" ) // Options type holds various configuration options for overlay @@ -180,7 +180,7 @@ func Unmount(contentDir string) error { } // Ignore EINVAL as the specified merge dir is not a mount point - if err := unix.Unmount(mergeDir, 0); err != nil && !errors.Is(err, os.ErrNotExist) && err != unix.EINVAL { + if err := system.Unmount(mergeDir); err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.EINVAL) { return fmt.Errorf("unmount overlay %s: %w", mergeDir, err) } return nil diff --git a/pkg/overlay/overlay_freebsd.go b/pkg/overlay/overlay_freebsd.go index e814a327c7a..b064ec57837 100644 --- a/pkg/overlay/overlay_freebsd.go +++ b/pkg/overlay/overlay_freebsd.go @@ -18,6 +18,9 @@ import ( // But allows api to set custom workdir, upperdir and other overlay options // Following API is being used by podman at the moment func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, Err error) { + if opts == nil { + opts = &Options{} + } if opts.ReadOnly { // Read-only overlay mounts can be simulated with nullfs mount.Source = source diff --git a/pkg/overlay/overlay_linux.go b/pkg/overlay/overlay_linux.go index 9bd72bc2406..46d0c44aa1f 100644 --- a/pkg/overlay/overlay_linux.go +++ b/pkg/overlay/overlay_linux.go @@ -17,6 +17,9 @@ import ( // But allows api to set custom workdir, upperdir and other overlay options // Following API is being used by podman at the moment func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, Err error) { + if opts == nil { + opts = &Options{} + } mergeDir := filepath.Join(contentDir, "merge") // Create overlay mount options for rw/ro. diff --git a/pkg/overlay/overlay_unsupported.go b/pkg/overlay/overlay_unsupported.go new file mode 100644 index 00000000000..538db65e0f7 --- /dev/null +++ b/pkg/overlay/overlay_unsupported.go @@ -0,0 +1,20 @@ +//go:build !freebsd && !linux +// +build !freebsd,!linux + +package overlay + +import ( + "fmt" + "runtime" + + "github.com/opencontainers/runtime-spec/specs-go" +) + +// MountWithOptions creates a subdir of the contentDir based on the source directory +// from the source system. It then mounts up the source directory on to the +// generated mount point and returns the mount point to the caller. +// But allows api to set custom workdir, upperdir and other overlay options +// Following API is being used by podman at the moment +func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, err error) { + return mount, fmt.Errorf("read/write overlay mounts not supported on %q", runtime.GOOS) +} diff --git a/tests/mkcw.bats b/tests/mkcw.bats index 1aa32c1c9c8..c1a185ef59a 100644 --- a/tests/mkcw.bats +++ b/tests/mkcw.bats @@ -49,12 +49,15 @@ function mkcw_check_image() { skip "cryptsetup not found" fi _prefetch busybox + _prefetch bash echo -n mkcw-convert > "$TEST_SCRATCH_DIR"/key + # image has one layer, check with all-lower-case TEE type name run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert busybox busybox-cw mkcw_check_image busybox-cw - run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert busybox busybox-cw - mkcw_check_image busybox-cw + # image has multiple layers, check with all-upper-case TEE type name + run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert bash bash-cw + mkcw_check_image bash-cw } @test "mkcw-commit" { @@ -63,10 +66,10 @@ function mkcw_check_image() { if ! which cryptsetup > /dev/null 2> /dev/null ; then skip "cryptsetup not found" fi - _prefetch busybox + _prefetch bash echo -n "mkcw commit" > "$TEST_SCRATCH_DIR"/key - run_buildah from busybox + run_buildah from bash ctrID="$output" run_buildah commit --iidfile "$TEST_SCRATCH_DIR"/iid --cw type=SEV,ignore_attestation_errors,passphrase="mkcw commit" "$ctrID" mkcw_check_image $(cat "$TEST_SCRATCH_DIR"/iid)