Merge pull request #5224 from nalind/mkcw-overlay

mkcw: populate the rootfs using an overlay

convertcw.go

@@ -171,6 +171,7 @@ func CWConvertImage(ctx context.Context, systemContext *types.SystemContext, sto
 		Slop:                     options.Slop,
 		FirmwareLibrary:          options.FirmwareLibrary,
 		Logger:                   logger,
+		GraphOptions:             store.GraphOptions(),
 	}
 	rc, workloadConfig, err := mkcw.Archive(sourceDir, &source.OCIv1, archiveOptions)
 	if err != nil {

convertcw_test.go

@@ -72,7 +72,7 @@ func TestCWConvertImage(t *testing.T) {
 	for _, status := range []int{http.StatusOK, http.StatusInternalServerError} {
 		for _, ignoreChainRetrievalErrors := range []bool{false, true} {
 			for _, ignoreAttestationErrors := range []bool{false, true} {
-				t.Run(fmt.Sprintf("status=%d,ignoreChainRetrievalErrors=%v,ignoreAttestationErrors=%v", status, ignoreChainRetrievalErrors, ignoreAttestationErrors), func(t *testing.T) {
+				t.Run(fmt.Sprintf("status~%d~ignoreChainRetrievalErrors~%v~ignoreAttestationErrors~%v", status, ignoreChainRetrievalErrors, ignoreAttestationErrors), func(t *testing.T) {
 					// create a per-test Store object
 					storeOptions := storage.StoreOptions{
 						GraphRoot:       t.TempDir(),

define/types.go

@@ -121,7 +121,7 @@ type ConfidentialWorkloadOptions struct {
 	AttestationURL           string
 	CPUs                     int
 	Memory                   int
-	TempDir                  string
+	TempDir                  string // used for the temporary plaintext copy of the disk image
 	TeeType                  TeeType
 	IgnoreAttestationErrors  bool
 	WorkloadID               string

image.go

@@ -171,6 +171,22 @@ func (i *containerImageRef) extractConfidentialWorkloadFS(options ConfidentialWo
 	if err := json.Unmarshal(i.oconfig, &image); err != nil {
 		return nil, fmt.Errorf("recreating OCI configuration for %q: %w", i.containerID, err)
 	}
+	if options.TempDir == "" {
+		cdir, err := i.store.ContainerDirectory(i.containerID)
+		if err != nil {
+			return nil, fmt.Errorf("getting the per-container data directory for %q: %w", i.containerID, err)
+		}
+		tempdir, err := os.MkdirTemp(cdir, "buildah-rootfs")
+		if err != nil {
+			return nil, fmt.Errorf("creating a temporary data directory to hold a rootfs image for %q: %w", i.containerID, err)
+		}
+		defer func() {
+			if err := os.RemoveAll(tempdir); err != nil {
+				logrus.Warnf("removing temporary directory %q: %v", tempdir, err)
+			}
+		}()
+		options.TempDir = tempdir
+	}
 	mountPoint, err := i.store.Mount(i.containerID, i.mountLabel)
 	if err != nil {
 		return nil, fmt.Errorf("mounting container %q: %w", i.containerID, err)
@@ -186,6 +202,7 @@ func (i *containerImageRef) extractConfidentialWorkloadFS(options ConfidentialWo
 		DiskEncryptionPassphrase: options.DiskEncryptionPassphrase,
 		Slop:                     options.Slop,
 		FirmwareLibrary:          options.FirmwareLibrary,
+		GraphOptions:             i.store.GraphOptions(),
 	}
 	rc, _, err := mkcw.Archive(mountPoint, &image, archiveOptions)
 	if err != nil {

internal/mkcw/archive.go

@@ -17,7 +17,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/buildah/internal/tmpdir"
+	"github.com/containers/buildah/pkg/overlay"
 	"github.com/containers/luksy"
+	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/mount"
+	"github.com/containers/storage/pkg/system"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/go-units"
 	digest "github.com/opencontainers/go-digest"
@@ -48,6 +53,7 @@ type ArchiveOptions struct {
 	DiskEncryptionPassphrase string
 	FirmwareLibrary          string
 	Logger                   *logrus.Logger
+	GraphOptions             []string // passed in from a storage Store, probably
 }
 
 type chainRetrievalError struct {
@@ -107,6 +113,9 @@ func Archive(path string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadC
 		Memory:         memory,
 		AttestationURL: options.AttestationURL,
 	}
+	if options.TempDir == "" {
+		options.TempDir = tmpdir.GetTempDir()
+	}
 
 	// Do things which are specific to the type of TEE we're building for.
 	var chainBytes []byte
@@ -165,6 +174,77 @@ func Archive(path string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadC
 		workloadConfig.TeeData = string(encodedTeeData)
 	}
 
+	// We're going to want to add some content to the rootfs, so set up an
+	// overlay that uses it as a lower layer so that we can write to it.
+	st, err := system.Stat(path)
+	if err != nil {
+		return nil, WorkloadConfig{}, fmt.Errorf("reading information about the container root filesystem: %w", err)
+	}
+	// Create a temporary directory to hold all of this.  Use tmpdir.GetTempDir()
+	// instead of the passed-in location, which a crafty caller might have put in an
+	// overlay filesystem in storage because there tends to be more room there than
+	// in, say, /var/tmp, and the plaintext disk image, which we put in the passed-in
+	// location, can get quite large.
+	rootfsParentDir, err := os.MkdirTemp(tmpdir.GetTempDir(), "buildah-rootfs")
+	if err != nil {
+		return nil, WorkloadConfig{}, fmt.Errorf("setting up parent for container root filesystem: %w", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(rootfsParentDir); err != nil {
+			logger.Warnf("cleaning up parent for container root filesystem: %v", err)
+		}
+	}()
+	// Create a mountpoint for the new overlay, which we'll use as the rootfs.
+	rootfsDir := filepath.Join(rootfsParentDir, "rootfs")
+	if err := idtools.MkdirAndChown(rootfsDir, fs.FileMode(st.Mode()), idtools.IDPair{UID: int(st.UID()), GID: int(st.GID())}); err != nil {
+		return nil, WorkloadConfig{}, fmt.Errorf("creating mount target for container root filesystem: %w", err)
+	}
+	defer func() {
+		if err := os.Remove(rootfsDir); err != nil {
+			logger.Warnf("removing mount target for container root filesystem: %v", err)
+		}
+	}()
+	// Create a directory to hold all of the overlay package's working state.
+	tempDir := filepath.Join(rootfsParentDir, "tmp")
+	if err = os.Mkdir(tempDir, 0o700); err != nil {
+		return nil, WorkloadConfig{}, err
+	}
+	// Create some working state in there.
+	overlayTempDir, err := overlay.TempDir(tempDir, int(st.UID()), int(st.GID()))
+	if err != nil {
+		return nil, WorkloadConfig{}, fmt.Errorf("setting up mount of container root filesystem: %w", err)
+	}
+	defer func() {
+		if err := overlay.RemoveTemp(overlayTempDir); err != nil {
+			logger.Warnf("cleaning up mount of container root filesystem: %v", err)
+		}
+	}()
+	// Create a mount point using that working state.
+	rootfsMount, err := overlay.Mount(overlayTempDir, path, rootfsDir, 0, 0, options.GraphOptions)
+	if err != nil {
+		return nil, WorkloadConfig{}, fmt.Errorf("setting up support for overlay of container root filesystem: %w", err)
+	}
+	defer func() {
+		if err := overlay.Unmount(overlayTempDir); err != nil {
+			logger.Warnf("unmounting support for overlay of container root filesystem: %v", err)
+		}
+	}()
+	// Follow through on the overlay or bind mount, whatever the overlay package decided
+	// to leave to us to do.
+	rootfsMountOptions := strings.Join(rootfsMount.Options, ",")
+	logrus.Debugf("mounting %q to %q as %q with options %v", rootfsMount.Source, rootfsMount.Destination, rootfsMount.Type, rootfsMountOptions)
+	if err := mount.Mount(rootfsMount.Source, rootfsMount.Destination, rootfsMount.Type, rootfsMountOptions); err != nil {
+		return nil, WorkloadConfig{}, fmt.Errorf("mounting overlay of container root filesystem: %w", err)
+	}
+	defer func() {
+		logrus.Debugf("unmounting %q", rootfsMount.Destination)
+		if err := mount.Unmount(rootfsMount.Destination); err != nil {
+			logger.Warnf("unmounting overlay of container root filesystem: %v", err)
+		}
+	}()
+	// Pretend that we didn't have to do any of the preceding.
+	path = rootfsDir
+
 	// Write part of the config blob where the krun init process will be
 	// looking for it.  The oci2cw tool used `buildah inspect` output, but
 	// init is just looking for fields that have the right names in any
@@ -178,11 +258,6 @@ func Archive(path string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadC
 	if err := ioutils.AtomicWriteFile(krunConfigPath, krunConfigBytes, 0o600); err != nil {
 		return nil, WorkloadConfig{}, fmt.Errorf("saving krun config: %w", err)
 	}
-	defer func() {
-		if err := os.Remove(krunConfigPath); err != nil {
-			logger.Warnf("removing krun configuration file: %v", err)
-		}
-	}()
 
 	// Encode the workload config, in case it fails for any reason.
 	cleanedUpWorkloadConfig := workloadConfig

pkg/overlay/overlay.go

@@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"errors"
 
@@ -14,7 +15,6 @@ import (
 	"github.com/containers/storage/pkg/unshare"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
 )
 
 // Options type holds various configuration options for overlay
@@ -180,7 +180,7 @@ func Unmount(contentDir string) error {
 	}
 
 	// Ignore EINVAL as the specified merge dir is not a mount point
-	if err := unix.Unmount(mergeDir, 0); err != nil && !errors.Is(err, os.ErrNotExist) && err != unix.EINVAL {
+	if err := system.Unmount(mergeDir); err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.EINVAL) {
 		return fmt.Errorf("unmount overlay %s: %w", mergeDir, err)
 	}
 	return nil

pkg/overlay/overlay_freebsd.go

@@ -18,6 +18,9 @@ import (
 // But allows api to set custom workdir, upperdir and other overlay options
 // Following API is being used by podman at the moment
 func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, Err error) {
+	if opts == nil {
+		opts = &Options{}
+	}
 	if opts.ReadOnly {
 		// Read-only overlay mounts can be simulated with nullfs
 		mount.Source = source

pkg/overlay/overlay_linux.go

@@ -17,6 +17,9 @@ import (
 // But allows api to set custom workdir, upperdir and other overlay options
 // Following API is being used by podman at the moment
 func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, Err error) {
+	if opts == nil {
+		opts = &Options{}
+	}
 	mergeDir := filepath.Join(contentDir, "merge")
 
 	// Create overlay mount options for rw/ro.

pkg/overlay/overlay_unsupported.go

@@ -0,0 +1,20 @@
+//go:build !freebsd && !linux
+// +build !freebsd,!linux
+
+package overlay
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// MountWithOptions creates a subdir of the contentDir based on the source directory
+// from the source system.  It then mounts up the source directory on to the
+// generated mount point and returns the mount point to the caller.
+// But allows api to set custom workdir, upperdir and other overlay options
+// Following API is being used by podman at the moment
+func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, err error) {
+	return mount, fmt.Errorf("read/write overlay mounts not supported on %q", runtime.GOOS)
+}

tests/mkcw.bats

@@ -49,12 +49,15 @@ function mkcw_check_image() {
     skip "cryptsetup not found"
   fi
   _prefetch busybox
+  _prefetch bash
 
   echo -n mkcw-convert > "$TEST_SCRATCH_DIR"/key
+  # image has one layer, check with all-lower-case TEE type name
   run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert busybox busybox-cw
   mkcw_check_image busybox-cw
-  run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert busybox busybox-cw
-  mkcw_check_image busybox-cw
+  # image has multiple layers, check with all-upper-case TEE type name
+  run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert bash bash-cw
+  mkcw_check_image bash-cw
 }
 
 @test "mkcw-commit" {
@@ -63,10 +66,10 @@ function mkcw_check_image() {
   if ! which cryptsetup > /dev/null 2> /dev/null ; then
     skip "cryptsetup not found"
   fi
-  _prefetch busybox
+  _prefetch bash
 
   echo -n "mkcw commit" > "$TEST_SCRATCH_DIR"/key
-  run_buildah from busybox
+  run_buildah from bash
   ctrID="$output"
   run_buildah commit --iidfile "$TEST_SCRATCH_DIR"/iid --cw type=SEV,ignore_attestation_errors,passphrase="mkcw commit" "$ctrID"
   mkcw_check_image $(cat "$TEST_SCRATCH_DIR"/iid)