feat: add noarchive to scoutfs part files

The part files for multipart uploads are considered temporary
files and should not be archived by default. This adds the
noarchive attribute to the part files to prevent scoutam from
trying to archive these.

There is a new parameter, disablenoarchive, that will prevent
adding the noarchive attribute to these files for the case
where there is a desire to archive these temp files.

Author: benmcclelland

backend/scoutfs/scoutfs.go

@@ -40,11 +40,12 @@ import (
 )
 
 type ScoutfsOpts struct {
-	ChownUID    bool
-	ChownGID    bool
-	GlacierMode bool
-	BucketLinks bool
-	NewDirPerm  fs.FileMode
+	ChownUID         bool
+	ChownGID         bool
+	GlacierMode      bool
+	BucketLinks      bool
+	NewDirPerm       fs.FileMode
+	DisableNoArchive bool
 }
 
 type ScoutFS struct {
@@ -78,6 +79,11 @@ type ScoutFS struct {
 
 	// newDirPerm is the permissions to use when creating new directories
 	newDirPerm fs.FileMode
+
+	// disableNoArchive is used to disable setting scoutam noarchive flag
+	// on mutlipart parts. This is enabled by default to prevent archive
+	// copies of temporary multipart parts.
+	disableNoArchive bool
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -156,6 +162,31 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
 	return uid, gid, needsChown
 }
 
+func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	out, err := s.Posix.UploadPart(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	if !s.disableNoArchive {
+		sum := sha256.Sum256([]byte(*input.Key))
+		partPath := filepath.Join(
+			*input.Bucket,                        // bucket
+			metaTmpMultipartDir,                  // temp multipart dir
+			fmt.Sprintf("%x", sum),               // hashed objname
+			*input.UploadId,                      // upload id
+			fmt.Sprintf("%v", *input.PartNumber), // part number
+		)
+
+		err = setNoArchive(partPath)
+		if err != nil {
+			return nil, fmt.Errorf("set noarchive: %w", err)
+		}
+	}
+
+	return out, err
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
@@ -932,56 +963,60 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput)
 	return nil
 }
 
-func setStaging(objname string) error {
+func isStaging(objname string) (bool, error) {
 	b, err := xattr.Get(objname, flagskey)
 	if err != nil && !isNoAttr(err) {
-		return err
+		return false, err
 	}
 
-	var oldflags uint64
+	var flags uint64
 	if !isNoAttr(err) {
-		err = json.Unmarshal(b, &oldflags)
+		err = json.Unmarshal(b, &flags)
 		if err != nil {
-			return err
+			return false, err
 		}
 	}
 
-	newflags := oldflags | Staging
-
-	if newflags == oldflags {
-		// no flags change, just return
-		return nil
-	}
-
-	return fSetNewGlobalFlags(objname, newflags)
+	return flags&Staging == Staging, nil
 }
 
-func isStaging(objname string) (bool, error) {
+func setFlag(objname string, flag uint64) error {
 	b, err := xattr.Get(objname, flagskey)
 	if err != nil && !isNoAttr(err) {
-		return false, err
+		return err
 	}
 
-	var flags uint64
+	var oldflags uint64
 	if !isNoAttr(err) {
-		err = json.Unmarshal(b, &flags)
+		err = json.Unmarshal(b, &oldflags)
 		if err != nil {
-			return false, err
+			return err
 		}
 	}
 
-	return flags&Staging == Staging, nil
-}
+	newflags := oldflags | flag
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
 
-func fSetNewGlobalFlags(objname string, flags uint64) error {
-	b, err := json.Marshal(&flags)
+	b, err = json.Marshal(&newflags)
 	if err != nil {
 		return err
 	}
 
 	return xattr.Set(objname, flagskey, b)
 }
 
+func setStaging(objname string) error {
+	return setFlag(objname, Staging)
+}
+
+func setNoArchive(objname string) error {
+	return setFlag(objname, NoArchive)
+}
+
 func isNoAttr(err error) bool {
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {

backend/scoutfs/scoutfs_compat.go

@@ -52,14 +52,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		meta:             metastore,
+		chownuid:         opts.ChownUID,
+		chowngid:         opts.ChownGID,
+		glaciermode:      opts.GlacierMode,
+		newDirPerm:       opts.NewDirPerm,
+		disableNoArchive: opts.DisableNoArchive,
 	}, nil
 }
 

cmd/versitygw/scoutfs.go

@@ -24,7 +24,8 @@ import (
 )
 
 var (
-	glacier bool
+	glacier          bool
+	disableNoArchive bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.BoolFlag{
+				Name:        "disablenoarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
 		},
 	}
 }
@@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
 	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {