Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

elide some syscalls in posix backend #923

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

elide some syscalls in posix backend #923

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
35fc651
(#920) - elide os.Stat(bucket) in posix GetObject
lunixbochs Oct 26, 2024
db9c3d8
(#920) - elide os.Stat(bucket) in posix GetBucketAcl
lunixbochs Oct 26, 2024
8afc158
(#920) - elide getxattr
lunixbochs Oct 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 41 additions & 37 deletions backend/posix/posix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"io/fs"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -1389,7 +1390,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM

userMetaData := make(map[string]string)
upiddir := filepath.Join(objdir, uploadID)
cType, cEnc := p.loadUserMetaData(bucket, upiddir, userMetaData)
cType, cEnc, _ := p.loadUserMetaData(bucket, upiddir, userMetaData)

objname := filepath.Join(bucket, object)
dir := filepath.Dir(objname)
Expand Down Expand Up @@ -1542,10 +1543,10 @@ func (p *Posix) retrieveUploadId(bucket, object string) (string, [32]byte, error

// fll out the user metadata map with the metadata for the object
// and return the content type and encoding
func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (string, string, []string) {
ents, err := p.meta.ListAttributes(bucket, object)
if err != nil || len(ents) == 0 {
return "", ""
return "", "", nil
}
for _, e := range ents {
if !isValidMeta(e) {
Expand All @@ -1563,13 +1564,16 @@ func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (st
}

var contentType, contentEncoding string
b, _ := p.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
contentType = string(b)

b, _ = p.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
contentEncoding = string(b)
if slices.Contains(ents, contentTypeHdr) {
b, _ := p.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr)
contentType = string(b)
}
if slices.Contains(ents, contentEncHdr) {
b, _ := p.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr)
contentEncoding = string(b)
}

return contentType, contentEncoding
return contentType, contentEncoding, ents
}

func isValidMeta(val string) bool {
Expand Down Expand Up @@ -2721,19 +2725,17 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId)
}

// NOTE: os.Stat(bucket) removed here, and moved inside the first fs.ErrNotExist handlers below
// if any more fs.ErrNotExist checks are added below for the file, they should also stat the bucket
bucket := *input.Bucket
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}

object := *input.Key
if versionId != "" {
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) {
// os.Stat(bucket) fallback 1
if _, err := os.Stat(bucket); errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
Expand All @@ -2753,6 +2755,10 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO

fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
// os.Stat(bucket) fallback 2
if _, err := os.Stat(bucket); errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if versionId != "" {
return nil, s3err.GetAPIError(s3err.ErrInvalidVersionId)
}
Expand Down Expand Up @@ -2822,7 +2828,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
if fi.IsDir() {
userMetaData := make(map[string]string)

_, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
_, contentEncoding, _ := p.loadUserMetaData(bucket, object, userMetaData)
contentType := backend.DirContentType

b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
Expand Down Expand Up @@ -2870,22 +2876,25 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO

userMetaData := make(map[string]string)

contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
contentType, contentEncoding, xattrs := p.loadUserMetaData(bucket, object, userMetaData)

b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
etag := string(b)
if err != nil {
etag = ""
var etag string
if slices.Contains(xattrs, etagkey) {
if b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey); err == nil {
etag = string(b)
}
}

var tagCount *int32
tags, err := p.getAttrTags(bucket, object)
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrBucketTaggingNotFound)) {
return nil, err
}
if tags != nil {
tgCount := int32(len(tags))
tagCount = &tgCount
if slices.Contains(xattrs, tagHdr) {
tags, err := p.getAttrTags(bucket, object)
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrBucketTaggingNotFound)) {
return nil, err
}
if tags != nil {
tgCount := int32(len(tags))
tagCount = &tgCount
}
}

f, err := os.Open(objPath)
Expand Down Expand Up @@ -3039,7 +3048,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
}

userMetaData := make(map[string]string)
contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
contentType, contentEncoding, _ := p.loadUserMetaData(bucket, object, userMetaData)

if fi.IsDir() {
contentType = backend.DirContentType
Expand Down Expand Up @@ -3470,15 +3479,10 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err := os.Stat(*input.Bucket)
b, err := p.meta.RetrieveAttribute(nil, *input.Bucket, "", aclkey)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}

b, err := p.meta.RetrieveAttribute(nil, *input.Bucket, "", aclkey)
if errors.Is(err, meta.ErrNoSuchKey) {
return []byte{}, nil
}
Expand Down
Loading