diff --git a/cmd/lakectl/cmd/local_commit_test.go b/cmd/lakectl/cmd/local_commit_test.go index c01a47912db..bf01f4d131d 100644 --- a/cmd/lakectl/cmd/local_commit_test.go +++ b/cmd/lakectl/cmd/local_commit_test.go @@ -29,7 +29,7 @@ func getTestClient(t *testing.T, endpoint string) *apigen.ClientWithResponses { httpClient := &http.Client{ Transport: transport, } - basicAuthProvider, err := securityprovider.NewSecurityProviderBasicAuth(string(defaultAdminAccessKeyID), string(defaultAdminSecretAccessKey)) + basicAuthProvider, err := securityprovider.NewSecurityProviderBasicAuth(defaultAdminAccessKeyID, defaultAdminSecretAccessKey) require.NoError(t, err) serverEndpoint, err := apiutil.NormalizeLakeFSEndpoint(endpoint) @@ -70,7 +70,7 @@ func TestUncommittedOutsideOfPrefix(t *testing.T) { w.Header().Set("Content-Type", "application/json") w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(res) + require.NoError(t, json.NewEncoder(w).Encode(res)) }), expectedResult: false, }, @@ -88,7 +88,7 @@ func TestUncommittedOutsideOfPrefix(t *testing.T) { w.Header().Set("Content-Type", "application/json") w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(res) + require.NoError(t, json.NewEncoder(w).Encode(res)) }), expectedResult: true, }, @@ -122,7 +122,7 @@ func TestUncommittedOutsideOfPrefix(t *testing.T) { w.WriteHeader(http.StatusNotFound) } require.NotNil(t, res, "Unexpected request") - json.NewEncoder(w).Encode(res) + require.NoError(t, json.NewEncoder(w).Encode(res)) }), expectedResult: false, }, @@ -162,7 +162,7 @@ func TestUncommittedOutsideOfPrefix(t *testing.T) { } require.NotNil(t, res, "Unexpected request") - json.NewEncoder(w).Encode(res) + require.NoError(t, json.NewEncoder(w).Encode(res)) }), expectedResult: true, }, @@ -198,7 +198,7 @@ func TestUncommittedOutsideOfPrefix(t *testing.T) { } require.NotNil(t, res, "Unexpected request") - json.NewEncoder(w).Encode(res) + require.NoError(t, json.NewEncoder(w).Encode(res)) }), expectedResult: true, }, diff --git a/go.mod b/go.mod index 9dd79d34cd1..a13c2d60174 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hnlq715/golang-lru v0.3.0 github.com/jamiealquiza/tachymeter v2.0.0+incompatible - github.com/jedib0t/go-pretty/v6 v6.4.8 + github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/manifoldco/promptui v0.9.0 github.com/matoous/go-nanoid/v2 v2.0.0 github.com/minio/minio-go/v7 v7.0.63 @@ -47,7 +47,7 @@ require ( github.com/xitongsys/parquet-go-source v0.0.0-20230607234618-40034c8066df golang.org/x/crypto v0.18.0 golang.org/x/oauth2 v0.15.0 - golang.org/x/term v0.16.0 + golang.org/x/term v0.17.0 google.golang.org/api v0.152.0 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -66,6 +66,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/IBM/pgxpoolprometheus v1.1.1 + github.com/Masterminds/sprig/v3 v3.2.3 + github.com/NYTimes/gziphandler v1.1.1 github.com/Shopify/go-lua v0.0.0-20221004153744-91867de107cf github.com/alitto/pond v1.8.3 github.com/antonmedv/expr v1.15.3 @@ -107,8 +109,6 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect - github.com/Masterminds/sprig/v3 v3.2.3 // indirect - github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ahmetb/go-linq/v3 v3.2.0 // indirect github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/aws/aws-sdk-go v1.48.11 // indirect @@ -145,11 +145,8 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect @@ -188,7 +185,6 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/VividCortex/ewma v1.1.1 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 github.com/cespare/xxhash/v2 v2.2.0 @@ -252,9 +248,9 @@ require ( golang.org/x/exp v0.0.0-20231127185646-65229373498e golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.20.0 - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/tools v0.17.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 // indirect diff --git a/go.sum b/go.sum index d86f7a744ba..217ed3ba2ea 100644 --- a/go.sum +++ b/go.sum @@ -192,7 +192,6 @@ github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk= github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I= -github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -645,8 +644,6 @@ github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+ github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -731,8 +728,8 @@ github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFr github.com/jamiealquiza/tachymeter v2.0.0+incompatible h1:mGiF1DGo8l6vnGT8FXNNcIXht/YmjzfraiUprXYwJ6g= github.com/jamiealquiza/tachymeter v2.0.0+incompatible/go.mod h1:Ayf6zPZKEnLsc3winWEXJRkTBhdHo58HODAu1oFJkYU= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jedib0t/go-pretty/v6 v6.4.8 h1:HiNzyMSEpsBaduKhmK+CwcpulEeBrTmxutz4oX/oWkg= -github.com/jedib0t/go-pretty/v6 v6.4.8/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= +github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU= +github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -849,7 +846,6 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -940,7 +936,6 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -1048,7 +1043,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -1416,19 +1410,18 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/local/progress.go b/pkg/local/progress.go index 2fe4f4feb89..78db071baea 100644 --- a/pkg/local/progress.go +++ b/pkg/local/progress.go @@ -119,7 +119,7 @@ func NewProgressPool() *ProgressPool { pw := progress.NewWriter() pw.SetAutoStop(false) // important pw.SetTrackerLength(progressTrackerLength) - pw.SetMessageWidth(progressTrackerWidth) + pw.SetMessageLength(progressTrackerWidth) pw.SetSortBy(progress.SortByValue) pw.SetStyle(progress.StyleDefault) pw.SetTrackerPosition(progress.PositionRight) diff --git a/pkg/local/sync.go b/pkg/local/sync.go index 568f8269021..ca61e980876 100644 --- a/pkg/local/sync.go +++ b/pkg/local/sync.go @@ -25,8 +25,9 @@ import ( ) const ( - DefaultDirectoryMask = 0o755 - ClientMtimeMetadataKey = apiutil.LakeFSMetadataPrefix + "client-mtime" + // DefaultDirectoryPermissions Octal representation of default folder permissions + DefaultDirectoryPermissions = 0o040777 + ClientMtimeMetadataKey = apiutil.LakeFSMetadataPrefix + "client-mtime" ) type SyncFlags struct { @@ -95,6 +96,9 @@ func (s *SyncManager) Sync(rootPath string, remote *uri.URI, changeSet <-chan *C if err := wg.Wait(); err != nil { return err } + if s.includePerm { // TODO (niro): Probably need to take care of pruning in deleteLocal flow + return nil // Do not prune directories in this case to preserve directories and permissions + } _, err := fileutil.PruneEmptyDirectories(rootPath) return err } @@ -136,41 +140,8 @@ func (s *SyncManager) apply(ctx context.Context, rootPath string, remote *uri.UR return nil } -func (s *SyncManager) download(ctx context.Context, rootPath string, remote *uri.URI, path string) error { - if err := fileutil.VerifyRelPath(strings.TrimPrefix(path, uri.PathSeparator), rootPath); err != nil { - return err - } - destination := filepath.Join(rootPath, path) - destinationDirectory := filepath.Dir(destination) - if err := os.MkdirAll(destinationDirectory, DefaultDirectoryMask); err != nil { - return err - } - statResp, err := s.client.StatObjectWithResponse(ctx, remote.Repository, remote.Ref, &apigen.StatObjectParams{ - Path: filepath.ToSlash(filepath.Join(remote.GetPath(), path)), - Presign: swag.Bool(s.flags.Presign), - UserMetadata: swag.Bool(true), - }) - if err != nil { - return err - } - if statResp.StatusCode() != http.StatusOK { - httpErr := apigen.Error{Message: "no content"} - _ = json.Unmarshal(statResp.Body, &httpErr) - return fmt.Errorf("(stat: HTTP %d, message: %s): %w", statResp.StatusCode(), httpErr.Message, ErrDownloadingFile) - } - // get mtime - mtimeSecs, err := getMtimeFromStats(*statResp.JSON200) - if err != nil { - return err - } - - if strings.HasSuffix(path, uri.PathSeparator) { - // Directory marker - skip - return nil - } - - lastModified := time.Unix(mtimeSecs, 0) - sizeBytes := swag.Int64Value(statResp.JSON200.SizeBytes) +func (s *SyncManager) downloadFile(ctx context.Context, remote *uri.URI, path, destination string, objStat apigen.ObjectStats) error { + sizeBytes := swag.Int64Value(objStat.SizeBytes) f, err := os.Create(destination) if err != nil { // Sometimes we get a file that is actually a directory marker (Spark loves writing those). @@ -188,11 +159,10 @@ func (s *SyncManager) download(ctx context.Context, rootPath string, remote *uri spinner := s.progressBar.AddSpinner("download " + path) atomic.AddUint64(&s.tasks.Downloaded, 1) defer spinner.Done() - } else { // Download file - // make request + } else { var body io.Reader if s.flags.Presign { - resp, err := s.httpClient.Get(statResp.JSON200.PhysicalAddress) + resp, err := s.httpClient.Get(objStat.PhysicalAddress) if err != nil { return err } @@ -229,15 +199,75 @@ func (s *SyncManager) download(ctx context.Context, rootPath string, remote *uri b.Done() } }() - _, err = io.Copy(f, barReader) + _, err = io.Copy(f, barReader) if err != nil { return fmt.Errorf("could not write file '%s': %w", destination, err) } } + return nil +} +func (s *SyncManager) download(ctx context.Context, rootPath string, remote *uri.URI, path string) error { + if err := fileutil.VerifyRelPath(strings.TrimPrefix(path, uri.PathSeparator), rootPath); err != nil { + return err + } + destination := fmt.Sprintf("%s%c%s", rootPath, os.PathSeparator, path) + destinationDirectory := filepath.Dir(destination) + + if err := os.MkdirAll(destinationDirectory, os.FileMode(DefaultDirectoryPermissions)); err != nil { + return err + } + statResp, err := s.client.StatObjectWithResponse(ctx, remote.Repository, remote.Ref, &apigen.StatObjectParams{ + Path: filepath.ToSlash(filepath.Join(remote.GetPath(), path)), + Presign: swag.Bool(s.flags.Presign), + UserMetadata: swag.Bool(true), + }) + if err != nil { + return err + } + if statResp.StatusCode() != http.StatusOK { + httpErr := apigen.Error{Message: "no content"} + _ = json.Unmarshal(statResp.Body, &httpErr) + return fmt.Errorf("(stat: HTTP %d, message: %s): %w", statResp.StatusCode(), httpErr.Message, ErrDownloadingFile) + } + objStat := *statResp.JSON200 + // get mtime + mtimeSecs, err := getMtimeFromStats(objStat) + if err != nil { + return err + } + lastModified := time.Unix(mtimeSecs, 0) + + var unixPerm *UnixPermissions + isDir := strings.HasSuffix(path, uri.PathSeparator) + if s.includePerm { // Optimization - fail on to get permissions from metadata before having to download the entire file + if unixPerm, err = getUnixPermissionFromStats(objStat); err != nil { + return err + } + } else if isDir { + // Directory marker - skip + return nil + } + + if !isDir { + if err = s.downloadFile(ctx, remote, path, destination, objStat); err != nil { + return err + } + } // set mtime to the server returned one err = os.Chtimes(destination, time.Now(), lastModified) // Explicit to catch in deferred func + if err != nil { + return err + } + + // change ownership and permissions + if s.includePerm { + if err = os.Chown(destination, unixPerm.UID, unixPerm.GID); err != nil { + return err + } + err = syscall.Chmod(destination, uint32(unixPerm.Mode)) + } return err } diff --git a/pkg/local/sync_test.go b/pkg/local/sync_test.go new file mode 100644 index 00000000000..1b112677bca --- /dev/null +++ b/pkg/local/sync_test.go @@ -0,0 +1,224 @@ +// Until https://github.com/jedib0t/go-pretty/issues/322 is resolved +//go:build !race + +package local_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/rs/xid" + "github.com/stretchr/testify/require" + "github.com/treeverse/lakefs/pkg/api/apigen" + "github.com/treeverse/lakefs/pkg/api/apiutil" + "github.com/treeverse/lakefs/pkg/local" + "github.com/treeverse/lakefs/pkg/uri" +) + +func TestSyncManager_download(t *testing.T) { + currentUID := os.Getuid() + currentGID := os.Getgid() + ctx := context.Background() + + testCases := []struct { + Name string + Contents []byte + Metadata map[string]string + Path string + UnixPerm bool + }{ + { + Name: "basic download", + Contents: []byte("foobar\n"), + Metadata: map[string]string{}, + Path: "my_object", + }, + { + Name: "download with client mtime", + Contents: []byte("foobar\n"), + Metadata: map[string]string{ + local.ClientMtimeMetadataKey: strconv.FormatInt(time.Now().Add(24*time.Hour).Unix(), 10), + }, + Path: "my_object", + }, + { + Name: "download file unix perm metadata disabled", + Contents: []byte("foobar\n"), + Metadata: map[string]string{ + local.UnixPermissionsMetadataKey: "{\"UID\":0,\"GID\":0,\"Mode\":775}", + }, + Path: "my_object", + }, + { + Name: "download file unix perm enabled no metadata", + Contents: []byte("foobar\n"), + Metadata: map[string]string{}, + Path: "my_object", + UnixPerm: true, + }, + { + Name: "download file unix perm enabled with metadata", + Contents: []byte("foobar\n"), + Metadata: map[string]string{ + local.UnixPermissionsMetadataKey: fmt.Sprintf("{\"UID\":%d, \"GID\": %d, \"Mode\":%d}", currentUID, currentGID, 0o100755), + }, + Path: "my_object", + UnixPerm: true, + }, + { + Name: "download folder unix perm no metadata", + Contents: nil, + Metadata: map[string]string{}, + Path: "folder1/", + UnixPerm: true, + }, + { + Name: "download folder unix perm with metadata", + Contents: nil, + Metadata: map[string]string{ + local.UnixPermissionsMetadataKey: fmt.Sprintf("{\"UID\":%d, \"GID\": %d, \"Mode\":%d}", currentUID, currentGID, 0o40770), + }, + Path: "folder2/", + UnixPerm: true, + }, + } + + for _, tt := range testCases { + umask := syscall.Umask(0) + syscall.Umask(umask) + + t.Run(tt.Name, func(t *testing.T) { + // We must create the test at the user home dir otherwise we will file to chown + home, err := os.UserHomeDir() + require.NoError(t, err) + testFolderPath := filepath.Join(home, fmt.Sprintf("sync_manager_test_%s", xid.New().String())) + require.NoError(t, os.MkdirAll(testFolderPath, os.ModePerm)) + defer func() { + _ = os.RemoveAll(testFolderPath) + }() + sizeBytes := int64(len(tt.Contents)) + mtime := time.Now().Unix() + metadata := &apigen.ObjectUserMetadata{AdditionalProperties: tt.Metadata} + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + var res interface{} + switch { + case strings.Contains(r.RequestURI, "/stat"): + w.Header().Set("Content-Type", "application/json") + res = &apigen.ObjectStats{ + Metadata: metadata, + Mtime: mtime, + Path: tt.Path, + SizeBytes: &sizeBytes, + } + require.NoError(t, json.NewEncoder(w).Encode(res)) + case strings.HasSuffix(r.URL.Path, "/objects"): + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Length", fmt.Sprintf("%d", sizeBytes)) + _, err := w.Write(tt.Contents) + require.NoError(t, err) + default: + t.Fatal("Unexpected request") + } + w.WriteHeader(http.StatusOK) + }) + server := httptest.NewServer(h) + defer server.Close() + + testClient := getTestClient(t, server.URL) + s := local.NewSyncManager(ctx, testClient, server.Client(), local.SyncFlags{ + Parallelism: 1, + Presign: false, + PresignMultipart: false, + }, tt.UnixPerm) + u := &uri.URI{ + Repository: "repo", + Ref: "main", + Path: nil, + } + changes := make(chan *local.Change, 2) + changes <- &local.Change{ + Source: local.ChangeSourceRemote, + Path: tt.Path, + Type: local.ChangeTypeAdded, + } + close(changes) + require.NoError(t, s.Sync(testFolderPath, u, changes)) + localPath := fmt.Sprintf("%s%c%s", testFolderPath, os.PathSeparator, tt.Path) // Have to build manually due to Clean + stat, err := os.Stat(localPath) + require.NoError(t, err) + + if !stat.IsDir() { + data, err := os.ReadFile(localPath) + require.NoError(t, err) + require.Equal(t, tt.Contents, data) + } + + // Check mtime + expectedMTime := mtime + if clientMTime, ok := tt.Metadata[local.ClientMtimeMetadataKey]; ok { + expectedMTime, err = strconv.ParseInt(clientMTime, 10, 64) + require.NoError(t, err) + } + require.Equal(t, expectedMTime, stat.ModTime().Unix()) + + // Check perm + expectedUser := os.Getuid() + expectedGroup := os.Getgid() + expectedMode := local.DefaultFilePermissions - umask + if stat.IsDir() { + expectedMode = local.DefaultDirectoryPermissions - umask + } + + if tt.UnixPerm { + if value, ok := tt.Metadata[local.UnixPermissionsMetadataKey]; ok { + unixPerm := &local.UnixPermissions{} + require.NoError(t, json.Unmarshal([]byte(value), &unixPerm)) + expectedUser = unixPerm.UID + expectedGroup = unixPerm.GID + expectedMode = int(unixPerm.Mode) + } + } + + if sys, ok := stat.Sys().(*syscall.Stat_t); ok { + uid := int(sys.Uid) + gid := int(sys.Gid) + require.Equal(t, expectedUser, uid) + require.Equal(t, expectedGroup, gid) + require.Equal(t, expectedMode, int(sys.Mode)) + } else { + t.Fatal("failed to get stat") + } + }) + } +} + +func getTestClient(t *testing.T, endpoint string) *apigen.ClientWithResponses { + t.Helper() + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConnsPerHost = 100 + httpClient := &http.Client{ + Transport: transport, + } + + serverEndpoint, err := apiutil.NormalizeLakeFSEndpoint(endpoint) + require.NoError(t, err) + + client, err := apigen.NewClientWithResponses( + serverEndpoint, + apigen.WithHTTPClient(httpClient), + ) + require.NoError(t, err) + + return client +} diff --git a/pkg/local/unix_permissions.go b/pkg/local/unix_permissions.go new file mode 100644 index 00000000000..32cb7557066 --- /dev/null +++ b/pkg/local/unix_permissions.go @@ -0,0 +1,80 @@ +package local + +import ( + "encoding/json" + "os" + "strings" + "sync" + "syscall" + + "github.com/treeverse/lakefs/pkg/api/apigen" + "github.com/treeverse/lakefs/pkg/api/apiutil" + "github.com/treeverse/lakefs/pkg/uri" +) + +const ( + // DefaultFilePermissions Octal representation of default file permissions + DefaultFilePermissions = 0o100666 + UnixPermissionsMetadataKey = apiutil.LakeFSMetadataPrefix + "unix-permissions" +) + +var ( + // umask - internal, init only once. Use only via getDefaultPermissions call + umask = -1 + // defaultOwnership - internal, init only once. Use only via getDefaultPermissions call + defaultOwnership *unixOwnership + getOwnershipMutex sync.Mutex +) + +type unixOwnership struct { + UID int + GID int +} + +type UnixPermissions struct { + unixOwnership + Mode os.FileMode +} + +func getUmask() int { + if umask < 0 { + umask = syscall.Umask(0) + syscall.Umask(umask) + } + return umask +} + +func getDefaultPermissions(isDir bool) UnixPermissions { + getOwnershipMutex.Lock() + defer getOwnershipMutex.Unlock() + mode := DefaultFilePermissions - getUmask() + if isDir { + mode = DefaultDirectoryPermissions - getUmask() + } + if defaultOwnership == nil { + defaultOwnership = &unixOwnership{ + UID: os.Getuid(), + GID: os.Getgid(), + } + } + return UnixPermissions{ + unixOwnership: *defaultOwnership, + Mode: os.FileMode(mode), + } +} + +// getUnixPermissionFromStats - Get unix mode and ownership from object metadata, fallback to default permissions in case metadata doesn't exist +func getUnixPermissionFromStats(stats apigen.ObjectStats) (*UnixPermissions, error) { + permissions := getDefaultPermissions(strings.HasSuffix(stats.Path, uri.PathSeparator)) + if stats.Metadata != nil { + unixPermissions, ok := stats.Metadata.Get(UnixPermissionsMetadataKey) + if ok { + // Unmarshal struct + if err := json.Unmarshal([]byte(unixPermissions), &permissions); err != nil { + return nil, err + } + } + } + + return &permissions, nil +}