Skip to content

Commit

Permalink
lakectl: get presigned url for object (#7603)
Browse files Browse the repository at this point in the history
  • Loading branch information
ozkatz authored Mar 27, 2024
1 parent 9c320c1 commit eed7c9c
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 13 deletions.
41 changes: 41 additions & 0 deletions cmd/lakectl/cmd/fs_presign.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package cmd

import (
"fmt"
"net/http"

"github.com/go-openapi/swag"
"github.com/spf13/cobra"
"github.com/treeverse/lakefs/pkg/api/apigen"
)

var fsPresignCmd = &cobra.Command{
Use: "presign <path URI>",
Short: "return a pre-signed URL for reading the specified object",
Args: cobra.ExactArgs(1),
ValidArgsFunction: ValidArgsRepository,
Run: func(cmd *cobra.Command, args []string) {
pathURI := MustParsePathURI("path URI", args[0])
client := getClient()
preSignMode := getServerPreSignMode(cmd.Context(), client)
if !preSignMode.Enabled {
Die("Pre-signed URL support is currently disabled for this lakeFS server", 1)
}

resp, err := client.StatObjectWithResponse(cmd.Context(), pathURI.Repository, pathURI.Ref, &apigen.StatObjectParams{
Path: *pathURI.Path,
Presign: swag.Bool(preSignMode.Enabled),
UserMetadata: swag.Bool(true),
})
DieOnErrorOrUnexpectedStatusCode(resp, err, http.StatusOK)
if resp.JSON200 == nil {
Die("Bad response from server", 1)
}
fmt.Printf("%s\n", resp.JSON200.PhysicalAddress)
},
}

//nolint:gochecknoinits
func init() {
fsCmd.AddCommand(fsPresignCmd)
}
27 changes: 14 additions & 13 deletions cmd/lakectl/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,30 +166,31 @@ type PresignMode struct {
Multipart bool
}

func getServerPreSignMode(ctx context.Context, client *apigen.ClientWithResponses) PresignMode {
resp, err := client.GetConfigWithResponse(ctx)
DieOnErrorOrUnexpectedStatusCode(resp, err, http.StatusOK)
if resp.JSON200 == nil {
Die("Bad response from server", 1)
}
storageConfig := resp.JSON200.StorageConfig
return PresignMode{
Enabled: storageConfig.PreSignSupport,
Multipart: swag.BoolValue(storageConfig.PreSignMultipartUpload),
}
}

func getPresignMode(cmd *cobra.Command, client *apigen.ClientWithResponses) PresignMode {
// use flags if set
presignFlag := cmd.Flags().Lookup(presignFlagName)

var presignMode PresignMode
if presignFlag.Changed {
presignMode.Enabled = Must(cmd.Flags().GetBool(presignFlagName))
}

// fetch server config if needed
// if presign flag is not set, use server config
// if presign flag is set, check if server supports multipart upload
var storageConfig *apigen.StorageConfig
if !presignFlag.Changed || presignMode.Enabled {
resp, err := client.GetConfigWithResponse(cmd.Context())
DieOnErrorOrUnexpectedStatusCode(resp, err, http.StatusOK)
if resp.JSON200 == nil {
Die("Bad response from server", 1)
}
storageConfig = resp.JSON200.StorageConfig
if !presignFlag.Changed {
presignMode.Enabled = storageConfig.PreSignSupport
}
presignMode.Multipart = swag.BoolValue(storageConfig.PreSignMultipartUpload)
presignMode = getServerPreSignMode(cmd.Context(), client)
}
return presignMode
}
Expand Down
17 changes: 17 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -2201,6 +2201,23 @@ lakectl fs ls <path URI> [flags]



### lakectl fs presign

return a pre-signed URL for reading the specified object

```
lakectl fs presign <path URI> [flags]
```

#### Options
{:.no_toc}

```
-h, --help help for presign
```



### lakectl fs rm

Delete object
Expand Down
1 change: 1 addition & 0 deletions esti/golden/lakectl_fs_presign.golden
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<PRE_SIGN_URL>
31 changes: 31 additions & 0 deletions esti/lakectl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,37 @@ func getStorageConfig(t *testing.T) *apigen.StorageConfig {
return storageResp.JSON200
}

func TestLakectlFsPresign(t *testing.T) {
config := getStorageConfig(t)
if !config.PreSignSupport {
t.Skip()
}
repoName := generateUniqueRepositoryName()
storage := generateUniqueStorageNamespace(repoName)
vars := map[string]string{
"REPO": repoName,
"STORAGE": storage,
"BRANCH": mainBranch,
}
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars)

// upload some data
const totalObjects = 2
for i := 0; i < totalObjects; i++ {
vars["FILE_PATH"] = fmt.Sprintf("data/ro/ro_1k.%d", i)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s files/ro_1k lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"], false, "lakectl_fs_upload", vars)
}

goldenFile := "lakectl_fs_presign"
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs presign lakefs://"+repoName+"/"+mainBranch+"/data/ro/ro_1k.0", false, goldenFile, map[string]string{
"REPO": repoName,
"STORAGE": storage,
"BRANCH": mainBranch,
"PATH": "data/ro",
"FILE": "ro_1k.0",
})
}

func TestLakectlFsStat(t *testing.T) {
repoName := generateUniqueRepositoryName()
storage := generateUniqueStorageNamespace(repoName)
Expand Down

0 comments on commit eed7c9c

Please sign in to comment.