Skip to content

Commit

Permalink
lakectl: Remove ingest command (#8567)
Browse files Browse the repository at this point in the history
* lakectl: Remove ingest command

* Remove esti test
  • Loading branch information
N-o-Z authored Jan 31, 2025
1 parent 9a5ab53 commit c58338e
Show file tree
Hide file tree
Showing 5 changed files with 0 additions and 239 deletions.
154 changes: 0 additions & 154 deletions cmd/lakectl/cmd/ingest.go

This file was deleted.

23 changes: 0 additions & 23 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -2119,29 +2119,6 @@ lakectl import --from <object store URI> --to <lakeFS path URI> [flags]



### lakectl ingest

Ingest objects from an external source into a lakeFS branch (without actually copying them)

```
lakectl ingest --from <object store URI> --to <lakeFS path URI> [--dry-run] [flags]
```

#### Options
{:.no_toc}

```
-C, --concurrency int max concurrent API calls to make to the lakeFS server (default 64)
--dry-run only print the paths to be ingested
--from string prefix to read from (e.g. "s3://bucket/sub/path/"). must not be in a storage namespace
-h, --help help for ingest
--s3-endpoint-url string URL to access S3 storage API (by default, use regular AWS S3 endpoint
--to string lakeFS path to load objects into (e.g. "lakefs://repo/branch/sub/path/")
-v, --verbose print stats for each individual object staged
```



### lakectl local

Sync local directories with lakeFS paths
Expand Down
7 changes: 0 additions & 7 deletions esti/golden/lakectl_fs_ls_after_ingest.golden

This file was deleted.

20 changes: 0 additions & 20 deletions esti/golden/lakectl_fs_ls_after_ingest_recursive.golden

This file was deleted.

35 changes: 0 additions & 35 deletions esti/lakectl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -585,41 +585,6 @@ func TestLakectlIdentity(t *testing.T) {
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" identity", false, "lakectl_identity", vars)
}

func TestLakectlIngestS3(t *testing.T) {
// Specific S3 test - due to the limitation on ingest source type that has to match lakefs underlying block store,
// this test can only run on AWS setup, and therefore is skipped for other store types
skipOnSchemaMismatch(t, IngestTestBucketPath)

repoName := generateUniqueRepositoryName()
storage := generateUniqueStorageNamespace(repoName)
vars := map[string]string{
"REPO": repoName,
"STORAGE": storage,
"BRANCH": mainBranch,
}

const (
lakectlIngestBucket = "lakectl-ingest-test-data"
expectedIngestOutput = "Staged 10 external objects (total of 10.2 kB)"
)

RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars)
RunCmdAndVerifyContainsText(t, Lakectl()+" ingest --from s3://"+lakectlIngestBucket+" --to lakefs://"+repoName+"/"+mainBranch+"/", false, expectedIngestOutput, vars)
RunCmdAndVerifyContainsText(t, Lakectl()+" ingest --from s3://"+lakectlIngestBucket+" --to lakefs://"+repoName+"/"+mainBranch+"/to-pref/", false, expectedIngestOutput, vars)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs ls lakefs://"+repoName+"/"+mainBranch+"/", false, "lakectl_fs_ls_after_ingest", vars)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs ls lakefs://"+repoName+"/"+mainBranch+"/ --recursive", false, "lakectl_fs_ls_after_ingest_recursive", vars)

// rerunning the same ingest command should succeed and have no effect
RunCmdAndVerifyContainsText(t, Lakectl()+" ingest --from s3://"+lakectlIngestBucket+" --to lakefs://"+repoName+"/"+mainBranch+"/", false, expectedIngestOutput, vars)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs ls lakefs://"+repoName+"/"+mainBranch+"/", false, "lakectl_fs_ls_after_ingest", vars)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs ls lakefs://"+repoName+"/"+mainBranch+"/ --recursive", false, "lakectl_fs_ls_after_ingest_recursive", vars)

// 'from' can also be specified with terminating "/"
RunCmdAndVerifyContainsText(t, Lakectl()+" ingest --from s3://"+lakectlIngestBucket+"/ --to lakefs://"+repoName+"/"+mainBranch+"/", false, expectedIngestOutput, vars)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs ls lakefs://"+repoName+"/"+mainBranch+"/", false, "lakectl_fs_ls_after_ingest", vars)
RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs ls lakefs://"+repoName+"/"+mainBranch+"/ --recursive", false, "lakectl_fs_ls_after_ingest_recursive", vars)
}

func TestLakectlFsDownload(t *testing.T) {
repoName := generateUniqueRepositoryName()
storage := generateUniqueStorageNamespace(repoName)
Expand Down

0 comments on commit c58338e

Please sign in to comment.