|
3 | 3 |
|
4 | 4 | The PyAirbyte CLI provides a command-line interface for testing connectors and running benchmarks.
|
5 | 5 |
|
6 |
| -PyAirbyte CLI can be invoked with the `pyairbyte` CLI executable, or the |
| 6 | +After installing PyAirbyte, the CLI can be invoked with the `pyairbyte` CLI executable, or the |
7 | 7 | shorter `pyab` alias.
|
8 | 8 |
|
9 | 9 | These are equivalent:
|
10 | 10 |
|
11 |
| - ```bash |
12 |
| - python -m airbyte.cli --help |
13 |
| - pyairbyte --help |
14 |
| - pyab --help |
15 |
| - ``` |
| 11 | +```bash |
| 12 | +python -m airbyte.cli --help |
| 13 | +pyairbyte --help |
| 14 | +pyab --help |
| 15 | +``` |
16 | 16 |
|
17 | 17 | You can also use `pipx` or the fast and powerful `uv` tool to run the PyAirbyte CLI
|
18 | 18 | without pre-installing:
|
19 | 19 |
|
20 |
| - ``` |
21 |
| - # Install `uv` if you haven't already: |
22 |
| - brew install uv |
| 20 | +```bash |
| 21 | +# Install `uv` if you haven't already: |
| 22 | +brew install uv |
23 | 23 |
|
24 |
| - # Run the PyAirbyte CLI using `uvx`: |
25 |
| - uvx --from=airbyte pyab --help |
26 |
| - ``` |
| 24 | +# Run the PyAirbyte CLI using `uvx`: |
| 25 | +uvx --from=airbyte pyab --help |
| 26 | +``` |
27 | 27 |
|
28 | 28 | Example `benchmark` Usage:
|
29 | 29 |
|
30 |
| - ``` |
31 |
| - # PyAirbyte System Benchmark (no-op): |
32 |
| - pyab benchmark --num-records=2.4e6 |
| 30 | +```bash |
| 31 | +# PyAirbyte System Benchmark (no-op): |
| 32 | +pyab benchmark --num-records=2.4e6 |
33 | 33 |
|
34 |
| - # Source Benchmark: |
35 |
| - pyab benchmark --source=source-hardcoded-records --config='{count: 400000}' |
36 |
| - pyab benchmark --source=source-hardcoded-records --config='{count: 400000}' --streams='*' |
37 |
| - pyab benchmark --source=source-hardcoded-records --config='{count: 4000}' --streams=dummy_fields |
| 34 | +# Source Benchmark: |
| 35 | +pyab benchmark --source=source-hardcoded-records --config='{count: 400000}' |
| 36 | +pyab benchmark --source=source-hardcoded-records --config='{count: 400000}' --streams='*' |
| 37 | +pyab benchmark --source=source-hardcoded-records --config='{count: 4000}' --streams=dummy_fields |
38 | 38 |
|
39 |
| - # Source Benchmark from Docker Image: |
40 |
| - pyab benchmark --source=airbyte/source-hardcoded-records:latest --config='{count: 400_000}' |
41 |
| - pyab benchmark --source=airbyte/source-hardcoded-records:dev --config='{count: 400_000}' |
42 |
| -
|
43 |
| - # Destination Benchmark: |
44 |
| - pyab benchmark --destination=destination-dev-null --config=/path/to/config.json |
45 |
| -
|
46 |
| - # Benchmark a Local Python Source (source-s3): |
47 |
| - pyab benchmark --source=$(poetry run which source-s3) --config=./secrets/config.json |
48 |
| - # Equivalent to: |
49 |
| - LOCAL_EXECUTABLE=$(poetry run which source-s3) |
50 |
| - CONFIG_PATH=$(realpath ./secrets/config.json) |
51 |
| - pyab benchmark --source=$LOCAL_EXECUTABLE --config=$CONFIG_PATH |
52 |
| - ``` |
53 |
| -
|
54 |
| -Example Usage with `uv`: |
| 39 | +# Source Benchmark from Docker Image: |
| 40 | +pyab benchmark --source=airbyte/source-hardcoded-records:latest --config='{count: 400_000}' |
| 41 | +pyab benchmark --source=airbyte/source-hardcoded-records:dev --config='{count: 400_000}' |
55 | 42 |
|
| 43 | +# Destination Benchmark: |
| 44 | +pyab benchmark --destination=destination-dev-null --config=/path/to/config.json |
56 | 45 |
|
| 46 | +# Benchmark a Local Python Source (source-s3): |
| 47 | +pyab benchmark --source=$(poetry run which source-s3) --config=./secrets/config.json |
| 48 | +# Equivalent to: |
| 49 | +LOCAL_EXECUTABLE=$(poetry run which source-s3) |
| 50 | +CONFIG_PATH=$(realpath ./secrets/config.json) |
| 51 | +pyab benchmark --source=$LOCAL_EXECUTABLE --config=$CONFIG_PATH |
| 52 | +``` |
57 | 53 |
|
58 | 54 | Example `validate` Usage:
|
59 | 55 |
|
60 |
| - ``` |
61 |
| - pyab validate --connector=source-hardcoded-records |
62 |
| - pyab validate --connector=source-hardcoded-records --config='{count: 400_000}' |
63 |
| - ``` |
| 56 | +```bash |
| 57 | +pyab validate --connector=source-hardcoded-records |
| 58 | +pyab validate --connector=source-hardcoded-records --config='{count: 400_000}' |
| 59 | +``` |
64 | 60 | """
|
65 | 61 |
|
66 | 62 | from __future__ import annotations
|
@@ -322,7 +318,7 @@ def validate(
|
322 | 318 | config: str | None = None,
|
323 | 319 | pip_url: str | None = None,
|
324 | 320 | ) -> None:
|
325 |
| - """Validate the connector.""" |
| 321 | + """CLI command to run a `benchmark` operation.""" |
326 | 322 | if not connector:
|
327 | 323 | raise PyAirbyteInputError(
|
328 | 324 | message="No connector provided.",
|
@@ -403,7 +399,7 @@ def benchmark(
|
403 | 399 | destination: str | None = None,
|
404 | 400 | config: str | None = None,
|
405 | 401 | ) -> None:
|
406 |
| - """Run benchmarks. |
| 402 | + """CLI command to run a `benchmark` operation. |
407 | 403 |
|
408 | 404 | You can provide either a source or a destination, but not both. If a destination is being
|
409 | 405 | benchmarked, you can use `--num-records` to specify the number of records to generate for the
|
@@ -506,7 +502,7 @@ def sync(
|
506 | 502 | destination_pip_url: str | None = None,
|
507 | 503 | streams: str | None = None,
|
508 | 504 | ) -> None:
|
509 |
| - """Run a sync operation. |
| 505 | + """CLI command to run a `sync` operation. |
510 | 506 |
|
511 | 507 | Currently, this only supports full refresh syncs. Incremental syncs are not yet supported.
|
512 | 508 | Custom catalog syncs are not yet supported.
|
@@ -536,7 +532,7 @@ def sync(
|
536 | 532 |
|
537 | 533 | @click.group()
|
538 | 534 | def cli() -> None:
|
539 |
| - """PyAirbyte CLI.""" |
| 535 | + """@private PyAirbyte CLI.""" |
540 | 536 | pass
|
541 | 537 |
|
542 | 538 |
|
|
0 commit comments