From 482c66fb46fafcae168e10e78e87fbd5b74859f6 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 2 Aug 2023 09:57:17 +0200 Subject: [PATCH] Refactor CLI using dtyper (#306) --- docs/docs/user/references/cli-commands.md | 8 ++-- kpops/__init__.py | 11 +++++ kpops/cli/main.py | 49 ++++++++++++++-------- kpops/pipeline_generator/pipeline.py | 6 +++ poetry.lock | 16 +++++++- pyproject.toml | 1 + tests/pipeline/test_pipeline.py | 50 ++++++++++++++--------- 7 files changed, 98 insertions(+), 43 deletions(-) diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index e48595a19..ab23dcaac 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -44,7 +44,7 @@ $ kpops clean [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE] * `--config FILE`: Path to the config.yaml file [env var: KPOPS_CONFIG_PATH; default: config.yaml] * `--steps TEXT`: Comma separated list of steps to apply the command on [env var: KPOPS_PIPELINE_STEPS] * `--dry-run / --execute`: Whether to dry run the command or execute it [default: dry-run] -* `--verbose / --no-verbose`: [default: no-verbose] +* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--help`: Show this message and exit. ## `kpops deploy` @@ -67,8 +67,8 @@ $ kpops deploy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE] * `--pipeline-base-dir DIRECTORY`: Base directory to the pipelines (default is current working directory) [env var: KPOPS_PIPELINE_BASE_DIR; default: .] * `--defaults DIRECTORY`: Path to defaults folder [env var: KPOPS_DEFAULT_PATH] * `--config FILE`: Path to the config.yaml file [env var: KPOPS_CONFIG_PATH; default: config.yaml] -* `--verbose / --no-verbose`: [default: no-verbose] * `--dry-run / --execute`: Whether to dry run the command or execute it [default: dry-run] +* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--steps TEXT`: Comma separated list of steps to apply the command on [env var: KPOPS_PIPELINE_STEPS] * `--help`: Show this message and exit. @@ -93,8 +93,8 @@ $ kpops destroy [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE] * `--defaults DIRECTORY`: Path to defaults folder [env var: KPOPS_DEFAULT_PATH] * `--config FILE`: Path to the config.yaml file [env var: KPOPS_CONFIG_PATH; default: config.yaml] * `--steps TEXT`: Comma separated list of steps to apply the command on [env var: KPOPS_PIPELINE_STEPS] +* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--dry-run / --execute`: Whether to dry run the command or execute it [default: dry-run] -* `--verbose / --no-verbose`: [default: no-verbose] * `--help`: Show this message and exit. ## `kpops generate` @@ -147,7 +147,7 @@ $ kpops reset [OPTIONS] PIPELINE_PATH [COMPONENTS_MODULE] * `--config FILE`: Path to the config.yaml file [env var: KPOPS_CONFIG_PATH; default: config.yaml] * `--steps TEXT`: Comma separated list of steps to apply the command on [env var: KPOPS_PIPELINE_STEPS] * `--dry-run / --execute`: Whether to dry run the command or execute it [default: dry-run] -* `--verbose / --no-verbose`: [default: no-verbose] +* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--help`: Show this message and exit. ## `kpops schema` diff --git a/kpops/__init__.py b/kpops/__init__.py index f708a9b20..7c7b449dc 100644 --- a/kpops/__init__.py +++ b/kpops/__init__.py @@ -1 +1,12 @@ __version__ = "1.3.2" + +# export public API functions +from kpops.cli.main import clean, deploy, destroy, generate, reset + +__all__ = ( + "generate", + "deploy", + "destroy", + "reset", + "clean", +) diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 505759a9f..0db07ba47 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Iterator, Optional +import dtyper import typer from kpops import __version__ @@ -25,7 +26,7 @@ LOG_DIVIDER = "#" * 100 -app = typer.Typer(pretty_exceptions_enable=False) +app = dtyper.Typer(pretty_exceptions_enable=False) BASE_DIR_PATH_OPTION: Path = typer.Option( default=Path("."), @@ -77,6 +78,8 @@ help="Whether to dry run the command or execute it", ) +VERBOSE_OPTION = typer.Option(False, help="Enable verbose printing") + COMPONENTS_MODULES: str | None = typer.Argument( default=None, help="Custom Python module containing your project-specific components", @@ -160,7 +163,9 @@ def get_steps_to_apply( return list(pipeline) -def reverse_pipeline_steps(pipeline, steps) -> Iterator[PipelineComponent]: +def reverse_pipeline_steps( + pipeline: Pipeline, steps: str | None +) -> Iterator[PipelineComponent]: return reversed(get_steps_to_apply(pipeline, steps)) @@ -185,7 +190,7 @@ def create_pipeline_config( return pipeline_config -@app.command( +@app.command( # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8 help=""" Generate json schema. @@ -215,16 +220,16 @@ def schema( gen_config_schema() -@app.command( +@app.command( # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8 help="Enriches pipelines steps with defaults. The output is used as input for the deploy/destroy/... commands." ) def generate( - pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, pipeline_path: Path = PIPELINE_PATH_ARG, components_module: Optional[str] = COMPONENTS_MODULES, + pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, defaults: Optional[Path] = DEFAULT_PATH_OPTION, config: Path = CONFIG_PATH_OPTION, - verbose: bool = typer.Option(False, help="Enable verbose printing"), + verbose: bool = VERBOSE_OPTION, template: bool = typer.Option(False, help="Run Helm template"), steps: Optional[str] = PIPELINE_STEPS, api_version: Optional[str] = typer.Option( @@ -259,15 +264,17 @@ def generate( return pipeline -@app.command(help="Deploy pipeline steps") +@app.command( + help="Deploy pipeline steps" +) # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8 def deploy( - pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, pipeline_path: Path = PIPELINE_PATH_ARG, components_module: Optional[str] = COMPONENTS_MODULES, + pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, defaults: Optional[Path] = DEFAULT_PATH_OPTION, config: Path = CONFIG_PATH_OPTION, - verbose: bool = False, dry_run: bool = DRY_RUN, + verbose: bool = VERBOSE_OPTION, steps: Optional[str] = PIPELINE_STEPS, ): pipeline_config = create_pipeline_config(config, defaults, verbose) @@ -281,16 +288,18 @@ def deploy( component.deploy(dry_run) -@app.command(help="Destroy pipeline steps") +@app.command( + help="Destroy pipeline steps" +) # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8 def destroy( - pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, pipeline_path: Path = PIPELINE_PATH_ARG, components_module: Optional[str] = COMPONENTS_MODULES, + pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, defaults: Optional[Path] = DEFAULT_PATH_OPTION, config: Path = CONFIG_PATH_OPTION, steps: Optional[str] = PIPELINE_STEPS, + verbose: bool = VERBOSE_OPTION, dry_run: bool = DRY_RUN, - verbose: bool = False, ): pipeline_config = create_pipeline_config(config, defaults, verbose) pipeline = setup_pipeline( @@ -302,16 +311,18 @@ def destroy( component.destroy(dry_run) -@app.command(help="Reset pipeline steps") +@app.command( + help="Reset pipeline steps" +) # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8 def reset( - pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, pipeline_path: Path = PIPELINE_PATH_ARG, components_module: Optional[str] = COMPONENTS_MODULES, + pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, defaults: Optional[Path] = DEFAULT_PATH_OPTION, config: Path = CONFIG_PATH_OPTION, steps: Optional[str] = PIPELINE_STEPS, dry_run: bool = DRY_RUN, - verbose: bool = False, + verbose: bool = VERBOSE_OPTION, ): pipeline_config = create_pipeline_config(config, defaults, verbose) pipeline = setup_pipeline( @@ -324,16 +335,18 @@ def reset( component.reset(dry_run) -@app.command(help="Clean pipeline steps") +@app.command( + help="Clean pipeline steps" +) # pyright: ignore[reportGeneralTypeIssues] https://github.com/rec/dtyper/issues/8 def clean( - pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, pipeline_path: Path = PIPELINE_PATH_ARG, components_module: Optional[str] = COMPONENTS_MODULES, + pipeline_base_dir: Path = BASE_DIR_PATH_OPTION, defaults: Optional[Path] = DEFAULT_PATH_OPTION, config: Path = CONFIG_PATH_OPTION, steps: Optional[str] = PIPELINE_STEPS, dry_run: bool = DRY_RUN, - verbose: bool = False, + verbose: bool = VERBOSE_OPTION, ): pipeline_config = create_pipeline_config(config, defaults, verbose) pipeline = setup_pipeline( diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py index 172d8a891..fce638d9d 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline_generator/pipeline.py @@ -56,6 +56,9 @@ def __bool__(self) -> bool: def __iter__(self) -> Iterator[PipelineComponent]: return iter(self.components) + def __len__(self) -> int: + return len(self.components) + def validate_unique_names(self) -> None: step_names = [component.name for component in self.components] duplicates = [name for name, count in Counter(step_names).items() if count > 1] @@ -287,6 +290,9 @@ def __str__(self) -> str: ) ) + def __len__(self) -> int: + return len(self.components) + def substitute_in_component(self, component_as_dict: dict) -> dict: """Substitute all $-placeholders in a component in dict representation diff --git a/poetry.lock b/poetry.lock index 4c8f1d783..35bedeba5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -225,6 +225,20 @@ files = [ {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] +[[package]] +name = "dtyper" +version = "2.1.0" +description = "🗝 Make `typer` commands callable, or dataclasses 🗝" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dtyper-2.1.0-py3-none-any.whl", hash = "sha256:331f513b33ccd43c1a803a2a06cdb879ed3925381aed9c04bd65470d58107ac6"}, + {file = "dtyper-2.1.0.tar.gz", hash = "sha256:c18a7198c4d9f9194f862307dc326f2594acaffb8e2a6f81335ebc8bf6ed9e40"}, +] + +[package.dependencies] +typer = "*" + [[package]] name = "exceptiongroup" version = "1.0.4" @@ -1584,4 +1598,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "2acbc93653bbe08133210f961c1bfe4087da573bb58ff824e8d53ecbe079bc01" +content-hash = "6777adf06024f4e4627262cf73c157d15333b2954aba237b19e8e0651734c9bd" diff --git a/pyproject.toml b/pyproject.toml index 6ce937552..0fc825a24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ pydantic = { extras = ["dotenv"], version = "^1.10.8" } rich = "^12.4.4" PyYAML = "^6.0" typer = { extras = ["all"], version = "^0.6.1" } +dtyper = "^2.1.0" pyhumps = "^3.7.3" cachetools = "^5.2.0" dictdiffer = "^0.9.0" diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index 57ad55c01..7f129815f 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -5,23 +5,33 @@ from snapshottest.module import SnapshotTest from typer.testing import CliRunner +import kpops from kpops.cli.main import app from kpops.pipeline_generator.pipeline import ParsingException, ValidationError runner = CliRunner() RESOURCE_PATH = Path(__file__).parent / "resources" -PIPELINE_BASE_DIR = str(RESOURCE_PATH.parent) +PIPELINE_BASE_DIR_PATH = RESOURCE_PATH.parent class TestPipeline: + def test_python_api(self): + pipeline = kpops.generate( + RESOURCE_PATH / "first-pipeline" / "pipeline.yaml", + "tests.pipeline.test_components", + pipeline_base_dir=PIPELINE_BASE_DIR_PATH, + defaults=RESOURCE_PATH, + ) + assert len(pipeline) == 3 + def test_load_pipeline(self, snapshot: SnapshotTest): result = runner.invoke( app, [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "first-pipeline/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -42,7 +52,7 @@ def test_name_equal_prefix_name_concatenation(self): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "name_prefix_concatenation/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -66,7 +76,7 @@ def test_pipelines_with_env_values(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "pipeline-with-envs/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -86,7 +96,7 @@ def test_inflate_pipeline(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "pipeline-with-inflate/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -106,7 +116,7 @@ def test_substitute_in_component(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "component-type-substitution/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -158,7 +168,7 @@ def test_substitute_in_component_infinite_loop(self): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str( RESOURCE_PATH / "component-type-substitution/infinite_pipeline.yaml" @@ -176,7 +186,7 @@ def test_kafka_connector_config_parsing(self): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "kafka-connect-sink-config/pipeline.yaml"), "--defaults", str(RESOURCE_PATH), @@ -198,7 +208,7 @@ def test_no_input_topic(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "no-input-topic-pipeline/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -218,7 +228,7 @@ def test_no_user_defined_components(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "no-user-defined-components/pipeline.yaml"), "--defaults", str(RESOURCE_PATH), @@ -238,7 +248,7 @@ def test_kafka_connect_sink_weave_from_topics(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "kafka-connect-sink/pipeline.yaml"), "--defaults", str(RESOURCE_PATH), @@ -257,7 +267,7 @@ def test_read_from_component(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "read-from-component/pipeline.yaml"), "tests.pipeline.test_components", "--defaults", @@ -277,7 +287,7 @@ def test_with_env_defaults(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "kafka-connect-sink/pipeline.yaml"), "--defaults", str(RESOURCE_PATH / "pipeline-with-env-defaults"), @@ -296,7 +306,7 @@ def test_prefix_pipeline_component(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str( RESOURCE_PATH / "pipeline-component-should-have-prefix/pipeline.yaml" @@ -320,7 +330,7 @@ def test_with_custom_config_with_relative_defaults_path( [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "custom-config/pipeline.yaml"), "--config", str(RESOURCE_PATH / "custom-config/config.yaml"), @@ -360,7 +370,7 @@ def test_with_custom_config_with_absolute_defaults_path( [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "custom-config/pipeline.yaml"), "--config", str(temp_config_path), @@ -391,7 +401,7 @@ def test_default_config(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "custom-config/pipeline.yaml"), "--defaults", str(RESOURCE_PATH / "no-topics-defaults"), @@ -421,7 +431,7 @@ def test_model_serialization(self, snapshot: SnapshotTest): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "pipeline-with-paths/pipeline.yaml"), "--defaults", str(RESOURCE_PATH), @@ -441,7 +451,7 @@ def test_kubernetes_app_name_validation(self): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str( RESOURCE_PATH / "pipeline-with-illegal-kubernetes-name/pipeline.yaml" @@ -463,7 +473,7 @@ def test_validate_unique_step_names(self): [ "generate", "--pipeline-base-dir", - PIPELINE_BASE_DIR, + str(PIPELINE_BASE_DIR_PATH), str(RESOURCE_PATH / "pipeline-duplicate-step-names/pipeline.yaml"), "--defaults", str(RESOURCE_PATH),