diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b998ea4..de5166d6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,18 +51,18 @@ jobs: - name: Build package run: | - uv build --wheel . + uvx --from=hatchling==1.26.3 hatchling build - name: Publish package on PyPI if: steps.check-version.outputs.tag - uses: pypa/gh-action-pypi-publish@v1.8.3 + uses: pypa/gh-action-pypi-publish@v1.12.3 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} - name: Publish package on TestPyPI if: "! steps.check-version.outputs.tag" - uses: pypa/gh-action-pypi-publish@v1.8.3 + uses: pypa/gh-action-pypi-publish@v1.12.3 with: user: __token__ password: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/README.md b/README.md index aee8c2c2..0f7aefc4 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,7 @@ [![Downloads](https://static.pepy.tech/badge/dbt-osmosis)](https://pepy.tech/project/dbt-osmosis) ![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-green.svg) ![black](https://img.shields.io/badge/code%20style-black-000000.svg) -[![Streamlit App](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://z3z1ma-dbt-osmosis-srcdbt-osmosisapp-v2-i0ico9.streamlit.app/) - +[![Streamlit App](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://dbt-osmosis-playground.streamlit.app/) [![Scc Count Badge](https://sloc.xyz/github/z3z1ma/dbt-osmosis/)](https://github.com/z3z1ma/dbt-osmosis/) [![Scc Count Badge](https://sloc.xyz/github/z3z1ma/dbt-osmosis/?category=cocomo)](https://github.com/z3z1ma/dbt-osmosis/) @@ -38,24 +37,12 @@ Hello and welcome to the project! [dbt-osmosis](https://github.com/z3z1ma/dbt-os > Automatically generate documentation based on upstream documented columns -2. A highly performant dbt server which integrates with tools such as dbt-power-user for VS Code to enable interactive querying + realtime compilation from your IDE - - 2a. `dbt-osmosis server serve --project-dir ... --profiles-dir ...` - - > Spins up a WSGI server. Can be passed --register-project to automatically register your local project - -3. Workbench for dbt Jinja SQL. This workbench is powered by streamlit and the badge at the top of the readme will take you to a demo on streamlit cloud with jaffle_shop loaded (requires extra `pip install "dbt-osmosis[workbench]"`). +2. Workbench for dbt Jinja SQL. This workbench is powered by streamlit and the badge at the top of the readme will take you to a demo on streamlit cloud with jaffle_shop loaded (requires extra `pip install "dbt-osmosis[workbench]"`). 3a. `dbt-osmosis workbench --project-dir ... --profiles-dir ...` > Spins up a streamlit app. This workbench offers similar functionality to the osmosis server + power-user combo without a reliance on VS code. Realtime compilation, query execution, pandas profiling all via copying and pasting whatever you are working on into the workbenchat your leisure. Spin it up and down as needed. -4. Diffs for data model outputs to model outputs across git revisions (🚧 this is in development) - - 4a. `dbt-osmosis diff -m some_model --project-dir ... --profiles-dir ...` - - > Run diffs on models dynamically. This pulls the state of the model before changes from your git history, injects it as a node to the dbt manifest, compiles the old and modified nodes, and diffs their query results optionally writing nodes to temp tables before running the diff query for warehouses with performance or query complexity limits (👀 bigquery) - ____ ## Pre-commit @@ -65,7 +52,7 @@ You can use dbt-osmosis as a pre-commit hook. This will run the `dbt-osmosis yam ```yaml title=".pre-commit-config.yaml" repos: - repo: https://github.com/z3z1ma/dbt-osmosis - rev: v0.11.11 # verify the latest version + rev: v1.0.1 # verify the latest version hooks: - id: dbt-osmosis files: ^models/ @@ -84,8 +71,7 @@ I also expect there is some untapped value in the workbench that is only pending Demo the workbench 👇 -[![Streamlit App](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://z3z1ma-dbt-osmosis-srcdbt-osmosisapp-v2-i0ico9.streamlit.app/) - +[![Streamlit App](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://dbt-osmosis-playground.streamlit.app/) ```sh # NOTE this requires the workbench extra as you can see @@ -97,7 +83,6 @@ dbt-osmosis workbench Press "r" to reload the workbench at any time. - ✔️ dbt Editor with instant dbt compilation side-by-side or pivoted ✔️ Full control over model and workbench theme, light and dark mode @@ -106,7 +91,6 @@ Press "r" to reload the workbench at any time. ✔️ Data Profiler (leverages pandas-profiling) - **Editor** The editor is able to compile models with control+enter or dynamically as you type. Its speedy! You can choose any target defined in your profiles yml for compilation and execution. @@ -117,7 +101,6 @@ You can pivot the editor for a fuller view while workbenching some dbt SQL. ![pivot](/screenshots/osmosis_editor_pivot.png?raw=true "dbt-osmosis Pivot Layout") - **Test Query** Test dbt models as you work against whatever profile you have selected and inspect the results. This allows very fast iterative feedback loops not possible with VS Code alone. @@ -130,14 +113,12 @@ Profile your datasets on the fly while you develop without switching context. Al ![profile-data](/screenshots/osmosis_profile.png?raw=true "dbt-osmosis Profile Data") - **Useful Links and RSS Feed** Some useful links and RSS feeds at the bottom. 🤓 ![profile-data](/screenshots/osmosis_links.png?raw=true "dbt-osmosis Profile Data") - ___ ![graph](https://repobeats.axiom.co/api/embed/df37714aa5780fc79871c60e6fc623f8f8e45c35.svg "Repobeats analytics image") diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index d8f30473..00000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.10-slim-bullseye - -# Set separate working directory for easier debugging. -WORKDIR /app - -RUN pip install --no-cache-dir 'poetry==1.2.1' -COPY pyproject.toml poetry.lock ./ -# Install the dependencies first, so that we can cache them. -RUN poetry install --with test --extras postgres - -# Copy everything. (Note: If needed, we can use .dockerignore to limit what's copied.) -COPY . . - -# Install again, now that we've copied the dbt-osmosis package files. Otherwise, -# dbt-osmosis itself won't be installed. -RUN poetry install --with test --extras postgres diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index b0d1f41a..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: "2.4" - -services: - app: - platform: linux/amd64 - build: - context: ../ - dockerfile: docker/Dockerfile.dev - volumes: - # Host source code directory - - ../:/app/ - # The line below is useful if using dbt interactively; should not be - # needed for automated tests. - #- ../tests/sqlfluff_templater/fixtures/dbt/profiles_yml:/root/.dbt - depends_on: - - postgres - entrypoint: /bin/bash - environment: - - POSTGRES_HOST=postgres - tty: true - postgres: - image: postgres:14-bullseye - environment: - - POSTGRES_PASSWORD=password - ports: - # NOTE: "5432:5432" makes the Postgres server accessible to both the host - # developer machine *and* the "app" container in Docker. If you don't want - # it available on the host machine, change this to simply "5432". - - 5432:5432 diff --git a/docker/shell b/docker/shell deleted file mode 100755 index 96bde346..00000000 --- a/docker/shell +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2034 -my_path="$( cd "$(dirname "$0")" || exit ; pwd -P)" -${1:-docker} compose -f docker/docker-compose.yml exec app bash -c "SHELL=bash poetry shell" diff --git a/docker/shutdown b/docker/shutdown deleted file mode 100755 index 2ac45742..00000000 --- a/docker/shutdown +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -my_path="$( cd "$(dirname "$0")" || exit ; pwd -P)" -${1:-docker-compose} -f "${my_path}/docker-compose.yml" down -v diff --git a/docker/startup b/docker/startup deleted file mode 100755 index cee7029b..00000000 --- a/docker/startup +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -ex -export COMPOSE_DOCKER_CLI_BUILD=1 -export DOCKER_BUILDKIT=1 -my_path="$( cd "$(dirname "$0")"; pwd -P)" -"${my_path}/shutdown" -${1:-docker-compose} -f "${my_path}/docker-compose.yml" build -${1:-docker-compose} -f "${my_path}/docker-compose.yml" up -d diff --git a/pyproject.toml b/pyproject.toml index 75f0d2b2..5e155172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["hatchling"] +requires = ["hatchling==1.26.3"] # hatch-vcs build-backend = "hatchling.build" [project] name = "dbt-osmosis" -version = "1.0.0" +version = "1.0.1" description = "A dbt utility for managing YAML to make developing with dbt more delightful." readme = "README.md" license = { text = "Apache-2.0" } @@ -38,7 +38,7 @@ sqlite = ["dbt-sqlite>=1.0.0,<2"] postgres = ["dbt-postgres>=1.0.0,<2"] workbench = [ - "streamlit>=1.20.0,<1.36.0", + "streamlit>=1.20.0,<1.30.0", "streamlit-ace~=0.1.1", "ydata-profiling~=4.12.1", "feedparser~=6.0.11", diff --git a/src/dbt_osmosis/cli/main.py b/src/dbt_osmosis/cli/main.py index 6cbfbadc..b0b7a243 100644 --- a/src/dbt_osmosis/cli/main.py +++ b/src/dbt_osmosis/cli/main.py @@ -157,7 +157,7 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: help="If specified, we will skip merging upstrean meta keys to the model columns.", ) @click.option( - "--skip-add-data-types", # TODO: make sure this is implemented + "--skip-add-data-types", is_flag=True, help="If specified, we will skip adding data types to the models.", ) @@ -188,7 +188,7 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: help="If specified, will add inheritance for the specified keys. IE policy_tags", ) @click.option( - "--output-to-lower", # TODO: validate this is implemented + "--output-to-lower", is_flag=True, help="If specified, output yaml file columns and data types in lowercase if possible.", ) @@ -352,7 +352,7 @@ def organize( help="If specified, we will skip merging upstrean meta keys to the model columns.", ) @click.option( - "--skip-add-data-types", # TODO: make sure this is implemented + "--skip-add-data-types", is_flag=True, help="If specified, we will skip adding data types to the models.", ) @@ -388,7 +388,7 @@ def organize( help="If specified, will add inheritance for the specified keys. IE policy_tags", ) @click.option( - "--output-to-lower", # TODO: validate this is implemented + "--output-to-lower", is_flag=True, help="If specified, output yaml file columns and data types in lowercase if possible.", ) diff --git a/src/dbt_osmosis/core/osmosis.py b/src/dbt_osmosis/core/osmosis.py index 347693c3..b6f20809 100644 --- a/src/dbt_osmosis/core/osmosis.py +++ b/src/dbt_osmosis/core/osmosis.py @@ -810,12 +810,12 @@ def create_missing_source_yamls(context: YamlRefactorContext) -> None: ) def _describe(rel: BaseRelation) -> dict[str, t.Any]: - return { + s = { "name": rel.identifier, "description": "", "columns": [ { - "name": name, + "name": name.lower() if context.settings.output_to_lower else name, "description": meta.comment or "", "data_type": meta.type.lower() if context.settings.output_to_lower @@ -824,6 +824,10 @@ def _describe(rel: BaseRelation) -> dict[str, t.Any]: for name, meta in get_columns(context, get_table_ref(rel)).items() ], } + if context.settings.skip_add_data_types: + for col in t.cast(list[dict[str, t.Any]], s["columns"]): + _ = col.pop("data_type", None) + return s tables = [ schema @@ -1183,6 +1187,12 @@ def _sync_doc_section( for k, v in cdict.items(): if k == "description" and not v: merged.pop("description", None) + elif ( + k == "data_type" + and merged.get("data_type") is None + and context.settings.skip_add_data_types + ): + pass else: merged[k] = v @@ -1197,6 +1207,9 @@ def _sync_doc_section( if not merged[k]: merged.pop(k) + if context.settings.output_to_lower: + merged["name"] = merged["name"].lower() + incoming_columns.append(merged) doc_section["columns"] = incoming_columns @@ -1578,7 +1591,7 @@ def inject_missing_columns(context: YamlRefactorContext, node: ResultNode | None node.unique_id, ) gen_col = {"name": incoming_name, "description": incoming_meta.comment or ""} - if dtype := incoming_meta.type: + if (dtype := incoming_meta.type) and not context.settings.skip_add_data_types: gen_col["data_type"] = dtype.lower() if context.settings.output_to_lower else dtype node.columns[incoming_name] = ColumnInfo.from_dict(gen_col) diff --git a/src/dbt_osmosis/workbench/requirements.txt b/src/dbt_osmosis/workbench/requirements.txt index a2823b16..573b45a7 100644 --- a/src/dbt_osmosis/workbench/requirements.txt +++ b/src/dbt_osmosis/workbench/requirements.txt @@ -1,2 +1,2 @@ -dbt-osmosis[workbench,duckdb]==1.0.0 +dbt-osmosis[workbench,duckdb]==1.0.1 setuptools>=70 diff --git a/uv.lock b/uv.lock index 511ffbb7..10d3a727 100644 --- a/uv.lock +++ b/uv.lock @@ -445,7 +445,7 @@ wheels = [ [[package]] name = "dbt-osmosis" -version = "1.0.0" +version = "1.0.1" source = { editable = "." } dependencies = [ { name = "click" }, @@ -499,7 +499,7 @@ requires-dist = [ { name = "ruamel-yaml", specifier = "~=0.18.7" }, { name = "ruff", marker = "extra == 'dev'", specifier = "~=0.8.4" }, { name = "setuptools", marker = "extra == 'workbench'", specifier = ">=70" }, - { name = "streamlit", marker = "extra == 'workbench'", specifier = ">=1.20.0,<1.36.0" }, + { name = "streamlit", marker = "extra == 'workbench'", specifier = ">=1.20.0,<1.30.0" }, { name = "streamlit-ace", marker = "extra == 'workbench'", specifier = "~=0.1.1" }, { name = "streamlit-elements-fluence", marker = "extra == 'workbench'", specifier = ">=0.1.4" }, { name = "typing-extensions", marker = "python_full_version < '3.10'", specifier = "~=4.12.2" },