diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000..3d5465e --- /dev/null +++ b/.cargo/config @@ -0,0 +1,16 @@ +# Auto-generated by pgx. You may edit this, or delete it to have a new one created. + +[target.x86_64-unknown-linux-gnu] +linker = "./.cargo/pgx-linker-script.sh" + +[target.aarch64-unknown-linux-gnu] +linker = "./.cargo/pgx-linker-script.sh" + +[target.x86_64-apple-darwin] +linker = "./.cargo/pgx-linker-script.sh" + +[target.aarch64-apple-darwin] +linker = "./.cargo/pgx-linker-script.sh" + +[target.x86_64-unknown-freebsd] +linker = "./.cargo/pgx-linker-script.sh" diff --git a/.cargo/linker-script.sh b/.cargo/linker-script.sh new file mode 100755 index 0000000..4d90b4d --- /dev/null +++ b/.cargo/linker-script.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash +# Auto-generated by pgx. You may edit this, or delete it to have a new one created. + +if [[ $CARGO_BIN_NAME == "sql-generator" ]]; then + gcc -Wl,-undefined,dynamic_lookup,-dynamic-list=$CARGO_MANIFEST_DIR/.cargo/pgx-dynamic-list.txt $@ +else + gcc -Wl,-undefined,dynamic_lookup $@ +fi \ No newline at end of file diff --git a/.cargo/pgx-dynamic-list.txt b/.cargo/pgx-dynamic-list.txt new file mode 100644 index 0000000..beef630 --- /dev/null +++ b/.cargo/pgx-dynamic-list.txt @@ -0,0 +1 @@ +{ __pgx_internals_*; }; \ No newline at end of file diff --git a/.cargo/pgx-linker-script.sh b/.cargo/pgx-linker-script.sh new file mode 100755 index 0000000..c432e14 --- /dev/null +++ b/.cargo/pgx-linker-script.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash +# Auto-generated by pgx. You may edit this, or delete it to have a new one created. + +if [[ $CARGO_BIN_NAME == "sql-generator" ]]; then + UNAME=$(uname) + if [[ $UNAME == "Darwin" ]]; then + TEMP=$(mktemp pgx-XXX) + echo "*_pgx_internals_*" > ${TEMP} + gcc -exported_symbols_list ${TEMP} $@ + rm -rf ${TEMP} + else + TEMP=$(mktemp pgx-XXX) + echo "{ __pgx_internals_*; };" > ${TEMP} + gcc -Wl,-dynamic-list=${TEMP} $@ + rm -rf ${TEMP} + fi +else + gcc -Wl,-undefined,dynamic_lookup $@ +fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8da534d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/target +*.iml +**/*.rs.bk +Cargo.lock +sql/*.generated.sql +extension.dot +extension.jpg \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index c5c77ff..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- - -stages: - - build - - run - - test - - publish - -before_script: - - docker info - -pre_clean: - stage: build - script: - - docker stop gl-pgdd && docker rm gl-pgdd - allow_failure: true - -build_image: - stage: build - script: - - docker build -t rustprooflabs/pgdd . - -start_container: - stage: run - script: - - docker run --name gl-pgdd -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD -p $PG_PORT:5432 -d rustprooflabs/pgdd - - sleep 8 - - -query_extension: - stage: test - script: - - PGPASSWORD=$POSTGRES_PASSWORD psql -h $PG_HOST -p $PG_PORT -U postgres -c "CREATE EXTENSION pgdd;" - - PGPASSWORD=$POSTGRES_PASSWORD psql -h $PG_HOST -p $PG_PORT -U postgres -c "SELECT t_name FROM dd.tables;" - - docker stop gl-pgdd - - docker rm gl-pgdd - - -create_non_extension_scripts: - stage: publish - script: - - cat pgdd--0.1.sql > pgdd_v0_3.sql - - cat pgdd--0.1--0.2.sql >> pgdd_v0_3.sql - - cat pgdd--0.2--0.3.sql >> pgdd_v0_3.sql - artifacts: - paths: - - pgdd_v0_3.sql - - -... \ No newline at end of file diff --git a/ADVANCED-INSTALL.md b/ADVANCED-INSTALL.md new file mode 100644 index 0000000..96fe06d --- /dev/null +++ b/ADVANCED-INSTALL.md @@ -0,0 +1,218 @@ +# PgDD Advanced installation + +This page covers installing PgDD from source locally, and the Docker build +method based on [ZomboDB's build system](https://github.com/zombodb/zombodb) +used to create the binaries. + + +## Install `pgdd` from source + + +One way to install `pgdd` is to install from source by cloning this repository. + +### Prereqs + +Pgx and its dependencies are the main prereq for PgDD. +Install Prereqs and ensure PostgreSQL dev tools are installed. + +> See the [Cargo PGX](https://github.com/zombodb/pgx/tree/master/cargo-pgx) +documentation for more information on using pgx. + + +```bash +sudo apt install postgresql-server-dev-all libreadline-dev zlib1g-dev curl \ + libssl-dev llvm-dev libclang-dev clang \ + graphviz +``` + +[Install Rust](https://www.rust-lang.org/tools/install) and Pgx. + +```bash +curl https://sh.rustup.rs -sSf | sh -s -- -y +source $HOME/.cargo/env +``` + +Install `cargo-pgx` regularly (see dev steps below for non-standard install). + + +```bash +cargo install cargo-pgx +``` + + +Install `cargo-deb` used for packaging binaries. + +```bash +cargo install cargo-deb +``` + + +Initialize pgx. Need to run this after install AND occasionally to get updates +to Postgres versions or glibc updates. Not typically required to follow pgx +developments. + + +```bash +cargo pgx init +``` + + +### Clone PgDD repo + +```bash +mkdir ~/git +cd ~/git +git clone https://github.com/rustprooflabs/pgdd.git +cd ~/git/pgdd +``` + +### Test deployment + +Specify version, `pg10` through `pg13` are currently supported. This command will +start a test instance of Postgres on port `28812`. Using a different version +changes the last two digits of the port! + + +```bash +cargo pgx run pg13 +``` + +Example output. + +```bash + Stopping Postgres v13 +building extension with features `pg13` +"cargo" "build" "--features" "pg13" "--no-default-features" + Finished dev [unoptimized + debuginfo] target(s) in 0.05s + +installing extension + Copying control file to `/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd.control` + Copying shared library to `/home/username/.pgx/13.4/pgx-install/lib/postgresql/pgdd.so` + Building SQL generator with features `pg13` +"cargo" "build" "--bin" "sql-generator" "--features" "pg13" "--no-default-features" + Finished dev [unoptimized + debuginfo] target(s) in 0.05s + Discovering SQL entities + Discovered 9 SQL entities: 0 schemas (0 unique), 6 functions, 0 types, 0 enums, 3 sqls, 0 ords, 0 hashes +running SQL generator with features `pg13` +"cargo" "run" "--bin" "sql-generator" "--features" "pg13" "--no-default-features" "--" "--sql" "/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.4.0-dev.sql" + Finished dev [unoptimized + debuginfo] target(s) in 0.06s + Running `target/debug/sql-generator --sql /home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.4.0-dev.sql` + Copying extension schema file to `/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.3.1--0.4.0-dev.sql` + Copying extension schema file to `/home/username/.pgx/13.4/pgx-install/share/postgresql/extension/pgdd--0.3--0.3.1.sql` + Finished installing pgdd + Starting Postgres v13 on port 28813 + Re-using existing database pgdd +``` + +In the test instance of psql, create the extension in database. + +```bash +CREATE EXTENSION pgdd; +``` + + +## Build binary packages + +Debian/Ubuntu Bionic binaries are available for 0.4.0 +(first [pgx](https://github.com/zombodb/pgx) version) +and later. More distributions will likely have binaries available in the future. + + +```bash +cd build/ +time bash ./build.sh +``` + +Tagged versions will be attached to their [releases](https://github.com/rustprooflabs/pgdd/releases). + +During development some versions may be copied to the `./standalone/` directory. + +```bash +cp ./target/artifacts/* ./standalone/ +``` + +## Pgx Generate graphviz + +```bash +cargo pgx schema -d +dot -Goverlap=prism -Gspline=ortho -Tjpg extension.dot > extension.jpg +``` + +![pgx dependencies for pgdd v0.4.0-dev](pgdd--0.4.0-dev.jpg) + + +## Non-standard dev + +When working against Pgx installed from a non-tagged branch, install pgx using: + +```bash +cargo install --force --git "https://github.com/zombodb/pgx" \ + --branch "develop" \ + "cargo-pgx" +``` + +Changes to `Cargo.toml` required in `[lib]` and `[dependencies]` sections. + + +```toml +[lib] +# rlib added to build against repo instead of crate (I think) +crate-type = ["cdylib", "rlib"] +#crate-type = ["cdylib"] +``` + + +```toml +[dependencies] + +pgx = { git = "https://github.com/zombodb/pgx", branch = "oh-no-type-resolution" } +pgx-macros = { git = "https://github.com/zombodb/pgx", branch = "develop" } +#pgx = "0.1.21" +#pgx-macros = "0.1.21" + +# Won't be needed in final version (hopefully!) +pgx-utils = { git = "https://github.com/zombodb/pgx", branch = "develop" } + +[dev-dependencies] +pgx-tests = { git = "https://github.com/zombodb/pgx", branch = "develop" } +#pgx-tests = "0.1.21" +``` + + + +The following command can be used to force pgx to overwrite the configs it needs to +for various dev related changes. + + +```bash +cargo pgx schema -f +``` + +Another option to try. + +```bash +cargo clean +``` + + +## Non-standard In Docker + +If testing this extension against non-standard pgx install, update the +Dockerfile to install from the specific branch. + +Change + +```bash +RUN /bin/bash rustup.sh -y \ + && cargo install cargo-pgx +``` + +To + +```bash +RUN /bin/bash rustup.sh -y \ + && cargo install --force --git "https://github.com/zombodb/pgx" \ + --branch "develop" \ + "cargo-pgx" +``` + diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..057040a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "pgdd" +version = "0.4.0-dev" +edition = "2018" +description = "In-database (PostgreSQL) data dictionary providing database introspection via standard SQL query syntax." + +[lib] +# rlib added to build against repo instead of crate (I think) +crate-type = ["cdylib", "rlib"] +#crate-type = ["cdylib"] + +[features] +default = ["pg13"] +pg10 = ["pgx/pg10"] +pg11 = ["pgx/pg11"] +pg12 = ["pgx/pg12"] +pg13 = ["pgx/pg13"] +pg_test = [] + +[dependencies] + +pgx = { git = "https://github.com/zombodb/pgx", branch = "develop" } +pgx-macros = { git = "https://github.com/zombodb/pgx", branch = "develop" } +#pgx = "0.2.0-beta.1" +#pgx-macros = "0.2.0-beta.1" + +# Won't be needed in final version (hopefully!) +pgx-utils = { git = "https://github.com/zombodb/pgx", branch = "develop" } +#pgx-utils = "0.2.0-beta.1" + +[dev-dependencies] +pgx-tests = { git = "https://github.com/zombodb/pgx", branch = "develop" } +#pgx-tests = "0.2.0-beta.1" + + +[profile.dev] +panic = "unwind" +lto = "thin" + +[profile.release] +panic = "unwind" +opt-level = 3 +lto = "thin" +codegen-units = 1 + diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 981c299..0000000 --- a/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM postgres:12 - -ENV PG_MAJOR 12 - -LABEL maintainer="PgDD Project - https://github.com/rustprooflabs/pgdd" - -RUN apt-get update \ - && apt-cache showpkg postgresql-$PG_MAJOR \ - && apt-get install -y --no-install-recommends \ - make \ - postgresql-server-dev-$PG_MAJOR \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /tmp/pgdd -COPY *.sql ./ -COPY pgdd.control ./ -COPY Makefile ./ - -RUN make install - diff --git a/LICENSE b/LICENSE index 9c3b718..1ba7a39 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Ryan Lambert +Copyright (c) 2018 - 2021 Ryan Lambert Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 81b47e8..4fbfdb4 100644 --- a/README.md +++ b/README.md @@ -1,226 +1,277 @@ # PostgreSQL Data Dictionary (pgdd) The PostgreSQL Data Dictionary (`pgdd`) is an in-database solution to provide -introspection via standard SQL query syntax. This extension makes it easy to +introspection via standard SQL query syntax. This extension makes it easy to provide a usable data dictionary to all users of a PostgreSQL database. -## Compatability +The extension is built on the Rust [pgx framework](https://github.com/zombodb/pgx) as of version 0.4.0. -PgDD has been tested and found to work against PostgreSQL 10 -through 13-beta2. -Docker images available on -[Docker Hub](https://hub.docker.com/r/rustprooflabs/pgdd). +## Compatibility +PgDD has been tested to work for PostgreSQL 10 through 13. -## Install `pgdd` from source -One way to install `pgdd` is to install from -source by downloading this repository. +## Upgrading from <= v0.3 -### Prereqs +Version 0.4.0 was a complete rewrite of the PgDD extension from a raw-SQL +extension to using the [pgx framework](https://github.com/zombodb/pgx). -Ensure PostgreSQL dev tools are installed. -```bash -sudo apt install postgresql-server-dev-all -``` -### Clone repo +Upgrading from Raw SQL version of PgDD (v0.3) to the pgx version (v0.4.0) is done with `DROP EXTENSION pgdd; CREATE EXTENSION pgdd;` +Care has been taken to provide a smooth upgrade experience but +**do not install the upgrade without testing the process on a test server!** -```bash -mkdir ~/git -cd ~/git -git clone https://github.com/rustprooflabs/pgdd.git -cd ~/git/pgdd -``` +If custom attributes were stored in the `dd` tables you will need to use +`pg_dump` to export the data and reload after recreating the extension +with pgx. If any of the three (3) queries below return a count > 0 +this applies to you. -### Install on Server -```bash -cd ~/git/pgdd -sudo make install +```sql +SELECT COUNT(*) + FROM dd.meta_table + WHERE s_name <> 'dd'; +SELECT COUNT(*) + FROM dd.meta_column + WHERE s_name <> 'dd'; +SELECT COUNT(*) + FROM dd.meta_schema + WHERE s_name <> 'dd'; ``` -### Create Extension in Database -```bash -sudo su - postgres -psql -d your_db -CREATE EXTENSION pgdd; -``` -## Docker Image +The last raw SQL version is still available to [download](https://raw.githubusercontent.com/rustprooflabs/pgdd/main/standalone/pgdd_v0_3.sql). This version is no longer maintained and may or may not +work on future Postgres versions. -PgDD can be deployed in a Docker image. Uses [main Postgres image](https://hub.docker.com/_/postgres/) as starting point, see that -repo for full instructions on using the core Postgres functionality. -``` -docker build -t rustprooflabs/pgdd . -``` +## Install from binary -Build with tag. +Binaries are available for Ubuntu 20.04 (bionic) and Ubuntu 21.04 (hirsute). -Run Postgres in Docker. +Download and install. -``` -docker run --name test-pgdd12 -e POSTGRES_PASSWORD=mysecretpassword -p 6512:5432 -d rustprooflabs/pgdd -``` - -Connect via `psql` using `postgres` role, provide password from prior step -when prompted. +```bash +wget https://github.com/rustprooflabs/pgdd/releases/download/0.4.0.rc2/pgdd_0.4.0.rc2_focal_pg13_amd64.deb +sudo dpkg -i ./pgdd_0.4.0.rc2_focal_pg13_amd64.deb ``` -psql -h host_or_ip -p 6512 -U postgres -``` - +In your database. -## Database Permissions - -Create Read-only group role to assign to users -that need access to query (read-only) the PgDD objects. +```sql +CREATE EXTENSION pgdd; ``` -CREATE ROLE dd_read WITH NOLOGIN; -COMMENT ON ROLE dd_read IS 'Group role to grant read-only permissions to PgDD views.'; -GRANT USAGE ON SCHEMA dd TO dd_read; -GRANT SELECT ON ALL TABLES IN SCHEMA dd TO dd_read; -ALTER DEFAULT PRIVILEGES IN SCHEMA dd GRANT SELECT ON TABLES TO dd_read; -``` +Check version. -Access can now be granted to other users using: +```sql +SELECT extname, extversion + FROM pg_catalog.pg_extension + WHERE extname = 'pgdd' +; +``` ``` -GRANT dd_read TO ; +┌─────────┬────────────┐ +│ extname │ extversion │ +╞═════════╪════════════╡ +│ pgdd │ 0.4.0.rc2 │ +└─────────┴────────────┘ ``` -For read-write access. +## Use Data Dictionary -``` -CREATE ROLE dd_readwrite WITH NOLOGIN; -COMMENT ON ROLE dd_readwrite IS 'Group role to grant write permissions to PgDD objects.'; +Connect to your database using your favorite SQL client. This +could be psql, DBeaver, PgAdmin, or Python code... all you need +is a place to execute SQL code and see the results. -GRANT dd_read TO dd_readwrite; +The main interaction with PgDD is through the views in the `dd` schema. -GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA dd TO dd_readwrite; -ALTER DEFAULT PRIVILEGES IN SCHEMA dd GRANT INSERT, UPDATE, DELETE ON TABLES TO dd_readwrite; -``` +The `dd.views` query can be used to query the views within a database. -This access can be granted using: - -``` -GRANT dd_readwrite TO ; +```sql +SELECT s_name, v_name, description + FROM dd.views +; ``` +```bash +┌────────┬───────────┬────────────────────────────────────────────────────────────────────┐ +│ s_name │ v_name │ description │ +╞════════╪═══════════╪════════════════════════════════════════════════════════════════════╡ +│ dd │ schemas │ Data dictionary view: Lists schemas, excluding system schemas. │ +│ dd │ tables │ Data dictionary view: Lists tables, excluding system tables. │ +│ dd │ views │ Data dictionary view: Lists views, excluding system views. │ +│ dd │ columns │ Data dictionary view: Lists columns, excluding system columns. │ +│ dd │ functions │ Data dictionary view: Lists functions, excluding system functions. │ +└────────┴───────────┴────────────────────────────────────────────────────────────────────┘ +``` +> PgDD views wrap around functions with the same name and enforce `WHERE NOT system_object`. Query the functions to include `system_object` results. e.g. `SELECT s_name, v_name FROM dd.views() WHERE system_object;` -## Use Data Dictionary - -Connect to your database using your favorite SQL client. This -could be psql, DBeaver, PgAdmin, or Python code... all you need -is a place to execute SQL code! ### Schema The highest level of querying provided by `pgdd` is at the schema level. +This provides counts of tables, views and functions along with the size on disk of the objects within the schema. ```sql -SELECT * +SELECT s_name, table_count, view_count, function_count, + size_plus_indexes, description FROM dd.schemas WHERE s_name = 'dd'; ``` -Yields results +Yields results such as this. ```bash -Name |Value | ------------|----------------------------------------------------------| -s_name |dd | -owner |rpl_db_admin | -data_source|Manually maintained | -sensitive |false | -description|Data Dictionary from https://github.com/rustprooflabs/pgdd| -table_count|3 | -size_pretty|48 kB | -size_bytes |49152 | +┌─[ RECORD 1 ]──────┬────────────────────────────────────────────────────────────────────────────────┐ +│ s_name │ dd │ +│ table_count │ 3 │ +│ view_count │ 5 │ +│ function_count │ 6 │ +│ size_plus_indexes │ 144 kB │ +│ description │ Schema for Data Dictionary objects. See https://github.com/rustprooflabs/pgdd │ +└───────────────────┴────────────────────────────────────────────────────────────────────────────────┘ ``` + ### Tables -The query: +The `dd.tables` view to examine tables created and populated by `pgbench`. + ```sql -SELECT t_name, size_pretty, description +SELECT t_name, size_pretty, rows, bytes_per_row FROM dd.tables - WHERE s_name = 'dd'; + WHERE s_name = 'public' + AND t_name LIKE 'pgbench%' + ORDER BY size_bytes DESC; ``` -Yields results ```bash -t_name |size_pretty|description | ------------|-----------|--------------------------------------------------------------| -meta_schema|16 kB |User definable meta-data at the schema level. | -meta_table |16 kB |User definable meta-data at the schema + table level. | -meta_column|16 kB |User definable meta-data at the schema + table + column level.| +┌──────────────────┬─────────────┬──────────┬───────────────┐ +│ t_name │ size_pretty │ rows │ bytes_per_row │ +╞══════════════════╪═════════════╪══════════╪═══════════════╡ +│ pgbench_accounts │ 1281 MB │ 10000000 │ 134 │ +│ pgbench_tellers │ 80 kB │ 1000 │ 82 │ +│ pgbench_branches │ 40 kB │ 100 │ 410 │ +│ pgbench_history │ 0 bytes │ 0 │ ¤ │ +└──────────────────┴─────────────┴──────────┴───────────────┘ ``` -A bit more interesting, with data initialized from `pgbench` and ran for a short while. +### Columns + ```sql -SELECT t_name, size_pretty, rows, bytes_per_row - FROM dd.tables - WHERE s_name IN ('public') - ORDER BY size_bytes DESC; +SELECT source_type, s_name, t_name, c_name, data_type + FROM dd.columns + WHERE data_type LIKE 'int%' +; ``` -Returns: - -```bash -t_name |size_pretty|rows |bytes_per_row | -----------------|-----------|-------|------------------| -pgbench_accounts|130 MB |1000000| 136.68352| -pgbench_history |4072 kB | 77379|53.887075304669224| -pgbench_branches|144 kB | 10| 14745.6| -pgbench_tellers |144 kB | 100| 1474.56| +``` +┌─────────────┬────────┬─────────────┬────────────────┬───────────┐ +│ source_type │ s_name │ t_name │ c_name │ data_type │ +╞═════════════╪════════╪═════════════╪════════════════╪═══════════╡ +│ table │ dd │ meta_schema │ meta_schema_id │ int8 │ +│ table │ dd │ meta_table │ meta_table_id │ int8 │ +│ table │ dd │ meta_column │ meta_column_id │ int8 │ +│ view │ dd │ schemas │ table_count │ int8 │ +│ view │ dd │ schemas │ view_count │ int8 │ +│ view │ dd │ schemas │ function_count │ int8 │ +│ view │ dd │ schemas │ size_bytes │ int8 │ +│ view │ dd │ tables │ size_bytes │ int8 │ +│ view │ dd │ tables │ rows │ int8 │ +│ view │ dd │ tables │ bytes_per_row │ int8 │ +│ view │ dd │ views │ rows │ int8 │ +│ view │ dd │ views │ size_bytes │ int8 │ +│ view │ dd │ columns │ position │ int8 │ +└─────────────┴────────┴─────────────┴────────────────┴───────────┘ ``` -### Columns + + +### Functions + ```sql -SELECT * FROM dd.columns; +SELECT s_name, f_name, argument_data_types, result_data_types + FROM dd.functions +; ``` -### Views + + + +## Database Permissions + +Create Read-only group role to assign to users +that need access to query (read-only) the PgDD objects. ```sql -SELECT * - FROM dd.views - WHERE s_name = 'dd'; +CREATE ROLE dd_read WITH NOLOGIN; +COMMENT ON ROLE dd_read IS 'Group role to grant read-only permissions to PgDD views.'; + +GRANT USAGE ON SCHEMA dd TO dd_read; +GRANT SELECT ON ALL TABLES IN SCHEMA dd TO dd_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA dd GRANT SELECT ON TABLES TO dd_read; ``` -Returns +Access can now be granted to other users using: -```bash -v_name |description | ----------|---------------------------------------------------------| -schemas |Data dictionary view: Lists schemas | -columns |Data dictionary view: Lists columns in tables | -tables |Data dictionary view: Lists tables | -functions|Data dictionary view: Lists functions (procedures) | -views |Data dictionary view: Lists views and materialized views| +```sql +GRANT dd_read TO ; ``` +For read-write access. -### Functions +```sql +CREATE ROLE dd_readwrite WITH NOLOGIN; +COMMENT ON ROLE dd_readwrite IS 'Group role to grant write permissions to PgDD objects.'; + +GRANT dd_read TO dd_readwrite; + +GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA dd TO dd_readwrite; +ALTER DEFAULT PRIVILEGES IN SCHEMA dd GRANT INSERT, UPDATE, DELETE ON TABLES TO dd_readwrite; +``` + +This access can be granted using: ```sql -SELECT * - FROM dd.fuctions; +GRANT dd_readwrite TO ; ``` + + +## PgDD UI + +The [PgDD UI](https://github.com/rustprooflabs/pgdd-ui) project provides +a lightweight Flask interface to the the PgDD extension. + +### Version checking + +PgDD UI handles version checking with the Python `packaging` module so +PgDD versioning must conform to +[PEP 440](https://www.python.org/dev/peps/pep-0440/). + + +## Caveats + +End user caveats: + +* `pg_dump` ignores rows where `s_name = 'dd'` + +Extension developer caveats: + +* DDL changes made in `src/lib.rs` need to be in version-to-version upgrade (e.g. ``sql/pgdd-0.3.1--0.4.0.sql``) + + diff --git a/Makefile b/archive/Makefile similarity index 76% rename from Makefile rename to archive/Makefile index 6054660..2af8b09 100644 --- a/Makefile +++ b/archive/Makefile @@ -2,7 +2,8 @@ EXTENSION = pgdd DATA = pgdd--0.1.sql \ pgdd--0.1.0--0.1.sql \ pgdd--0.1--0.2.sql \ - pgdd--0.2--0.3.sql + pgdd--0.2--0.3.sql \ + pgdd--0.3--0.4alpha1.sql PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) diff --git a/pgdd--0.1--0.2.sql b/archive/pgdd--0.1--0.2.sql similarity index 100% rename from pgdd--0.1--0.2.sql rename to archive/pgdd--0.1--0.2.sql diff --git a/pgdd--0.1.0--0.1.sql b/archive/pgdd--0.1.0--0.1.sql similarity index 100% rename from pgdd--0.1.0--0.1.sql rename to archive/pgdd--0.1.0--0.1.sql diff --git a/pgdd--0.1.sql b/archive/pgdd--0.1.sql similarity index 100% rename from pgdd--0.1.sql rename to archive/pgdd--0.1.sql diff --git a/pgdd--0.2--0.3.sql b/archive/pgdd--0.2--0.3.sql similarity index 100% rename from pgdd--0.2--0.3.sql rename to archive/pgdd--0.2--0.3.sql diff --git a/archive/pgdd--0.3--0.4alpha1.sql b/archive/pgdd--0.3--0.4alpha1.sql new file mode 100644 index 0000000..aa706e3 --- /dev/null +++ b/archive/pgdd--0.3--0.4alpha1.sql @@ -0,0 +1,44 @@ + +DROP VIEW dd."columns"; +CREATE VIEW dd."columns" AS +SELECT n.nspname AS s_name, + CASE c.relkind + WHEN 'r'::"char" THEN 'table'::text + WHEN 'v'::"char" THEN 'view'::text + WHEN 'm'::"char" THEN 'materialized view'::text + WHEN 's'::"char" THEN 'special'::text + WHEN 'f'::"char" THEN 'foreign table'::text + WHEN 'p'::"char" THEN 'table'::text + ELSE NULL::text + END AS type, + c.relname AS t_name, + a.attname AS column_name, + t.typname AS data_type, + a.attnum AS "position", + col_description(c.oid, a.attnum::integer) AS description, + mc.data_source, + mc.sensitive, + CASE WHEN (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND n.nspname !~ '^pg_toast'::text + THEN False + ELSE True + END AS system_object + a.attnotnull, + (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) AS default_expression + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid + AND d.adnum = a.attnum + AND a.atthasdef), + CASE WHEN a.attgenerated = '' THEN False + ELSE True + END AS generated_column + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN dd.meta_column mc ON n.nspname = mc.s_name AND c.relname = mc.t_name AND a.attname = mc.c_name + WHERE (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND n.nspname !~ '^pg_toast'::text AND a.attnum > 0 AND (c.relkind = ANY (ARRAY['r'::"char", 'p'::"char", 's'::"char", 'v'::"char", 'f'::"char", 'm'::"char"])) +; + + +COMMENT ON VIEW dd.columns IS 'Data dictionary view: Lists columns in tables'; +COMMENT ON COLUMN dd.columns.system_object IS 'Allows to easily show/hide system objects.'; diff --git a/archive/pgdd_sql.control b/archive/pgdd_sql.control new file mode 100644 index 0000000..49f502a --- /dev/null +++ b/archive/pgdd_sql.control @@ -0,0 +1,4 @@ +# pgdd extension +comment = 'An in-database data dictionary providing database introspection via standard SQL query syntax.' +default_version = '0.3' +relocatable = false \ No newline at end of file diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 0000000..38dce6d --- /dev/null +++ b/build/build.sh @@ -0,0 +1,79 @@ +# Borrowed heavily from https://github.com/zombodb/zombodb/blob/master/build/build.sh +# +# Copyright 2018-2021 RustProof Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#!/bin/bash + +BUILDDIR=`pwd` +BASE=$(dirname `pwd`) +VERSION=$(cat $BASE/pgdd.control | grep default_version | cut -f2 -d\') +LOGDIR=${BASE}/target/logs +ARTIFACTDIR=${BASE}/target/artifacts +PGXVERSION=0.2.0-beta.1 + +PG_VERS=("pg10" "pg11" "pg12" "pg13") +#PG_VERS=("pg13") + +echo $BASE +echo $VERSION +echo $LOGDIR +echo $ARTIFACTDIR +echo "PGX Version: ${PGXVERSION}" + +mkdir -p ${LOGDIR} +mkdir -p ${ARTIFACTDIR} + +for image in `ls docker/` ; do + + OS_DIST=$(echo ${image}|cut -f2 -d-) + OS_VER=$(echo ${image}|cut -f3 -d-) + + echo $OS_DIST + echo $OS_VER + echo "Pg Version: ${PG_VER}" + + cd ${BUILDDIR} + + cd docker/${image} + echo " Building Docker image: ${image}" + docker build -t ${image} --build-arg PGXVERSION=${PGXVERSION} . 2>&1 > ${LOGDIR}/${image}-build.log || exit 1 + + for PG_VER in ${PG_VERS[@]} ; do + + echo "Build PgDD: ${image}-${PG_VER}" + docker run \ + -e pgver=${PG_VER} \ + -e image=${image} \ + -v ${BASE}:/build \ + --rm \ + ${image} \ + /bin/bash -c '/build/build/package.sh ${pgver} ${image}' \ + > ${LOGDIR}/${image}-${PG_VER}-package.sh.log 2>&1 || echo 'Building this version might have encountered error.' + + echo "${image}-${PG_VER}: finished" + done + + + echo "Copying artifacts for ${OS_DIST} ${OS_VER}" + cd $BASE + + for f in $(find target -name "*.deb") $(find target -name "*.rpm") $(find target -name "*.apk"); do + echo "copy: ${f}" + cp $f $ARTIFACTDIR/ + done +done + +tar -zcvf $BUILDDIR/pgdd-binaries.tar.gz -C $ARTIFACTDIR . + diff --git a/build/docker/pgdd-ubuntu-focal/Dockerfile b/build/docker/pgdd-ubuntu-focal/Dockerfile new file mode 100644 index 0000000..5e701e8 --- /dev/null +++ b/build/docker/pgdd-ubuntu-focal/Dockerfile @@ -0,0 +1,58 @@ +FROM ubuntu:focal + +LABEL maintainer="PgDD Project - https://github.com/rustprooflabs/pgdd" + +ARG USER=docker +ARG UID=1000 +ARG GID=1000 +ARG PGXVERSION + +RUN useradd -m ${USER} --uid=${UID} + + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get upgrade -y && apt-get install -y make wget curl gnupg git +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN apt-get update +RUN apt-get install -y \ + clang-10 llvm-10 clang libz-dev strace pkg-config \ + libxml2 libxml2-dev libreadline8 libreadline-gplv2-dev \ + flex bison libbison-dev build-essential \ + zlib1g-dev libxslt-dev libssl-dev libxml2-utils xsltproc libgss-dev \ + libldap2-dev libkrb5-dev gettext tcl-tclreadline tcl-dev libperl-dev \ + libpython3-dev libprotobuf-c-dev libprotobuf-dev gcc \ + ruby ruby-dev rubygems \ + postgresql-server-dev-10 \ + postgresql-server-dev-11 \ + postgresql-server-dev-12 \ + postgresql-server-dev-13 + + +RUN gem install --no-document fpm + + +USER ${UID}:${GID} +WORKDIR /home/${USER} + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh +ENV PATH="/home/${USER}/.cargo/bin:${PATH}" + +#RUN /bin/bash rustup.sh -y \ +# && cargo install cargo-pgx --version ${PGXVERSION} +RUN /bin/bash rustup.sh -y \ + && cargo install --force --git "https://github.com/zombodb/pgx" \ + --branch "develop" \ + "cargo-pgx" + + +RUN mkdir -p /home/${USER}/.pgx/data-10 \ + && mkdir /home/${USER}/.pgx/data-11 \ + && mkdir /home/${USER}/.pgx/data-12 \ + && mkdir /home/${USER}/.pgx/data-13 + +RUN cargo pgx init \ + --pg10=/usr/lib/postgresql/10/bin/pg_config \ + --pg11=/usr/lib/postgresql/11/bin/pg_config \ + --pg12=/usr/lib/postgresql/12/bin/pg_config \ + --pg13=/usr/lib/postgresql/13/bin/pg_config diff --git a/build/docker/pgdd-ubuntu-hirsute/Dockerfile b/build/docker/pgdd-ubuntu-hirsute/Dockerfile new file mode 100644 index 0000000..0fa8408 --- /dev/null +++ b/build/docker/pgdd-ubuntu-hirsute/Dockerfile @@ -0,0 +1,58 @@ +FROM ubuntu:hirsute + +LABEL maintainer="PgDD Project - https://github.com/rustprooflabs/pgdd" + +ARG USER=docker +ARG UID=1000 +ARG GID=1000 +ARG PGXVERSION + +RUN useradd -m ${USER} --uid=${UID} + + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get upgrade -y && apt-get install -y make wget curl gnupg git +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN apt-get update +RUN apt-get install -y \ + clang-12 llvm-12 clang libz-dev strace pkg-config \ + libxml2 libxml2-dev libreadline8 libreadline-dev \ + flex bison libbison-dev build-essential \ + zlib1g-dev libxslt-dev libssl-dev libxml2-utils xsltproc libgss-dev \ + libldap2-dev libkrb5-dev gettext tcl-tclreadline tcl-dev libperl-dev \ + libpython3-dev libprotobuf-c-dev libprotobuf-dev gcc \ + ruby ruby-dev rubygems \ + postgresql-server-dev-10 \ + postgresql-server-dev-11 \ + postgresql-server-dev-12 \ + postgresql-server-dev-13 + + +RUN gem install --no-document fpm + + +USER ${UID}:${GID} +WORKDIR /home/${USER} + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh +ENV PATH="/home/${USER}/.cargo/bin:${PATH}" + +#RUN /bin/bash rustup.sh -y \ +# && cargo install cargo-pgx --version ${PGXVERSION} +RUN /bin/bash rustup.sh -y \ + && cargo install --force --git "https://github.com/zombodb/pgx" \ + --branch "develop" \ + "cargo-pgx" + + +RUN mkdir -p /home/${USER}/.pgx/data-10 \ + && mkdir /home/${USER}/.pgx/data-11 \ + && mkdir /home/${USER}/.pgx/data-12 \ + && mkdir /home/${USER}/.pgx/data-13 + +RUN cargo pgx init \ + --pg10=/usr/lib/postgresql/10/bin/pg_config \ + --pg11=/usr/lib/postgresql/11/bin/pg_config \ + --pg12=/usr/lib/postgresql/12/bin/pg_config \ + --pg13=/usr/lib/postgresql/13/bin/pg_config diff --git a/build/package.sh b/build/package.sh new file mode 100755 index 0000000..ba81491 --- /dev/null +++ b/build/package.sh @@ -0,0 +1,73 @@ +# Borrowed heavily from https://github.com/zombodb/zombodb/blob/master/build/package.sh +# +# Copyright 2021 RustProof Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#! /bin/bash +set -x + +PG_VER=$1 +IMAGE=$2 + +if [ "x${PG_VER}" == "x" ] || [ "x${IMAGE}" == "x" ] ; then + echo 'usage: ./package.sh ' + exit 1 +fi + +PKG_FORMAT=deb + +echo "Changing to build dir..." +cd /build + +OSNAME=$(echo ${IMAGE} | cut -f3-4 -d-) +VERSION=$(cat pgdd.control | grep default_version | cut -f2 -d\') + +echo "PgDD Building for: ${OSNAME}-${VERSION}" + +PG_CONFIG_DIR=$(dirname $(grep ${PG_VER} ~/.pgx/config.toml | cut -f2 -d= | cut -f2 -d\")) +export PATH=${PG_CONFIG_DIR}:${PATH} + +echo " Packaging pgx" +cargo pgx package || exit $? + + +# +# cd into the package directory +# +cd target/release/pgdd-${PG_VER} || exit $? + +# strip the binaries to make them smaller +find ./ -name "*.so" -exec strip {} \; + +# +# use 'fpm' to build a .deb +# +OUTNAME=pgdd_${VERSION}_${OSNAME}_${PG_VER}_amd64 +if [ "${PKG_FORMAT}" == "deb" ]; then + fpm \ + -s dir \ + -t deb \ + -n pgdd-${OSNAME}-${PG_VER} \ + -v ${VERSION} \ + --deb-no-default-config-files \ + -p ${OUTNAME}.deb \ + -a amd64 \ + . || exit 1 + +else + echo "Unrecognized value for PKG_FORMAT: ${PKG_FORMAT}" + exit 1 +fi + +echo "Packing complete: ${OUTNAME}.deb" diff --git a/pgdd--0.4.0-rc2.jpg b/pgdd--0.4.0-rc2.jpg new file mode 100644 index 0000000..c5659c8 Binary files /dev/null and b/pgdd--0.4.0-rc2.jpg differ diff --git a/pgdd.control b/pgdd.control index 49f502a..b90c8be 100644 --- a/pgdd.control +++ b/pgdd.control @@ -1,4 +1,6 @@ -# pgdd extension -comment = 'An in-database data dictionary providing database introspection via standard SQL query syntax.' -default_version = '0.3' -relocatable = false \ No newline at end of file +comment = 'An in-database data dictionary providing database introspection via standard SQL query syntax. Developed using pgx (https://github.com/zombodb/pgx).' +default_version = '0.4.0.rc2' +module_pathname = '$libdir/pgdd' +relocatable = false +schema = dd +superuser = false diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 0000000..045530a --- /dev/null +++ b/sql/README.md @@ -0,0 +1 @@ +pgx generated SQL files go in this folder. \ No newline at end of file diff --git a/src/bin/sql-generator.rs b/src/bin/sql-generator.rs new file mode 100644 index 0000000..54f8f0b --- /dev/null +++ b/src/bin/sql-generator.rs @@ -0,0 +1,2 @@ +/* Auto-generated by pgx. You may edit this, or delete it to have a new one created. */ +pgx::pg_binary_magic!(pgdd); diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..b2964dd --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,263 @@ +use pgx::*; + +pg_module_magic!(); + +// NOTE: DDL must be defined here to be created by `CREATE EXTENSION pgdd;` +// Changes to DDL must be made BOTH here and +// in the associated version-to-version upgrade script +// +// FIXME: WATCH FOR WHEN THIS COMMENT BECOMES OBSOLETE! ^^^ + +extension_sql_file!("sql/create_extension_tables_all.sql", + bootstrap +); + + +extension_sql_file!("sql/load_default_data.sql", + requires = ["create_extension_tables_all"] +); + + + +#[pg_extern] +fn schemas( +) -> impl std::iter::Iterator), + name!(owner, Option), + name!(data_source, Option), + name!(sensitive, Option), + name!(description, Option), + name!(system_object, Option), + name!(table_count, Option), + name!(view_count, Option), + name!(function_count, Option), + name!(size_pretty, Option), + name!(size_plus_indexes, Option), + name!(size_bytes, Option))> +{ + let query = include_str!("sql/function_query/schemas-all.sql"); + + let mut results = Vec::new(); + Spi::connect(|client| { + client + .select(query, None, None) + .map(|row| (row.by_ordinal(1).unwrap().value::(), + row.by_ordinal(2).unwrap().value::(), + row.by_ordinal(3).unwrap().value::(), + row.by_ordinal(4).unwrap().value::(), + row.by_ordinal(5).unwrap().value::(), + row.by_ordinal(6).unwrap().value::(), + row.by_ordinal(7).unwrap().value::(), + row.by_ordinal(8).unwrap().value::(), + row.by_ordinal(9).unwrap().value::(), + row.by_ordinal(10).unwrap().value::(), + row.by_ordinal(11).unwrap().value::(), + row.by_ordinal(12).unwrap().value::() + )) + .for_each(|tuple| results.push(tuple)); + Ok(Some(())) + }); + + results.into_iter() +} + +#[pg_extern] +fn columns( +) -> impl std::iter::Iterator), + name!(source_type, Option), + name!(t_name, Option), + name!(c_name, Option), + name!(data_type, Option), + name!(position, Option), + name!(description, Option), + name!(data_source, Option), + name!(sensitive, Option), + name!(system_object, Option), + name!(default_value, Option), + name!(generated_column, Option))> +{ + #[cfg(any(feature = "pg10", feature="pg11"))] + let query = include_str!("sql/function_query/columns-pre-12.sql"); + #[cfg(any(feature = "pg12", feature="pg13"))] + let query = include_str!("sql/function_query/columns-12.sql"); + + let mut results = Vec::new(); + Spi::connect(|client| { + client + .select(query, None, None) + .map(|row| (row.by_ordinal(1).unwrap().value::(), + row.by_ordinal(2).unwrap().value::(), + row.by_ordinal(3).unwrap().value::(), + row.by_ordinal(4).unwrap().value::(), + row.by_ordinal(5).unwrap().value::(), + row.by_ordinal(6).unwrap().value::(), + row.by_ordinal(7).unwrap().value::(), + row.by_ordinal(8).unwrap().value::(), + row.by_ordinal(9).unwrap().value::(), + row.by_ordinal(10).unwrap().value::(), + row.by_ordinal(11).unwrap().value::(), + row.by_ordinal(12).unwrap().value::() + )) + .for_each(|tuple| results.push(tuple)); + Ok(Some(())) + }); + + results.into_iter() +} + + +#[pg_extern] +fn functions( +) -> impl std::iter::Iterator), + name!(f_name, Option), + name!(result_data_types, Option), + name!(argument_data_types, Option), + name!(owned_by, Option), + name!(proc_security, Option), + name!(access_privileges, Option), + name!(proc_language, Option), + name!(source_code, Option), + name!(description, Option), + name!(system_object, Option))> +{ + let query = include_str!("sql/function_query/functions-all.sql"); + + let mut results = Vec::new(); + Spi::connect(|client| { + client + .select(query, None, None) + .map(|row| (row.by_ordinal(1).unwrap().value::(), + row.by_ordinal(2).unwrap().value::(), + row.by_ordinal(3).unwrap().value::(), + row.by_ordinal(4).unwrap().value::(), + row.by_ordinal(5).unwrap().value::(), + row.by_ordinal(6).unwrap().value::(), + row.by_ordinal(7).unwrap().value::(), + row.by_ordinal(8).unwrap().value::(), + row.by_ordinal(9).unwrap().value::(), + row.by_ordinal(10).unwrap().value::(), + row.by_ordinal(11).unwrap().value::() + )) + .for_each(|tuple| results.push(tuple)); + Ok(Some(())) + }); + + results.into_iter() +} + + + +#[pg_extern] +fn tables( +) -> impl std::iter::Iterator), + name!(t_name, Option), + name!(type, Option), + name!(owned_by, Option), + name!(size_pretty, Option), + name!(size_bytes, Option), + name!(rows, Option), + name!(bytes_per_row, Option), + name!(size_plus_indexes, Option), + name!(description, Option), + name!(system_object, Option), + name!(data_source, Option), + name!(sensitive, Option))> +{ + let query = include_str!("sql/function_query/tables-all.sql"); + + let mut results = Vec::new(); + Spi::connect(|client| { + client + .select(query, None, None) + .map(|row| (row.by_ordinal(1).unwrap().value::(), + row.by_ordinal(2).unwrap().value::(), + row.by_ordinal(3).unwrap().value::(), + row.by_ordinal(4).unwrap().value::(), + row.by_ordinal(5).unwrap().value::(), + row.by_ordinal(6).unwrap().value::(), + row.by_ordinal(7).unwrap().value::(), + row.by_ordinal(8).unwrap().value::(), + row.by_ordinal(9).unwrap().value::(), + row.by_ordinal(10).unwrap().value::(), + row.by_ordinal(11).unwrap().value::(), + row.by_ordinal(12).unwrap().value::(), + row.by_ordinal(13).unwrap().value::() + )) + .for_each(|tuple| results.push(tuple)); + Ok(Some(())) + }); + + results.into_iter() +} + + + +#[pg_extern] +fn views( +) -> impl std::iter::Iterator), + name!(v_name, Option), + name!(view_type, Option), + name!(owned_by, Option), + name!(rows, Option), + name!(size_pretty, Option), + name!(size_bytes, Option), + name!(description, Option), + name!(system_object, Option))> +{ + let query = include_str!("sql/function_query/views-all.sql"); + + let mut results = Vec::new(); + Spi::connect(|client| { + client + .select(query, None, None) + .map(|row| (row.by_ordinal(1).unwrap().value::(), + row.by_ordinal(2).unwrap().value::(), + row.by_ordinal(3).unwrap().value::(), + row.by_ordinal(4).unwrap().value::(), + row.by_ordinal(5).unwrap().value::(), + row.by_ordinal(6).unwrap().value::(), + row.by_ordinal(7).unwrap().value::(), + row.by_ordinal(8).unwrap().value::(), + row.by_ordinal(9).unwrap().value::() + )) + .for_each(|tuple| results.push(tuple)); + Ok(Some(())) + }); + + results.into_iter() +} + + +#[pg_extern] +fn about() -> &'static str { + "PgDD: PostgreSQL Data Dictionary extension. See https://github.com/rustprooflabs/pgdd for details!" +} + + +extension_sql_file!("sql/create_extension_views_all.sql", + finalize +); + + + +#[cfg(any(test, feature = "pg_test"))] +mod tests { + use pgx::*; + + #[pg_test] + fn test_hello_pgdd_rust() { + assert_eq!("Hello, pgdd_rust", crate::hello_pgdd_rust()); + } + +} + +#[cfg(test)] +pub mod pg_test { + pub fn setup(_options: Vec<&str>) { + // perform one-off initialization when the pg_test framework starts + } + + pub fn postgresql_conf_options() -> Vec<&'static str> { + // return any postgresql.conf settings that are required for your tests + vec![] + } +} diff --git a/src/sql/create_extension_tables_all.sql b/src/sql/create_extension_tables_all.sql new file mode 100644 index 0000000..e772478 --- /dev/null +++ b/src/sql/create_extension_tables_all.sql @@ -0,0 +1,60 @@ +CREATE TABLE dd.meta_schema +( + meta_schema_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + s_name name NOT NULL, + data_source TEXT NULL, + sensitive BOOLEAN NOT NULL DEFAULT False, + CONSTRAINT PK_dd_meta_schema_id PRIMARY KEY (meta_schema_id), + CONSTRAINT UQ_dd_meta_schema_name UNIQUE (s_name) +); + + +CREATE TABLE dd.meta_table +( + meta_table_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + s_name name NOT NULL, + t_name name NOT NULL, + data_source TEXT NULL, + sensitive BOOLEAN NOT NULL DEFAULT False, + CONSTRAINT PK_dd_meta_table_id PRIMARY KEY (meta_table_id), + CONSTRAINT UQ_dd_meta_table_schema_table UNIQUE (s_name, t_name) +); + + +CREATE TABLE dd.meta_column +( + meta_column_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + s_name name NOT NULL, + t_name name NOT NULL, + c_name name NOT NULL, + data_source TEXT NULL, + sensitive BOOLEAN NOT NULL DEFAULT False, + CONSTRAINT PK_dd_meta_column_id PRIMARY KEY (meta_column_id), + CONSTRAINT UQ_dd_meta_column_schema_table_column UNIQUE (s_name, t_name, c_name) +); + + +COMMENT ON SCHEMA dd IS 'Schema for Data Dictionary objects. See https://github.com/rustprooflabs/pgdd'; + +COMMENT ON TABLE dd.meta_schema IS 'User definable meta-data at the schema level.'; +COMMENT ON TABLE dd.meta_table IS 'User definable meta-data at the schema + table level.'; +COMMENT ON TABLE dd.meta_column IS 'User definable meta-data at the schema + table + column level.'; + +COMMENT ON COLUMN dd.meta_schema.meta_schema_id IS 'Primary key for meta table'; +COMMENT ON COLUMN dd.meta_table.meta_table_id IS 'Primary key for meta table'; +COMMENT ON COLUMN dd.meta_column.meta_column_id IS 'Primary key for meta table'; +COMMENT ON COLUMN dd.meta_schema.s_name IS 'Name of the schema for the object described.'; +COMMENT ON COLUMN dd.meta_table.s_name IS 'Name of the schema for the object described.'; +COMMENT ON COLUMN dd.meta_column.s_name IS 'Name of the schema for the object described.'; +COMMENT ON COLUMN dd.meta_table.t_name IS 'Name of the table for the object described.'; +COMMENT ON COLUMN dd.meta_column.t_name IS 'Name of the table for the object described.'; +COMMENT ON COLUMN dd.meta_column.c_name IS 'Name of the column for the object described.'; + +COMMENT ON COLUMN dd.meta_schema.data_source IS 'Optional field to describe the data source(s) for the data in this schema. Most helpful when objects are intentionally organized by schema.'; +COMMENT ON COLUMN dd.meta_table.data_source IS 'Optional field to describe the data source(s) for this table.'; +COMMENT ON COLUMN dd.meta_column.data_source IS 'Optional field to describe the data source(s) for this column.'; +COMMENT ON COLUMN dd.meta_schema.sensitive IS 'Manually updated indicator. Does the schema contain store sensitive data?'; +COMMENT ON COLUMN dd.meta_table.sensitive IS 'Manually updated indicator. Does the table contain store sensitive data?'; +COMMENT ON COLUMN dd.meta_column.sensitive IS 'Manually updated indicator. Does the column contain store sensitive data?'; + + diff --git a/src/sql/create_extension_views_all.sql b/src/sql/create_extension_views_all.sql new file mode 100644 index 0000000..f3ab647 --- /dev/null +++ b/src/sql/create_extension_views_all.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE VIEW dd.schemas AS +SELECT * FROM dd.schemas() + WHERE NOT system_object +; + +CREATE OR REPLACE VIEW dd.tables AS +SELECT * FROM dd.tables() + WHERE NOT system_object +; + +CREATE OR REPLACE VIEW dd.views AS +SELECT * FROM dd.views() + WHERE NOT system_object +; + +CREATE OR REPLACE VIEW dd.columns AS +SELECT * FROM dd.columns() + WHERE NOT system_object +; + +CREATE OR REPLACE VIEW dd.functions AS +SELECT * FROM dd.functions() + WHERE NOT system_object +; + + +COMMENT ON VIEW dd.schemas IS 'Data dictionary view: Lists schemas, excluding system schemas.'; +COMMENT ON VIEW dd.tables IS 'Data dictionary view: Lists tables, excluding system tables.'; +COMMENT ON VIEW dd.views IS 'Data dictionary view: Lists views, excluding system views.'; +COMMENT ON VIEW dd.columns IS 'Data dictionary view: Lists columns, excluding system columns.'; +COMMENT ON VIEW dd.functions IS 'Data dictionary view: Lists functions, excluding system functions.'; + +COMMENT ON FUNCTION dd.about IS 'Basic details about PgDD extension'; + +COMMENT ON FUNCTION dd.schemas IS 'Data dictionary function: Lists all schemas'; +COMMENT ON FUNCTION dd.tables IS 'Data dictionary function: Lists all tables'; +COMMENT ON FUNCTION dd.views IS 'Data dictionary function: Lists all views.'; +COMMENT ON FUNCTION dd.columns IS 'Data dictionary function: Lists all columns'; +COMMENT ON FUNCTION dd.functions IS 'Data dictionary function: Lists all functions'; diff --git a/src/sql/function_query/columns-12.sql b/src/sql/function_query/columns-12.sql new file mode 100644 index 0000000..a4f93d0 --- /dev/null +++ b/src/sql/function_query/columns-12.sql @@ -0,0 +1,37 @@ +-- Adds generated column details for Pg12+ +SELECT n.nspname::TEXT AS s_name, + CASE c.relkind + WHEN 'r'::"char" THEN 'table'::text + WHEN 'v'::"char" THEN 'view'::text + WHEN 'm'::"char" THEN 'materialized view'::text + WHEN 's'::"char" THEN 'special'::text + WHEN 'f'::"char" THEN 'foreign table'::text + WHEN 'p'::"char" THEN 'table'::text + ELSE NULL::text + END AS source_type, + c.relname::TEXT AS t_name, + a.attname::TEXT AS c_name, + t.typname::TEXT AS data_type, + a.attnum AS "position", + col_description(c.oid, a.attnum::integer) AS description, + mc.data_source, + mc.sensitive, + CASE WHEN (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND n.nspname !~ '^pg_toast'::text + THEN False + ELSE True + END AS system_object, + (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) AS default_expression + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid + AND d.adnum = a.attnum + AND a.atthasdef), + CASE WHEN a.attgenerated = '' THEN False + ELSE True + END AS generated_column + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN dd.meta_column mc ON n.nspname = mc.s_name AND c.relname = mc.t_name AND a.attname = mc.c_name + WHERE a.attnum > 0 AND (c.relkind = ANY (ARRAY['r'::"char", 'p'::"char", 's'::"char", 'v'::"char", 'f'::"char", 'm'::"char"])) +; diff --git a/src/sql/function_query/columns-pre-12.sql b/src/sql/function_query/columns-pre-12.sql new file mode 100644 index 0000000..aaf0b61 --- /dev/null +++ b/src/sql/function_query/columns-pre-12.sql @@ -0,0 +1,35 @@ +-- Adds generated column details for Pg12+ +SELECT n.nspname::TEXT AS s_name, + CASE c.relkind + WHEN 'r'::"char" THEN 'table'::text + WHEN 'v'::"char" THEN 'view'::text + WHEN 'm'::"char" THEN 'materialized view'::text + WHEN 's'::"char" THEN 'special'::text + WHEN 'f'::"char" THEN 'foreign table'::text + WHEN 'p'::"char" THEN 'table'::text + ELSE NULL::text + END AS source_type, + c.relname::TEXT AS t_name, + a.attname::TEXT AS c_name, + t.typname::TEXT AS data_type, + a.attnum AS "position", + col_description(c.oid, a.attnum::integer) AS description, + mc.data_source, + mc.sensitive, + CASE WHEN (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND n.nspname !~ '^pg_toast'::text + THEN False + ELSE True + END AS system_object, + (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) AS default_expression + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid + AND d.adnum = a.attnum + AND a.atthasdef) AS default_value, + False AS generated_column + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN dd.meta_column mc ON n.nspname = mc.s_name AND c.relname = mc.t_name AND a.attname = mc.c_name + WHERE a.attnum > 0 AND (c.relkind = ANY (ARRAY['r'::"char", 'p'::"char", 's'::"char", 'v'::"char", 'f'::"char", 'm'::"char"])) +; diff --git a/src/sql/function_query/functions-all.sql b/src/sql/function_query/functions-all.sql new file mode 100644 index 0000000..aa88cb1 --- /dev/null +++ b/src/sql/function_query/functions-all.sql @@ -0,0 +1,20 @@ +SELECT n.nspname::TEXT AS s_name, + p.proname::TEXT AS f_name, + pg_get_function_result(p.oid)::TEXT AS result_data_types, + pg_get_function_arguments(p.oid)::TEXT AS argument_data_types, + pg_get_userbyid(p.proowner)::TEXT AS owned_by, + CASE + WHEN p.prosecdef THEN 'definer'::text + ELSE 'invoker'::text + END AS proc_security, + array_to_string(p.proacl, ''::text) AS access_privileges, + l.lanname::TEXT AS proc_language, + p.prosrc::TEXT AS source_code, + obj_description(p.oid, 'pg_proc'::name)::TEXT AS description, + CASE + WHEN n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name]) THEN false + ELSE true + END AS system_object + FROM pg_proc p + LEFT JOIN pg_namespace n ON n.oid = p.pronamespace + LEFT JOIN pg_language l ON l.oid = p.prolang; \ No newline at end of file diff --git a/src/sql/function_query/schemas-all.sql b/src/sql/function_query/schemas-all.sql new file mode 100644 index 0000000..eb2db0a --- /dev/null +++ b/src/sql/function_query/schemas-all.sql @@ -0,0 +1,49 @@ +WITH s AS ( + SELECT n.oid, + n.nspname AS s_name, + pg_get_userbyid(n.nspowner) AS owner, + ms.data_source, + ms.sensitive, + obj_description(n.oid, 'pg_namespace'::name) AS description, + CASE + WHEN n.nspname !~ '^pg_'::text AND (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) THEN false + ELSE true + END AS system_object + FROM pg_namespace n + LEFT JOIN dd.meta_schema ms ON n.nspname = ms.s_name + ), f AS ( + SELECT n.nspname AS s_name, + count(DISTINCT p.oid) AS function_count + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + GROUP BY n.nspname + ), v AS ( + SELECT n.nspname AS s_name, + count(DISTINCT c_1.oid) AS view_count + FROM pg_class c_1 + JOIN pg_namespace n ON n.oid = c_1.relnamespace + WHERE c_1.relkind = ANY (ARRAY['v'::"char", 'm'::"char"]) + GROUP BY n.nspname + ) +SELECT s.s_name::TEXT, + s.owner::TEXT, + s.data_source::TEXT, + s.sensitive::BOOLEAN, + s.description::TEXT, + s.system_object, + COALESCE(count(c.*), 0::bigint)::BIGINT AS table_count, + COALESCE(v.view_count, 0::bigint)::BIGINT AS view_count, + COALESCE(f.function_count, 0::bigint)::BIGINT AS function_count, + pg_size_pretty(sum(pg_table_size(c.oid::regclass)))::TEXT AS size_pretty, + pg_size_pretty(sum(pg_total_relation_size(c.oid::regclass)))::TEXT AS size_plus_indexes, + sum(pg_table_size(c.oid::regclass))::BIGINT AS size_bytes + FROM s + LEFT JOIN pg_class c + ON s.oid = c.relnamespace + AND (c.relkind = ANY (ARRAY['r'::"char", 'p'::"char"])) + LEFT JOIN f ON f.s_name = s.s_name + LEFT JOIN v ON v.s_name = s.s_name + GROUP BY s.s_name, s.owner, s.data_source, s.sensitive, + s.description, s.system_object, v.view_count, + f.function_count + ; \ No newline at end of file diff --git a/src/sql/function_query/tables-all.sql b/src/sql/function_query/tables-all.sql new file mode 100644 index 0000000..0b9675b --- /dev/null +++ b/src/sql/function_query/tables-all.sql @@ -0,0 +1,31 @@ +-- dd."tables" source +SELECT n.nspname::TEXT AS s_name, + c.relname::TEXT AS t_name, + CASE + WHEN c.relkind = ANY (ARRAY['r'::"char", 'p'::"char"]) THEN 'table'::text + WHEN c.relkind = 's'::"char" THEN 'special'::text + WHEN c.relkind = 'f'::"char" THEN 'foreign table'::text + ELSE NULL::text + END AS type, + pg_get_userbyid(c.relowner)::TEXT AS owned_by, + pg_size_pretty(pg_table_size(c.oid::regclass))::TEXT AS size_pretty, + pg_table_size(c.oid::regclass)::BIGINT AS size_bytes, + c.reltuples::BIGINT AS rows, + CASE + WHEN c.reltuples > 0::BIGINT + THEN (pg_table_size(c.oid::regclass)::double precision / c.reltuples)::BIGINT + ELSE NULL::BIGINT + END AS bytes_per_row, + pg_size_pretty(pg_total_relation_size(c.oid::regclass))::TEXT AS size_plus_indexes, + obj_description(c.oid, 'pg_class'::name)::TEXT AS description, + CASE + WHEN n.nspname !~ '^pg_toast'::text AND (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) THEN false + ELSE true + END AS system_object, + mt.data_source, + mt.sensitive + FROM pg_class c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN dd.meta_table mt ON n.nspname = mt.s_name AND c.relname = mt.t_name + WHERE c.relkind = ANY (ARRAY['r'::"char", 'p'::"char", 's'::"char", 'f'::"char"]) +; \ No newline at end of file diff --git a/src/sql/function_query/views-all.sql b/src/sql/function_query/views-all.sql new file mode 100644 index 0000000..7662b6d --- /dev/null +++ b/src/sql/function_query/views-all.sql @@ -0,0 +1,21 @@ +SELECT n.nspname::TEXT AS s_name, + c.relname::TEXT AS v_name, + CASE c.relkind + WHEN 'v'::"char" THEN 'view'::text + WHEN 'm'::"char" THEN 'materialized view'::text + ELSE NULL::text + END AS view_type, + pg_get_userbyid(c.relowner)::TEXT AS owned_by, + c.reltuples::BIGINT AS rows, + pg_size_pretty(pg_table_size(c.oid::regclass))::TEXT AS size_pretty, + pg_table_size(c.oid::regclass)::BIGINT AS size_bytes, + obj_description(c.oid, 'pg_class'::name)::TEXT AS description, + CASE + WHEN n.nspname !~ '^pg_toast'::text AND (n.nspname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) + THEN false + ELSE true + END AS system_object + FROM pg_class c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE (c.relkind = ANY (ARRAY['v'::"char", 'm'::"char"])) +; \ No newline at end of file diff --git a/src/sql/load_default_data.sql b/src/sql/load_default_data.sql new file mode 100644 index 0000000..e152bda --- /dev/null +++ b/src/sql/load_default_data.sql @@ -0,0 +1,24 @@ +INSERT INTO dd.meta_schema (s_name, data_source, sensitive) + VALUES ('dd', 'Manually maintained', False); + + +INSERT INTO dd.meta_table (s_name, t_name, data_source, sensitive) + VALUES ('dd', 'meta_schema', 'Manually maintained', False); +INSERT INTO dd.meta_table (s_name, t_name, data_source, sensitive) + VALUES ('dd', 'meta_table', 'Manually maintained', False); +INSERT INTO dd.meta_table (s_name, t_name, data_source, sensitive) + VALUES ('dd', 'meta_column', 'Manually maintained', False); + + +INSERT INTO dd.meta_column (s_name, t_name, c_name, data_source, sensitive) + VALUES ('dd', 'meta_column', 'sensitive', 'Manually defined', False) +; + +SELECT pg_catalog.pg_extension_config_dump('dd.meta_schema'::regclass, 'WHERE s_name <> ''dd'' '); +SELECT pg_catalog.pg_extension_config_dump('dd.meta_schema_meta_schema_id_seq'::regclass, ''); + +SELECT pg_catalog.pg_extension_config_dump('dd.meta_table'::regclass, 'WHERE s_name <> ''dd'' '); +SELECT pg_catalog.pg_extension_config_dump('dd.meta_table_meta_table_id_seq'::regclass, ''); + +SELECT pg_catalog.pg_extension_config_dump('dd.meta_column'::regclass, 'WHERE s_name <> ''dd'' '); +SELECT pg_catalog.pg_extension_config_dump('dd.meta_column_meta_column_id_seq'::regclass, ''); diff --git a/standalone/pgdd_0.4.0-dev_bionic_pg10_amd64.deb b/standalone/pgdd_0.4.0-dev_bionic_pg10_amd64.deb new file mode 100644 index 0000000..4d7c583 Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_bionic_pg10_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_bionic_pg11_amd64.deb b/standalone/pgdd_0.4.0-dev_bionic_pg11_amd64.deb new file mode 100644 index 0000000..b43aa85 Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_bionic_pg11_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_bionic_pg12_amd64.deb b/standalone/pgdd_0.4.0-dev_bionic_pg12_amd64.deb new file mode 100644 index 0000000..11371bb Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_bionic_pg12_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_bionic_pg13_amd64.deb b/standalone/pgdd_0.4.0-dev_bionic_pg13_amd64.deb new file mode 100644 index 0000000..b9f6678 Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_bionic_pg13_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_focal_pg10_amd64.deb b/standalone/pgdd_0.4.0-dev_focal_pg10_amd64.deb new file mode 100644 index 0000000..011a769 Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_focal_pg10_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_focal_pg11_amd64.deb b/standalone/pgdd_0.4.0-dev_focal_pg11_amd64.deb new file mode 100644 index 0000000..b5b0805 Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_focal_pg11_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_focal_pg12_amd64.deb b/standalone/pgdd_0.4.0-dev_focal_pg12_amd64.deb new file mode 100644 index 0000000..bfa403b Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_focal_pg12_amd64.deb differ diff --git a/standalone/pgdd_0.4.0-dev_focal_pg13_amd64.deb b/standalone/pgdd_0.4.0-dev_focal_pg13_amd64.deb new file mode 100644 index 0000000..a8d17d8 Binary files /dev/null and b/standalone/pgdd_0.4.0-dev_focal_pg13_amd64.deb differ