diff --git a/demo/config.yaml b/demo/config.yaml new file mode 100644 index 0000000..5d9fa41 --- /dev/null +++ b/demo/config.yaml @@ -0,0 +1,63 @@ +# overrides default values for underlying postgres connection +postgres: + hostname: localhost + port: 5432 + database: postgres + username: postgres + password: postgres + +servers: + mysql_customers: + description: MySQL - Customers + fdw_name: mysql_fdw + foreign_server: + host: mysql + port: 3306 + user_mapping: + username: mysql + password: mysql + + postgres_products: + description: Postgres - Products + fdw_name: postgres_fdw + foreign_server: + host: postgres + port: 5432 + dbname: factory + user_mapping: + user: postgres + password: postgres + + # using system generated name to allow re-use for the manually and automatically created servers + mongo_fdw_1: + description: Mongo - Orders + fdw_name: mongo_fdw + foreign_server: + address: mongo + port: 27017 + authentication_database: admin + user_mapping: + username: mongo + password: mongo + + mssql_employees: + description: MSSQL - Employees + fdw_name: tds_fdw + foreign_server: + servername: mssql + port: 1433 + database: hr + user_mapping: + username: sa + password: Mssql_2019 + + sqlite_job-roles: + description: SQLite - Job Roles + fdw_name: sqlite_fdw + foreign_server: + database: /data/sqlite/job_roles.db + + # using system generated name to allow re-use for the manually and automatically created servers + file_fdw_1: + description: CSV - Departments + fdw_name: file_fdw diff --git a/demo/csv_datero_setup.sql b/demo/csv/datero_setup.sql similarity index 68% rename from demo/csv_datero_setup.sql rename to demo/csv/datero_setup.sql index 409bee8..381514f 100644 --- a/demo/csv_datero_setup.sql +++ b/demo/csv/datero_setup.sql @@ -6,5 +6,5 @@ CREATE FOREIGN TABLE csv.departments , name varchar ) SERVER file_fdw_1 -OPTIONS (filename '/home/data/departments.csv', format 'csv', header 'true') +OPTIONS (filename '/data/departments.csv', format 'csv', header 'true') ; diff --git a/demo/datero_config.yml b/demo/datero_config.yml deleted file mode 100644 index 4775f0d..0000000 --- a/demo/datero_config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# overrides default values for underlying postgres connection -postgres: - hostname: localhost - port: 5432 - database: postgres - username: postgres - password: postgres diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index c3870b6..10aded9 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -10,9 +10,11 @@ services: - 8080:80 - 4444:5432 volumes: - - ./datero_config.yml:/home/instance/config.yaml # Datero config. Must be specified if postgres image default values are changed - - ./sqlite_job_roles.db:/home/data/job_roles.db - - ../data/tutorial/departments.csv:/home/data/departments.csv + # Datero config. Must be specified if postgres image default values are changed. + # Also allows to specify servers explicitly thus avoiding manual creation + #- ./config.yaml:/home/instance/config.yaml + - ./sqlite:/data/sqlite + - ../data/tutorial/departments.csv:/data/departments.csv # --8<-- [end:datero] # --8<-- [start:mysql] @@ -27,7 +29,7 @@ services: ports: - 3306:3306 volumes: - - ./mysql_customers.sql:/docker-entrypoint-initdb.d/setup.sql + - ./mysql/customers.sql:/docker-entrypoint-initdb.d/setup.sql # --8<-- [end:mysql] # --8<-- [start:postgres] @@ -41,7 +43,7 @@ services: ports: - 5432:5432 volumes: - - ./postgres_products.sql:/docker-entrypoint-initdb.d/setup.sql + - ./postgres/products.sql:/docker-entrypoint-initdb.d/setup.sql # --8<-- [end:postgres] # --8<-- [start:mongo] @@ -55,7 +57,7 @@ services: ports: - 27017:27017 volumes: - - ./mongo_orders.js:/docker-entrypoint-initdb.d/setup.js + - ./mongo/orders.js:/docker-entrypoint-initdb.d/setup.js # --8<-- [end:mongo] # --8<-- [start:mssql] @@ -70,8 +72,8 @@ services: entrypoint: - /usr/config/entrypoint.sh volumes: - - ./mssql_entrypoint.sh:/usr/config/entrypoint.sh - - ./mssql_configure_db.sh:/usr/config/configure-db.sh - - ./mssql_employees.sql:/usr/config/setup.sql + - ./mssql/entrypoint.sh:/usr/config/entrypoint.sh + - ./mssql/configure_db.sh:/usr/config/configure-db.sh + - ./mssql/employees.sql:/usr/config/setup.sql # --8<-- [end:mssql] diff --git a/demo/mongo_datero_setup.sql b/demo/mongo/datero_setup.sql similarity index 100% rename from demo/mongo_datero_setup.sql rename to demo/mongo/datero_setup.sql diff --git a/demo/mongo_orders.js b/demo/mongo/orders.js similarity index 100% rename from demo/mongo_orders.js rename to demo/mongo/orders.js diff --git a/demo/mssql/configure_db.sh b/demo/mssql/configure_db.sh new file mode 100755 index 0000000..1bee57e --- /dev/null +++ b/demo/mssql/configure_db.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +check_db_status() { + # Wait 60 seconds for SQL Server to start up by ensuring that + # calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible + # and that system and user databases return "0" which means all databases are in an "online" state + # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017 + + DBSTATUS=1 + ERRCODE=1 + MAX_ITERATIONS=60 + i=0 + + while { [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; } && [[ $i -lt $MAX_ITERATIONS ]]; do + i=$((i+1)) + OUTPUT=$( + /opt/mssql-tools18/bin/sqlcmd -h -1 -t 1 -No \ + -U sa -P $MSSQL_SA_PASSWORD \ + -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases" + ) + echo "OUTPUT: $OUTPUT" # Print the output of sqlcmd + ERRCODE=$? + DBSTATUS=$(echo $OUTPUT | xargs) # This will trim the spaces + sleep 1 + echo "dbstatus: $DBSTATUS, errcode: $ERRCODE, i: $i" + done + + if [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; then + echo "SQL Server took more than $MAX_ITERATIONS seconds to start up or one or more databases are not in an ONLINE state" + exit 1 + fi +} + +# Execute the function twice with a sleep interval +check_db_status +sleep 10 +check_db_status + +# Run the setup script to create the DB and the schema in the DB +/opt/mssql-tools18/bin/sqlcmd -S localhost -No -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql diff --git a/demo/mssql_employees.sql b/demo/mssql/employees.sql similarity index 100% rename from demo/mssql_employees.sql rename to demo/mssql/employees.sql diff --git a/demo/mssql_entrypoint.sh b/demo/mssql/entrypoint.sh similarity index 99% rename from demo/mssql_entrypoint.sh rename to demo/mssql/entrypoint.sh index fda39d1..4f27408 100755 --- a/demo/mssql_entrypoint.sh +++ b/demo/mssql/entrypoint.sh @@ -5,4 +5,3 @@ # Start SQL Server /opt/mssql/bin/sqlservr - diff --git a/demo/mssql_configure_db.sh b/demo/mssql_configure_db.sh deleted file mode 100755 index 9959624..0000000 --- a/demo/mssql_configure_db.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Wait 60 seconds for SQL Server to start up by ensuring that -# calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible -# and that system and user databases return "0" which means all databases are in an "online" state -# https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017 - -DBSTATUS=1 -ERRCODE=1 -i=0 - -while { [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; } && [[ $i -lt 60 ]]; do - i=$((i+1)) - OUTPUT=$(/opt/mssql-tools/bin/sqlcmd -h -1 -t 1 -U sa -P $MSSQL_SA_PASSWORD -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases") - echo "OUTPUT: $OUTPUT" # Print the output of sqlcmd - ERRCODE=$? - DBSTATUS=$(echo $OUTPUT | xargs) # This will trim the spaces - sleep 1 - echo "dbstatus: $DBSTATUS, errcode: $ERRCODE, i: $i" -done - - -if [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; then - echo "SQL Server took more than 60 seconds to start up or one or more databases are not in an ONLINE state" - exit 1 -fi - -# Run the setup script to create the DB and the schema in the DB -/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql diff --git a/demo/mysql_customers.sql b/demo/mysql/customers.sql similarity index 100% rename from demo/mysql_customers.sql rename to demo/mysql/customers.sql diff --git a/demo/postgres_products.sql b/demo/postgres/products.sql similarity index 100% rename from demo/postgres_products.sql rename to demo/postgres/products.sql diff --git a/demo/sqlite_job_roles.db b/demo/sqlite/job_roles.db similarity index 100% rename from demo/sqlite_job_roles.db rename to demo/sqlite/job_roles.db diff --git a/demo/sqlite_job_roles.sql b/demo/sqlite/job_roles.sql similarity index 100% rename from demo/sqlite_job_roles.sql rename to demo/sqlite/job_roles.sql diff --git a/docs/images/tutorial/csv_import_table.jpg b/docs/images/tutorial/csv_import_table.jpg deleted file mode 100644 index abaa3d0..0000000 --- a/docs/images/tutorial/csv_import_table.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:464ecfafd6b4f8cf0df2fcd6dd99019c6d4ca671fe37138536cc5bd54f19958b -size 40031 diff --git a/docs/images/tutorial/sqlite_server.jpg b/docs/images/tutorial/sqlite_server.jpg index 460617e..d7399ea 100644 --- a/docs/images/tutorial/sqlite_server.jpg +++ b/docs/images/tutorial/sqlite_server.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f1f6fe8e1a77693e4875de9b86f300ba2caf67c4b6c86c5b769968c01664c507 -size 23416 +oid sha256:aa45061879f22f06dc708c791d091e513da8e92f6503d2dd075dad55bd5b47d8 +size 23690 diff --git a/docs/tutorial.md b/docs/tutorial.md index 9f54331..afe09e7 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -44,12 +44,12 @@ In terms of datasources, this diagram looks like this: ``` mermaid flowchart BT - customers[("customers\n\nMySQL")] - orders[("orders\n\nMongoDB")] - products[(products\n\nPostgreSQL)] - employees[(employees\n\nMSSQL)] - job_roles[(job_roles\n\nSQLite)] - departments[(departments\n\nCSV)] + customers[("customers
MySQL")] + orders[("orders
MongoDB")] + products[(products
PostgreSQL)] + employees[(employees
MSSQL)] + job_roles[(job_roles
SQLite)] + departments[(departments
CSV)] customers --> orders products --> orders @@ -95,7 +95,8 @@ Here is per-service breakdown of a compose file. !!! info "Main datero container" Web ui is available on port `8080`. - To distinguish from `postgres` datasource we run it on port `4444`. + This is done intentionally to allow run on linux hosts where port `80` might require root permissions. + To distinguish from `postgres` datasource we run `datero` database on port `4444`. For `sqlite` and `csv` datasources we must mount them into the file system of this `datero` container. See corresponding service sections for detals. @@ -143,7 +144,7 @@ Here is per-service breakdown of a compose file. To handle this gracefuly there are `mssql_entrypoint.sh` and `mssql_configure_db.sh` supplementary scripts are used. === "sqlite" - ```yaml linenums="1" hl_lines="12" + ```yaml linenums="1" hl_lines="14" --8<-- "demo/docker-compose.yml:datero" ``` @@ -152,11 +153,12 @@ Here is per-service breakdown of a compose file. It doesn't have any listener over some port to connect to. Hence, we must mount it inside the `datero` container to enable access to it through its file system. - Database `sqlite_job_roles.db` will be mounted to the `/home/data` folder of the container as a `job_roles.db` file. - It contains `job_roles` table defined via `sqlite_job_roles.sql` setup script from the `demo` folder. + Database `job_roles.db` will be mounted to the `/data/sqlite` folder of the container. + This directory is made writable by `other` users to allow `datero` container to write to it. + The database contains `job_roles` table defined via `job_roles.sql` setup script from the `demo/sqlite` folder. === "csv" - ```yaml linenums="1" hl_lines="13" + ```yaml linenums="1" hl_lines="15" --8<-- "demo/docker-compose.yml:datero" ``` @@ -165,14 +167,14 @@ Here is per-service breakdown of a compose file. It doesn't have any listener over some port to connect to. To read the file, it must be accessible from local file system of the `datero` container. - File `departments.csv` from sibling `data/tutorial` directory will be mounted to the `/home/data` folder of the container as `departments.csv` file. + File `departments.csv` from the sibling `../data/tutorial` directory will be mounted to the `/data` folder of the container. To spin-up all the containers, clone this [docs :octicons-tab-external-16:](https://github.com/chumaky/datero-docs){: target="_blank" rel="noopener noreferrer" } repository and run the following command. It will first fetch all the images if they are absent on your local registry and then start all the containers. ``` bash -docker-compose -f demo/docker-compose.yml up -d +docker compose -f demo/docker-compose.yml up -d ``` After execution of this command, you should see output similar to below. @@ -314,6 +316,23 @@ In the left navigation pane of the dashboard in the Connectors section click on +### Automatic servers creation +The previous step is a manual process. But we can do better. +Datero supports automatic servers creation based on the configuration file which could be mounted to the `/home/instance/config.yaml` file. +If you would uncomment the line 13 in the `docker-compose.yml` file and reinstantinate all the containers, then servers will be created automatically. + +```yaml linenums="1" hl_lines="13" +--8<-- "demo/docker-compose.yml:datero" +``` +``` bash +docker compose -f demo/docker-compose.yml down +docker compose -f demo/docker-compose.yml up -d +``` + +Datero supports fully fledged configuration file with all the possible FDW options. +For more details, please refer to the [configuration](../administration/configuration/#configuration-file) section. + + ### Import schemas Once all the servers are created, we can import schemas/databases from them. @@ -338,7 +357,7 @@ Once all the servers are created, we can import schemas/databases from them. To do this, open [Query Editor](overview.md#query-data) and execute the following query: ``` sql title="Mongo foreign table creation" - --8<-- "demo/mongo_datero_setup.sql" + --8<-- "demo/mongo/datero_setup.sql" ```
@@ -367,15 +386,9 @@ Once all the servers are created, we can import schemas/databases from them. To do this, open [Query Editor](overview.md#query-data) and execute the following query: ``` sql title="File based foreign table creation" - --8<-- "demo/csv_datero_setup.sql" + --8<-- "demo/csv/datero_setup.sql" ``` -
- ![File based foreign table](./images/tutorial/csv_import_table.jpg){ loading=lazy } -
File based foreign table
-
- - --8<-- "include/schema_import.md"