diff --git a/demo/config.yaml b/demo/config.yaml
new file mode 100644
index 0000000..5d9fa41
--- /dev/null
+++ b/demo/config.yaml
@@ -0,0 +1,63 @@
+# overrides default values for underlying postgres connection
+postgres:
+ hostname: localhost
+ port: 5432
+ database: postgres
+ username: postgres
+ password: postgres
+
+servers:
+ mysql_customers:
+ description: MySQL - Customers
+ fdw_name: mysql_fdw
+ foreign_server:
+ host: mysql
+ port: 3306
+ user_mapping:
+ username: mysql
+ password: mysql
+
+ postgres_products:
+ description: Postgres - Products
+ fdw_name: postgres_fdw
+ foreign_server:
+ host: postgres
+ port: 5432
+ dbname: factory
+ user_mapping:
+ user: postgres
+ password: postgres
+
+ # using system generated name to allow re-use for the manually and automatically created servers
+ mongo_fdw_1:
+ description: Mongo - Orders
+ fdw_name: mongo_fdw
+ foreign_server:
+ address: mongo
+ port: 27017
+ authentication_database: admin
+ user_mapping:
+ username: mongo
+ password: mongo
+
+ mssql_employees:
+ description: MSSQL - Employees
+ fdw_name: tds_fdw
+ foreign_server:
+ servername: mssql
+ port: 1433
+ database: hr
+ user_mapping:
+ username: sa
+ password: Mssql_2019
+
+ sqlite_job-roles:
+ description: SQLite - Job Roles
+ fdw_name: sqlite_fdw
+ foreign_server:
+ database: /data/sqlite/job_roles.db
+
+ # using system generated name to allow re-use for the manually and automatically created servers
+ file_fdw_1:
+ description: CSV - Departments
+ fdw_name: file_fdw
diff --git a/demo/csv_datero_setup.sql b/demo/csv/datero_setup.sql
similarity index 68%
rename from demo/csv_datero_setup.sql
rename to demo/csv/datero_setup.sql
index 409bee8..381514f 100644
--- a/demo/csv_datero_setup.sql
+++ b/demo/csv/datero_setup.sql
@@ -6,5 +6,5 @@ CREATE FOREIGN TABLE csv.departments
, name varchar
)
SERVER file_fdw_1
-OPTIONS (filename '/home/data/departments.csv', format 'csv', header 'true')
+OPTIONS (filename '/data/departments.csv', format 'csv', header 'true')
;
diff --git a/demo/datero_config.yml b/demo/datero_config.yml
deleted file mode 100644
index 4775f0d..0000000
--- a/demo/datero_config.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-# overrides default values for underlying postgres connection
-postgres:
- hostname: localhost
- port: 5432
- database: postgres
- username: postgres
- password: postgres
diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml
index c3870b6..10aded9 100644
--- a/demo/docker-compose.yml
+++ b/demo/docker-compose.yml
@@ -10,9 +10,11 @@ services:
- 8080:80
- 4444:5432
volumes:
- - ./datero_config.yml:/home/instance/config.yaml # Datero config. Must be specified if postgres image default values are changed
- - ./sqlite_job_roles.db:/home/data/job_roles.db
- - ../data/tutorial/departments.csv:/home/data/departments.csv
+ # Datero config. Must be specified if postgres image default values are changed.
+ # Also allows to specify servers explicitly thus avoiding manual creation
+ #- ./config.yaml:/home/instance/config.yaml
+ - ./sqlite:/data/sqlite
+ - ../data/tutorial/departments.csv:/data/departments.csv
# --8<-- [end:datero]
# --8<-- [start:mysql]
@@ -27,7 +29,7 @@ services:
ports:
- 3306:3306
volumes:
- - ./mysql_customers.sql:/docker-entrypoint-initdb.d/setup.sql
+ - ./mysql/customers.sql:/docker-entrypoint-initdb.d/setup.sql
# --8<-- [end:mysql]
# --8<-- [start:postgres]
@@ -41,7 +43,7 @@ services:
ports:
- 5432:5432
volumes:
- - ./postgres_products.sql:/docker-entrypoint-initdb.d/setup.sql
+ - ./postgres/products.sql:/docker-entrypoint-initdb.d/setup.sql
# --8<-- [end:postgres]
# --8<-- [start:mongo]
@@ -55,7 +57,7 @@ services:
ports:
- 27017:27017
volumes:
- - ./mongo_orders.js:/docker-entrypoint-initdb.d/setup.js
+ - ./mongo/orders.js:/docker-entrypoint-initdb.d/setup.js
# --8<-- [end:mongo]
# --8<-- [start:mssql]
@@ -70,8 +72,8 @@ services:
entrypoint:
- /usr/config/entrypoint.sh
volumes:
- - ./mssql_entrypoint.sh:/usr/config/entrypoint.sh
- - ./mssql_configure_db.sh:/usr/config/configure-db.sh
- - ./mssql_employees.sql:/usr/config/setup.sql
+ - ./mssql/entrypoint.sh:/usr/config/entrypoint.sh
+ - ./mssql/configure_db.sh:/usr/config/configure-db.sh
+ - ./mssql/employees.sql:/usr/config/setup.sql
# --8<-- [end:mssql]
diff --git a/demo/mongo_datero_setup.sql b/demo/mongo/datero_setup.sql
similarity index 100%
rename from demo/mongo_datero_setup.sql
rename to demo/mongo/datero_setup.sql
diff --git a/demo/mongo_orders.js b/demo/mongo/orders.js
similarity index 100%
rename from demo/mongo_orders.js
rename to demo/mongo/orders.js
diff --git a/demo/mssql/configure_db.sh b/demo/mssql/configure_db.sh
new file mode 100755
index 0000000..1bee57e
--- /dev/null
+++ b/demo/mssql/configure_db.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+check_db_status() {
+ # Wait 60 seconds for SQL Server to start up by ensuring that
+ # calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible
+ # and that system and user databases return "0" which means all databases are in an "online" state
+ # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017
+
+ DBSTATUS=1
+ ERRCODE=1
+ MAX_ITERATIONS=60
+ i=0
+
+ while { [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; } && [[ $i -lt $MAX_ITERATIONS ]]; do
+ i=$((i+1))
+ OUTPUT=$(
+ /opt/mssql-tools18/bin/sqlcmd -h -1 -t 1 -No \
+ -U sa -P $MSSQL_SA_PASSWORD \
+ -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases"
+ )
+ echo "OUTPUT: $OUTPUT" # Print the output of sqlcmd
+ ERRCODE=$?
+ DBSTATUS=$(echo $OUTPUT | xargs) # This will trim the spaces
+ sleep 1
+ echo "dbstatus: $DBSTATUS, errcode: $ERRCODE, i: $i"
+ done
+
+ if [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; then
+ echo "SQL Server took more than $MAX_ITERATIONS seconds to start up or one or more databases are not in an ONLINE state"
+ exit 1
+ fi
+}
+
+# Execute the function twice with a sleep interval
+check_db_status
+sleep 10
+check_db_status
+
+# Run the setup script to create the DB and the schema in the DB
+/opt/mssql-tools18/bin/sqlcmd -S localhost -No -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql
diff --git a/demo/mssql_employees.sql b/demo/mssql/employees.sql
similarity index 100%
rename from demo/mssql_employees.sql
rename to demo/mssql/employees.sql
diff --git a/demo/mssql_entrypoint.sh b/demo/mssql/entrypoint.sh
similarity index 99%
rename from demo/mssql_entrypoint.sh
rename to demo/mssql/entrypoint.sh
index fda39d1..4f27408 100755
--- a/demo/mssql_entrypoint.sh
+++ b/demo/mssql/entrypoint.sh
@@ -5,4 +5,3 @@
# Start SQL Server
/opt/mssql/bin/sqlservr
-
diff --git a/demo/mssql_configure_db.sh b/demo/mssql_configure_db.sh
deleted file mode 100755
index 9959624..0000000
--- a/demo/mssql_configure_db.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Wait 60 seconds for SQL Server to start up by ensuring that
-# calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible
-# and that system and user databases return "0" which means all databases are in an "online" state
-# https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017
-
-DBSTATUS=1
-ERRCODE=1
-i=0
-
-while { [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; } && [[ $i -lt 60 ]]; do
- i=$((i+1))
- OUTPUT=$(/opt/mssql-tools/bin/sqlcmd -h -1 -t 1 -U sa -P $MSSQL_SA_PASSWORD -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases")
- echo "OUTPUT: $OUTPUT" # Print the output of sqlcmd
- ERRCODE=$?
- DBSTATUS=$(echo $OUTPUT | xargs) # This will trim the spaces
- sleep 1
- echo "dbstatus: $DBSTATUS, errcode: $ERRCODE, i: $i"
-done
-
-
-if [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; then
- echo "SQL Server took more than 60 seconds to start up or one or more databases are not in an ONLINE state"
- exit 1
-fi
-
-# Run the setup script to create the DB and the schema in the DB
-/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql
diff --git a/demo/mysql_customers.sql b/demo/mysql/customers.sql
similarity index 100%
rename from demo/mysql_customers.sql
rename to demo/mysql/customers.sql
diff --git a/demo/postgres_products.sql b/demo/postgres/products.sql
similarity index 100%
rename from demo/postgres_products.sql
rename to demo/postgres/products.sql
diff --git a/demo/sqlite_job_roles.db b/demo/sqlite/job_roles.db
similarity index 100%
rename from demo/sqlite_job_roles.db
rename to demo/sqlite/job_roles.db
diff --git a/demo/sqlite_job_roles.sql b/demo/sqlite/job_roles.sql
similarity index 100%
rename from demo/sqlite_job_roles.sql
rename to demo/sqlite/job_roles.sql
diff --git a/docs/images/tutorial/csv_import_table.jpg b/docs/images/tutorial/csv_import_table.jpg
deleted file mode 100644
index abaa3d0..0000000
--- a/docs/images/tutorial/csv_import_table.jpg
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:464ecfafd6b4f8cf0df2fcd6dd99019c6d4ca671fe37138536cc5bd54f19958b
-size 40031
diff --git a/docs/images/tutorial/sqlite_server.jpg b/docs/images/tutorial/sqlite_server.jpg
index 460617e..d7399ea 100644
--- a/docs/images/tutorial/sqlite_server.jpg
+++ b/docs/images/tutorial/sqlite_server.jpg
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:f1f6fe8e1a77693e4875de9b86f300ba2caf67c4b6c86c5b769968c01664c507
-size 23416
+oid sha256:aa45061879f22f06dc708c791d091e513da8e92f6503d2dd075dad55bd5b47d8
+size 23690
diff --git a/docs/tutorial.md b/docs/tutorial.md
index 9f54331..afe09e7 100644
--- a/docs/tutorial.md
+++ b/docs/tutorial.md
@@ -44,12 +44,12 @@ In terms of datasources, this diagram looks like this:
``` mermaid
flowchart BT
- customers[("customers\n\nMySQL")]
- orders[("orders\n\nMongoDB")]
- products[(products\n\nPostgreSQL)]
- employees[(employees\n\nMSSQL)]
- job_roles[(job_roles\n\nSQLite)]
- departments[(departments\n\nCSV)]
+ customers[("customers MySQL")]
+ orders[("orders MongoDB")]
+ products[(products PostgreSQL)]
+ employees[(employees MSSQL)]
+ job_roles[(job_roles SQLite)]
+ departments[(departments CSV)]
customers --> orders
products --> orders
@@ -95,7 +95,8 @@ Here is per-service breakdown of a compose file.
!!! info "Main datero container"
Web ui is available on port `8080`.
- To distinguish from `postgres` datasource we run it on port `4444`.
+ This is done intentionally to allow run on linux hosts where port `80` might require root permissions.
+ To distinguish from `postgres` datasource we run `datero` database on port `4444`.
For `sqlite` and `csv` datasources we must mount them into the file system of this `datero` container.
See corresponding service sections for detals.
@@ -143,7 +144,7 @@ Here is per-service breakdown of a compose file.
To handle this gracefuly there are `mssql_entrypoint.sh` and `mssql_configure_db.sh` supplementary scripts are used.
=== "sqlite"
- ```yaml linenums="1" hl_lines="12"
+ ```yaml linenums="1" hl_lines="14"
--8<-- "demo/docker-compose.yml:datero"
```
@@ -152,11 +153,12 @@ Here is per-service breakdown of a compose file.
It doesn't have any listener over some port to connect to.
Hence, we must mount it inside the `datero` container to enable access to it through its file system.
- Database `sqlite_job_roles.db` will be mounted to the `/home/data` folder of the container as a `job_roles.db` file.
- It contains `job_roles` table defined via `sqlite_job_roles.sql` setup script from the `demo` folder.
+ Database `job_roles.db` will be mounted to the `/data/sqlite` folder of the container.
+ This directory is made writable by `other` users to allow `datero` container to write to it.
+ The database contains `job_roles` table defined via `job_roles.sql` setup script from the `demo/sqlite` folder.
=== "csv"
- ```yaml linenums="1" hl_lines="13"
+ ```yaml linenums="1" hl_lines="15"
--8<-- "demo/docker-compose.yml:datero"
```
@@ -165,14 +167,14 @@ Here is per-service breakdown of a compose file.
It doesn't have any listener over some port to connect to.
To read the file, it must be accessible from local file system of the `datero` container.
- File `departments.csv` from sibling `data/tutorial` directory will be mounted to the `/home/data` folder of the container as `departments.csv` file.
+ File `departments.csv` from the sibling `../data/tutorial` directory will be mounted to the `/data` folder of the container.
To spin-up all the containers, clone this [docs :octicons-tab-external-16:](https://github.com/chumaky/datero-docs){: target="_blank" rel="noopener noreferrer" } repository and run the following command.
It will first fetch all the images if they are absent on your local registry and then start all the containers.
``` bash
-docker-compose -f demo/docker-compose.yml up -d
+docker compose -f demo/docker-compose.yml up -d
```
After execution of this command, you should see output similar to below.
@@ -314,6 +316,23 @@ In the left navigation pane of the dashboard in the Connectors section click on
+### Automatic servers creation
+The previous step is a manual process. But we can do better.
+Datero supports automatic servers creation based on the configuration file which could be mounted to the `/home/instance/config.yaml` file.
+If you would uncomment the line 13 in the `docker-compose.yml` file and reinstantinate all the containers, then servers will be created automatically.
+
+```yaml linenums="1" hl_lines="13"
+--8<-- "demo/docker-compose.yml:datero"
+```
+``` bash
+docker compose -f demo/docker-compose.yml down
+docker compose -f demo/docker-compose.yml up -d
+```
+
+Datero supports fully fledged configuration file with all the possible FDW options.
+For more details, please refer to the [configuration](../administration/configuration/#configuration-file) section.
+
+
### Import schemas
Once all the servers are created, we can import schemas/databases from them.
@@ -338,7 +357,7 @@ Once all the servers are created, we can import schemas/databases from them.
To do this, open [Query Editor](overview.md#query-data) and execute the following query:
``` sql title="Mongo foreign table creation"
- --8<-- "demo/mongo_datero_setup.sql"
+ --8<-- "demo/mongo/datero_setup.sql"
```
@@ -367,15 +386,9 @@ Once all the servers are created, we can import schemas/databases from them.
To do this, open [Query Editor](overview.md#query-data) and execute the following query:
``` sql title="File based foreign table creation"
- --8<-- "demo/csv_datero_setup.sql"
+ --8<-- "demo/csv/datero_setup.sql"
```
-
- ![File based foreign table](./images/tutorial/csv_import_table.jpg){ loading=lazy }
- File based foreign table
-
-
-
--8<-- "include/schema_import.md"