actualizing tutorial to support manual and automatic server creation …

…scenarios

demo/config.yaml

@@ -0,0 +1,63 @@
+# overrides default values for underlying postgres connection
+postgres:
+  hostname: localhost
+  port: 5432
+  database: postgres
+  username: postgres
+  password: postgres
+
+servers:
+  mysql_customers:
+    description: MySQL - Customers
+    fdw_name: mysql_fdw
+    foreign_server:
+      host: mysql
+      port: 3306
+    user_mapping:
+      username: mysql
+      password: mysql
+
+  postgres_products:
+    description: Postgres - Products
+    fdw_name: postgres_fdw
+    foreign_server:
+      host: postgres
+      port: 5432
+      dbname: factory
+    user_mapping:
+      user: postgres
+      password: postgres
+
+  # using system generated name to allow re-use for the manually and automatically created servers
+  mongo_fdw_1:
+    description: Mongo - Orders
+    fdw_name: mongo_fdw
+    foreign_server:
+      address: mongo
+      port: 27017
+      authentication_database: admin
+    user_mapping:
+      username: mongo
+      password: mongo
+
+  mssql_employees:
+    description: MSSQL - Employees
+    fdw_name: tds_fdw
+    foreign_server:
+      servername: mssql
+      port: 1433
+      database: hr
+    user_mapping:
+      username: sa
+      password: Mssql_2019
+
+  sqlite_job-roles:
+    description: SQLite - Job Roles
+    fdw_name: sqlite_fdw
+    foreign_server:
+      database: /data/sqlite/job_roles.db
+
+  # using system generated name to allow re-use for the manually and automatically created servers
+  file_fdw_1:
+    description: CSV - Departments
+    fdw_name: file_fdw

demo/csv_datero_setup.sql → demo/csv/datero_setup.sql

@@ -6,5 +6,5 @@ CREATE FOREIGN TABLE csv.departments
 , name      varchar
 )
 SERVER file_fdw_1
-OPTIONS (filename '/home/data/departments.csv', format 'csv', header 'true')
+OPTIONS (filename '/data/departments.csv', format 'csv', header 'true')
 ;

demo/docker-compose.yml

@@ -10,9 +10,11 @@ services:
       - 8080:80
       - 4444:5432
     volumes:
-      - ./datero_config.yml:/home/instance/config.yaml              # Datero config. Must be specified if postgres image default values are changed
-      - ./sqlite_job_roles.db:/home/data/job_roles.db
-      - ../data/tutorial/departments.csv:/home/data/departments.csv
+      # Datero config. Must be specified if postgres image default values are changed.
+      # Also allows to specify servers explicitly thus avoiding manual creation
+      #- ./config.yaml:/home/instance/config.yaml              
+      - ./sqlite:/data/sqlite
+      - ../data/tutorial/departments.csv:/data/departments.csv
   # --8<-- [end:datero]
 
   # --8<-- [start:mysql]
@@ -27,7 +29,7 @@ services:
     ports:
       - 3306:3306
     volumes:
-      - ./mysql_customers.sql:/docker-entrypoint-initdb.d/setup.sql
+      - ./mysql/customers.sql:/docker-entrypoint-initdb.d/setup.sql
   # --8<-- [end:mysql]
 
   # --8<-- [start:postgres]
@@ -41,7 +43,7 @@ services:
     ports:
       - 5432:5432
     volumes:
-      - ./postgres_products.sql:/docker-entrypoint-initdb.d/setup.sql
+      - ./postgres/products.sql:/docker-entrypoint-initdb.d/setup.sql
   # --8<-- [end:postgres]
 
   # --8<-- [start:mongo]
@@ -55,7 +57,7 @@ services:
     ports:
       - 27017:27017
     volumes:
-      - ./mongo_orders.js:/docker-entrypoint-initdb.d/setup.js
+      - ./mongo/orders.js:/docker-entrypoint-initdb.d/setup.js
   # --8<-- [end:mongo]
 
   # --8<-- [start:mssql]
@@ -70,8 +72,8 @@ services:
     entrypoint:
       - /usr/config/entrypoint.sh
     volumes:
-      - ./mssql_entrypoint.sh:/usr/config/entrypoint.sh
-      - ./mssql_configure_db.sh:/usr/config/configure-db.sh
-      - ./mssql_employees.sql:/usr/config/setup.sql
+      - ./mssql/entrypoint.sh:/usr/config/entrypoint.sh
+      - ./mssql/configure_db.sh:/usr/config/configure-db.sh
+      - ./mssql/employees.sql:/usr/config/setup.sql
   # --8<-- [end:mssql]
 

demo/mssql/configure_db.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+check_db_status() {
+    # Wait 60 seconds for SQL Server to start up by ensuring that
+    # calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible
+    # and that system and user databases return "0" which means all databases are in an "online" state
+    # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017
+
+    DBSTATUS=1
+    ERRCODE=1
+    MAX_ITERATIONS=60
+    i=0
+
+    while { [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; } && [[ $i -lt $MAX_ITERATIONS ]]; do
+      i=$((i+1))
+      OUTPUT=$(
+          /opt/mssql-tools18/bin/sqlcmd -h -1 -t 1 -No \
+              -U sa -P $MSSQL_SA_PASSWORD \
+              -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases"
+      )
+      echo "OUTPUT: $OUTPUT"  # Print the output of sqlcmd
+      ERRCODE=$?
+      DBSTATUS=$(echo $OUTPUT | xargs)  # This will trim the spaces
+      sleep 1
+      echo "dbstatus: $DBSTATUS, errcode: $ERRCODE, i: $i"
+    done
+
+    if [[ $DBSTATUS -ne 0 ]] || [[ $ERRCODE -ne 0 ]]; then
+      echo "SQL Server took more than $MAX_ITERATIONS seconds to start up or one or more databases are not in an ONLINE state"
+      exit 1
+    fi
+}
+
+# Execute the function twice with a sleep interval
+check_db_status
+sleep 10
+check_db_status
+
+# Run the setup script to create the DB and the schema in the DB
+/opt/mssql-tools18/bin/sqlcmd -S localhost -No -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql

demo/mssql_entrypoint.sh → demo/mssql/entrypoint.sh

@@ -5,4 +5,3 @@
 
 # Start SQL Server
 /opt/mssql/bin/sqlservr
-

docs/tutorial.md

@@ -44,12 +44,12 @@ In terms of datasources, this diagram looks like this:
 
 ``` mermaid
 flowchart BT
-  customers[("customers\n\nMySQL")]
-  orders[("orders\n\nMongoDB")]
-  products[(products\n\nPostgreSQL)]
-  employees[(employees\n\nMSSQL)]
-  job_roles[(job_roles\n\nSQLite)]
-  departments[(departments\n\nCSV)]
+  customers[("customers<br />MySQL")]
+  orders[("orders<br />MongoDB")]
+  products[(products<br />PostgreSQL)]
+  employees[(employees<br />MSSQL)]
+  job_roles[(job_roles<br />SQLite)]
+  departments[(departments<br />CSV)]
 
   customers --> orders
   products --> orders
@@ -95,7 +95,8 @@ Here is per-service breakdown of a compose file.
 
     !!! info "Main datero container"
         Web ui is available on port `8080`.
-        To distinguish from `postgres` datasource we run it on port `4444`.
+        This is done intentionally to allow run on linux hosts where port `80` might require root permissions.
+        To distinguish from `postgres` datasource we run `datero` database on port `4444`.
         For `sqlite` and `csv` datasources we must mount them into the file system of this `datero` container.
         See corresponding service sections for detals.
 
@@ -143,7 +144,7 @@ Here is per-service breakdown of a compose file.
         To handle this gracefuly there are `mssql_entrypoint.sh` and `mssql_configure_db.sh` supplementary scripts are used.
 
 === "sqlite"
-    ```yaml linenums="1" hl_lines="12"
+    ```yaml linenums="1" hl_lines="14"
     --8<-- "demo/docker-compose.yml:datero"
     ```
 
@@ -152,11 +153,12 @@ Here is per-service breakdown of a compose file.
         It doesn't have any listener over some port to connect to.
         Hence, we must mount it inside the `datero` container to enable access to it through its file system.
 
-        Database `sqlite_job_roles.db` will be mounted to the `/home/data` folder of the container as a `job_roles.db` file.
-        It contains `job_roles` table defined via `sqlite_job_roles.sql` setup script from the `demo` folder.
+        Database `job_roles.db` will be mounted to the `/data/sqlite` folder of the container.
+        This directory is made writable by `other` users to allow `datero` container to write to it.
+        The database contains `job_roles` table defined via `job_roles.sql` setup script from the `demo/sqlite` folder.
 
 === "csv"
-    ```yaml linenums="1" hl_lines="13"
+    ```yaml linenums="1" hl_lines="15"
     --8<-- "demo/docker-compose.yml:datero"
     ```
 
@@ -165,14 +167,14 @@ Here is per-service breakdown of a compose file.
         It doesn't have any listener over some port to connect to.
         To read the file, it must be accessible from local file system of the `datero` container.
 
-        File `departments.csv` from sibling `data/tutorial` directory will be mounted to the `/home/data` folder of the container as `departments.csv` file.
+        File `departments.csv` from the sibling `../data/tutorial` directory will be mounted to the `/data` folder of the container.
 
 
 To spin-up all the containers, clone this [docs :octicons-tab-external-16:](https://github.com/chumaky/datero-docs){: target="_blank" rel="noopener noreferrer" } repository and run the following command.
 It will first fetch all the images if they are absent on your local registry and then start all the containers.
 
 ``` bash
-docker-compose -f demo/docker-compose.yml up -d
+docker compose -f demo/docker-compose.yml up -d
 ```
 
 After execution of this command, you should see output similar to below.
@@ -314,6 +316,23 @@ In the left navigation pane of the dashboard in the Connectors section click on
     </figure>
 
 
+### Automatic servers creation
+The previous step is a manual process. But we can do better.
+Datero supports automatic servers creation based on the configuration file which could be mounted to the `/home/instance/config.yaml` file.
+If you would uncomment the line 13 in the `docker-compose.yml` file and reinstantinate all the containers, then servers will be created automatically.
+
+```yaml linenums="1" hl_lines="13"
+--8<-- "demo/docker-compose.yml:datero"
+```
+``` bash
+docker compose -f demo/docker-compose.yml down
+docker compose -f demo/docker-compose.yml up -d
+```
+
+Datero supports fully fledged configuration file with all the possible FDW options.
+For more details, please refer to the [configuration](../administration/configuration/#configuration-file) section.
+
+
 ### Import schemas
 Once all the servers are created, we can import schemas/databases from them.
 
@@ -338,7 +357,7 @@ Once all the servers are created, we can import schemas/databases from them.
         To do this, open [Query Editor](overview.md#query-data) and execute the following query:
 
     ``` sql title="Mongo foreign table creation"
-    --8<-- "demo/mongo_datero_setup.sql"
+    --8<-- "demo/mongo/datero_setup.sql"
     ```
 
     <figure markdown>
@@ -367,15 +386,9 @@ Once all the servers are created, we can import schemas/databases from them.
         To do this, open [Query Editor](overview.md#query-data) and execute the following query:
 
     ``` sql title="File based foreign table creation"
-    --8<-- "demo/csv_datero_setup.sql"
+    --8<-- "demo/csv/datero_setup.sql"
     ```
 
-    <figure markdown>
-      ![File based foreign table](./images/tutorial/csv_import_table.jpg){ loading=lazy }
-      <figcaption>File based foreign table</figcaption>
-    </figure>
-
-
 --8<-- "include/schema_import.md"