Merge remote-tracking branch 'origin/main'

dbterd/adapters/algos/base.py

@@ -152,7 +152,7 @@ def get_table_from_metadata(model_metadata, exposures=[], **kwargs) -> Table:
             ),
         ),
         node_name=node_name,
-        raw_sql="TODO",
+        raw_sql=None,
         database=node_database,
         schema=node_schema,
         columns=[],
@@ -561,6 +561,8 @@ def get_table_map_from_metadata(test_node, **kwargs):
     to_model_possible_values = [
         f"{first_test_parent_resource_type}('{first_test_parent_parts[2]}','{first_test_parent_parts[-1]}')",
         f"{first_test_parent_resource_type}('{first_test_parent_parts[-1]}')",
+        f'{first_test_parent_resource_type}("{first_test_parent_parts[2]}","{first_test_parent_parts[-1]}")',
+        f'{first_test_parent_resource_type}("{first_test_parent_parts[-1]}")',
     ]
     if test_metadata_to in to_model_possible_values:
         return test_parents

dbterd/adapters/algos/test_relationship.py

@@ -101,4 +101,7 @@ def parse(manifest, catalog, **kwargs):
     logger.info(
         f"Collected {len(tables)} table(s) and {len(relationships)} relationship(s)"
     )
-    return (tables, relationships)
+    return (
+        sorted(tables, key=lambda tbl: tbl.node_name),
+        sorted(relationships, key=lambda rel: rel.name),
+    )

dbterd/adapters/targets/mermaid/mermaid_test_relationship.py

@@ -1,3 +1,6 @@
+import re
+from typing import Optional
+
 from dbterd.adapters.algos import test_relationship
 
 
@@ -14,6 +17,56 @@ def run(manifest, catalog, **kwargs):
     return ("output.md", parse(manifest, catalog, **kwargs))
 
 
+def replace_column_name(column_name: str) -> str:
+    """Replace column names containing special characters.
+    To prevent mermaid from not being able to render column names that may contain special characters.
+
+    Args:
+        column_name (str): column name
+
+    Returns:
+        str: Column name with special characters substituted
+    """
+    return column_name.replace(" ", "-").replace(".", "__")
+
+
+def match_complex_column_type(column_type: str) -> Optional[str]:
+    """Returns the root type from nested complex types.
+    As an example, if the input is `Struct<field1 string, field2 string>`, return `Struct`.
+
+    Args:
+        column_type (str): column type
+
+    Returns:
+        Optional[str]: Returns root type if input type is nested complex type, otherwise returns `None` for primitive types
+    """
+    pattern = r"(\w+)<(\w+\s+\w+(\s*,\s*\w+\s+\w+)*)>"
+    match = re.match(pattern, column_type)
+    if match:
+        return match.group(1)
+    else:
+        return None
+
+
+def replace_column_type(column_type: str) -> str:
+    """If type of column contains special characters that cannot be drawn by mermaid, replace them with strings that can be drawn.
+    If the type string contains a nested complex type, omit it to make it easier to read.
+
+    Args:
+        column_type (str): column type
+
+    Returns:
+        str: Type of column with special characters are substituted or omitted
+    """
+    # Some specific DWHs may have types that cannot be drawn in mermaid, such as `Struct<first_name string, last_name string>`.
+    # These types may be nested and can be very long, so omit them
+    complex_column_type = match_complex_column_type(column_type)
+    if complex_column_type:
+        return f"{complex_column_type}[OMITTED]"
+    else:
+        return column_type.replace(" ", "-")
+
+
 def parse(manifest, catalog, **kwargs):
     """Get the Mermaid content from dbt artifacts
 
@@ -35,7 +88,7 @@ def parse(manifest, catalog, **kwargs):
         table_name = table.name.upper()
         columns = "\n".join(
             [
-                f'    {x.data_type.replace(" ","-")} {x.name.replace(" ","-")}'
+                f"    {replace_column_type(x.data_type)} {replace_column_name(x.name)}"
                 for x in table.columns
             ]
         )
@@ -49,9 +102,9 @@ def parse(manifest, catalog, **kwargs):
     for rel in relationships:
         key_from = f'"{rel.table_map[1]}"'
         key_to = f'"{rel.table_map[0]}"'
-        reference_text = rel.column_map[0].replace(" ", "-")
+        reference_text = replace_column_name(rel.column_map[0])
         if rel.column_map[0] != rel.column_map[1]:
-            reference_text += f"--{ rel.column_map[1].replace(' ','-')}"
+            reference_text += f"--{ replace_column_name(rel.column_map[1])}"
         mermaid += f"  {key_from.upper()} {get_rel_symbol(rel.type)} {key_to.upper()}: {reference_text}\n"
 
     return mermaid

docs/nav/guide/cli-references.md

@@ -403,7 +403,9 @@ Check [Download artifacts from a Job Run](./dbt-cloud/download-artifact-from-a-j
 
 ## dbterd run-metadata
 
-Command to generate diagram-as-a-code file by connecting to dbt Cloud Discovery API using GraphQL connection
+Command to generate diagram-as-a-code file by connecting to dbt Cloud Discovery API using GraphQL connection.
+
+Check [this guideline](./dbt-cloud/read-artifact-from-an-environment.md) for more details.
 
 **Examples:**
 === "CLI"

mkdocs.yml

@@ -28,7 +28,7 @@ nav:
         - Download the latest artifacts from a Job: nav/guide/dbt-cloud/download-artifact-from-a-job.md
         - Read the latest artifacts from an environment: nav/guide/dbt-cloud/read-artifact-from-an-environment.md
     - Contribution Guideline ❤️: nav/development/contributing-guide.md
-    - License: license.md
+    - License 🔑: license.md
     - Change Log ↗️: https://github.com/datnguye/dbterd/releases" target="_blank
 
 theme: