feat: Schema.to_mermaid() (#3364)

* Add dlt.Schema.to_mermaid() method

---------

Co-authored-by: jayant <[email protected]>

Author: zilto

dlt/_workspace/cli/_pipeline_command.py

@@ -378,6 +378,8 @@ def _display_pending_packages() -> Tuple[Sequence[str], Sequence[str]]:
             schema_str = s.to_dbml()
         elif format_ == "dot":
             schema_str = s.to_dot()
+        elif format_ == "mermaid":
+            schema_str = s.to_mermaid()
         else:
             schema_str = s.to_pretty_yaml(remove_defaults=remove_defaults_)
 

dlt/_workspace/cli/commands.py

@@ -489,6 +489,8 @@ def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool)
                 schema_str = s.to_dbml()
             elif format_ == "dot":
                 schema_str = s.to_dot()
+            elif format == "mermaid":
+                schema_str = s.to_mermaid()
             else:
                 schema_str = s.to_pretty_yaml(remove_defaults=remove_defaults)
 

dlt/common/schema/schema.py

@@ -795,6 +795,42 @@ def to_dot(
         )
         return dot
 
+    def to_mermaid(
+        self,
+        remove_processing_hints: bool = False,
+        hide_columns: bool = False,
+        hide_descriptions: bool = False,
+        include_dlt_tables: bool = True,
+    ) -> str:
+        """Convert schema to a Mermaid diagram string.
+        Args:
+            remove_processing_hints: If True, remove hints used for data processing and redundant information.
+                This reduces the size of the schema and improves readability.
+            hide_columns: If True, the diagram hides columns details. This helps readability of large diagrams.
+            hide_descriptions: If True, hide the column descriptions
+            include_dlt_tables: If `True` (the default), internal dlt tables (`_dlt_version`,
+                `_dlt_loads`, `_dlt_pipeline_state`)
+
+        Returns:
+            A string containing a Mermaid ERdiagram of the schema.
+        """
+        from dlt.helpers.mermaid import schema_to_mermaid
+
+        stored_schema = self.to_dict(
+            # setting this to `True` removes `name` fields that are used in `schema_to_dbml()`
+            # if required, we can refactor `dlt.helpers.dbml` to support this
+            remove_defaults=False,
+            remove_processing_hints=remove_processing_hints,
+        )
+
+        return schema_to_mermaid(
+            stored_schema,
+            references=self.references,
+            hide_columns=hide_columns,
+            hide_descriptions=hide_descriptions,
+            include_dlt_tables=include_dlt_tables,
+        )
+
     def clone(
         self,
         with_name: str = None,

dlt/common/storages/configuration.py

@@ -22,7 +22,7 @@
 from dlt.common.utils import digest128
 
 
-TSchemaFileFormat = Literal["json", "yaml", "dbml", "dot"]
+TSchemaFileFormat = Literal["json", "yaml", "dbml", "dot", "mermaid"]
 SCHEMA_FILES_EXTENSIONS = get_args(TSchemaFileFormat)
 
 

dlt/common/storages/schema_storage.py

@@ -284,6 +284,8 @@ def _parse_schema_str(schema_str: str, extension: TSchemaFileFormat) -> DictStrA
             raise ValueError(extension, "Schema parser for `dbml` not yet implemented")
         elif extension == "dot":
             raise ValueError(extension, "Schema parser for `dot` not yet implemented")
+        elif extension == "mermaid":
+            raise ValueError(extension, "Schema parser for `mermaid` not yet implemented")
         else:
             raise ValueError(extension)
         return imported_schema

dlt/helpers/mermaid.py

@@ -0,0 +1,125 @@
+"""Build a mermaid graph representation using raw strings without additional dependencies"""
+from enum import Enum
+
+from dlt.common.schema.typing import (
+    TColumnSchema,
+    TReferenceCardinality,
+    TStoredSchema,
+    TTableReferenceStandalone,
+    TTableSchema,
+)
+
+
+INDENT = "    "
+
+
+def schema_to_mermaid(
+    schema: TStoredSchema,
+    *,
+    references: list[TTableReferenceStandalone],
+    hide_columns: bool = False,
+    hide_descriptions: bool = False,
+    include_dlt_tables: bool = True,
+) -> str:
+    mermaid_er_diagram = "erDiagram\n"
+
+    for table_name, table_schema in schema["tables"].items():
+        if not include_dlt_tables and table_name.startswith("_dlt"):
+            continue
+
+        mermaid_er_diagram += INDENT + _to_mermaid_table(
+            table_schema,
+            hide_columns=hide_columns,
+            hide_descriptions=hide_descriptions,
+        )
+
+    for ref in references:
+        if not include_dlt_tables:
+            if ref["table"].startswith("_dlt") or ref["referenced_table"].startswith("_dlt"):
+                continue
+
+        mermaid_er_diagram += INDENT + _to_mermaid_reference(ref)
+
+    return mermaid_er_diagram
+
+
+def _to_mermaid_table(
+    table: TTableSchema, hide_columns: bool = False, hide_descriptions: bool = False
+) -> str:
+    mermaid_table: str = table["name"]
+    mermaid_table += "{\n"
+
+    if hide_columns is False:
+        for column in table["columns"].values():
+            mermaid_table += INDENT + _to_mermaid_column(
+                column,
+                hide_descriptions=hide_descriptions,
+            )
+
+    mermaid_table += "}\n"
+    return mermaid_table
+
+
+# TODO add scale & precision to `data_type`
+def _to_mermaid_column(column: TColumnSchema, hide_descriptions: bool = False) -> str:
+    mermaid_col = column["data_type"] + " " + column["name"]
+    keys = []
+    if column.get("primary_key"):
+        keys.append("PK")
+
+    if column.get("unique"):
+        keys.append("UK")
+
+    if keys:
+        mermaid_col += " " + ",".join(keys)
+
+    if hide_descriptions is False:
+        if description := column.get("description"):
+            mermaid_col += f' "{description}"'
+
+    mermaid_col += "\n"
+    return mermaid_col
+
+
+class TMermaidArrows(str, Enum):
+    ONE_TO_MANY = "||--|{"
+    MANY_TO_ONE = "}|--||"
+    ZERO_TO_MANY = "|o--|{"
+    MANY_TO_ZERO = "}|--o|"
+    ONE_TO_MORE = "||--o{"
+    MORE_TO_ONE = "}o--||"
+    ONE_TO_ONE = "||--||"
+    MANY_TO_MANY = "}|--|{"
+    ZERO_TO_ONE = "|o--o|"
+
+
+_CARDINALITY_ARROW: dict[TReferenceCardinality, TMermaidArrows] = {
+    "one_to_many": TMermaidArrows.ONE_TO_MANY,
+    "many_to_one": TMermaidArrows.MANY_TO_ONE,
+    "zero_to_many": TMermaidArrows.ZERO_TO_MANY,
+    "many_to_zero": TMermaidArrows.MANY_TO_ZERO,
+    "one_to_one": TMermaidArrows.ONE_TO_ONE,
+    "many_to_many": TMermaidArrows.MANY_TO_MANY,
+    "zero_to_one": TMermaidArrows.ZERO_TO_ONE,
+    "one_to_zero": TMermaidArrows.ZERO_TO_ONE,
+}
+
+
+def _to_mermaid_reference(ref: TTableReferenceStandalone) -> str:
+    """Builds references in the following format using cardinality and label to describe
+    the relationship
+
+    <left-entity> [<relationship> <right-entity> : <relationship-label>]
+    """
+    left_table = ref.get("table")
+    right_table = ref.get("referenced_table")
+    cardinality = ref.get("cardinality", "one_to_many")
+    label = ref.get("label", '""')
+    arrow: str = _CARDINALITY_ARROW.get(cardinality).value
+
+    mermaid_reference = f"{left_table} {arrow} {right_table}"
+    if label:
+        mermaid_reference += f" : {label}"
+
+    mermaid_reference += "\n"
+    return mermaid_reference

docs/website/docs/general-usage/dataset-access/view-dlt-schema.md

@@ -837,7 +837,7 @@ TableGroup "_dlt" {
 
 
 ## Export to Graphviz
-[Graphviz](https://www.graphviz.org/) is an open soruce graph visualization engine which uses the [DOT language](https://graphviz.org/doc/info/lang.html). dlt allows you to export your `dlt.Schema` as DOT string, which can be rendered using the Python `graphviz` library, lightweight JS libraries (e.g., [d3-graphviz](https://github.com/magjac/d3-graphviz)), or IDE extensions.
+[Graphviz](https://www.graphviz.org/) is an open source graph visualization engine which uses the [DOT language](https://graphviz.org/doc/info/lang.html). dlt allows you to export your `dlt.Schema` as DOT string, which can be rendered using the Python `graphviz` library, lightweight JS libraries (e.g., [d3-graphviz](https://github.com/magjac/d3-graphviz)), or IDE extensions.
 
 Note that the conversion is lossy. You can't fully recreate `dlt.Schema` from a DOT string.
 
@@ -1278,3 +1278,74 @@ _dlt_version:f4:_ -> _dlt_loads:f2:_ [dir=both, penwidth=1, color="#1c1c34", arr
 </details>
 
 ![graphviz dot render](https://storage.googleapis.com/dlt-blog-images/schema_dot_export.svg)
+
+
+## Export to Mermaid
+
+[Mermaid](https://www.mermaidchart.com/) is a widely-supported diagramming language. dlt allows you to export your `dlt.Schema` as Mermaid string. This can natively rendered by many tools (GitHub markdown, Notion, marimo notebooks).
+
+Note that the conversion is lossy. You can't fully recreate `dlt.Schema` from a Mermaid string.
+
+```py
+schema_mermaid = pipeline.default_schema.to_mermaid()
+```
+
+```sh
+# `chess_pipeline` is the name of the pipeline
+dlt pipeline chess_pipeline schema --format mermaid
+```
+
+<details>
+  <summary>See Mermaid</summary>
+
+  ```mermaid
+    erDiagram
+      _dlt_version{
+      bigint version
+      bigint engine_version
+      timestamp inserted_at
+      text schema_name
+      text version_hash
+      text schema
+  }
+      _dlt_loads{
+      text load_id
+      text schema_name
+      bigint status
+      timestamp inserted_at
+      text schema_version_hash
+  }
+      customers{
+      bigint id PK
+      text name
+      text city
+      text _dlt_load_id
+      text _dlt_id UK
+  }
+      purchases{
+      bigint id PK
+      bigint customer_id
+      bigint inventory_id
+      bigint quantity
+      text date
+      text _dlt_load_id
+      text _dlt_id UK
+  }
+      _dlt_pipeline_state{
+      bigint version
+      bigint engine_version
+      text pipeline_name
+      text state
+      timestamp created_at
+      text version_hash
+      text _dlt_load_id
+      text _dlt_id UK
+  }
+      customers }|--|| _dlt_loads : _dlt_load
+      purchases }|--|| _dlt_loads : _dlt_load
+      purchases ||--|{ customers : ""
+      _dlt_pipeline_state }|--|| _dlt_loads : _dlt_load
+  ```
+</details>
+
+![mermaid render](https://storage.googleapis.com/dlt-blog-images/schema_mermaid_export.png)

docs/website/docs/reference/command-line-interface.md

@@ -89,7 +89,7 @@ Shows, converts and upgrades schemas.
 
 **Usage**
 ```sh
-dlt schema [-h] [--format {json,yaml,dbml,dot}] [--remove-defaults] file
+dlt schema [-h] [--format {json,yaml,dbml,dot,mermaid}] [--remove-defaults] file
 ```
 
 **Description**
@@ -107,7 +107,7 @@ Inherits arguments from [`dlt`](#dlt).
 
 **Options**
 * `-h, --help` - Show this help message and exit
-* `--format {json,yaml,dbml,dot}` - Display schema in this format
+* `--format {json,yaml,dbml,dot,mermaid}` - Display schema in this format
 * `--remove-defaults` - Does not show default hint values
 
 </details>
@@ -334,7 +334,7 @@ Displays default schema.
 
 **Usage**
 ```sh
-dlt pipeline [pipeline_name] schema [-h] [--format {json,yaml,dbml,dot}]
+dlt pipeline [pipeline_name] schema [-h] [--format {json,yaml,dbml,dot,mermaid}]
     [--remove-defaults]
 ```
 
@@ -350,7 +350,7 @@ Inherits arguments from [`dlt pipeline`](#dlt-pipeline).
 
 **Options**
 * `-h, --help` - Show this help message and exit
-* `--format {json,yaml,dbml,dot}` - Display schema in this format
+* `--format {json,yaml,dbml,dot,mermaid}` - Display schema in this format
 * `--remove-defaults` - Does not show default hint values
 
 </details>