Dashboard Improvements (#2965)

* remove uneeded file

* fix forwarding of pipelines dir to marimo app

* disable state sync and display all schemas and remote state and schemas in pipeline overview

* add support for multiple schemas

* fix e2e tests, further updates pending

* use dropdown instead of multiselect for schema selection
add multi schema pipeline to fixtures

* add last run info in pipeline overview
add buttons to open pipeline folder and local data folder if present

* fix loads browser to select correct schema

* allow to start dashboard for a pipeline that is not there yet and add helpful error message in this case

* nicer last run time formatting
show pipeline error screen also when manually chnaing the pipeline name in the url

* move buttons to top, add refresh buttons to sections

* use raw query when constructing queries

* lazy load remote state tab

* fix traces and trace typing (mostly)

* add exception traces to ui

* add file watcher

* remove test code

* add source and resource state viewer to data panel

* update existing unit tests

* add unit test for new utils

* make marimo dashboard the default app for pipeline show

* update docs

* update existing e2e tests for new yaml based rendering of state

* move streamlit app down in sidebar

* grammar fixes for dashboard strings

* open duckdb in readme mode in datapanel in dashboard

* remove old tests
re-enable dashboard main command

* add missing args to dashboard command

* small fixes to e2e tests

* add tests for exceptions

* re-organize e2e tests into invidual tests

* add basic schema selection checks

* improve dashboard help and dashboard docs page

* short some strings in testing to make selecting predictable

* merge devel

* typo

---------

Co-authored-by: djudjuu <[email protected]>

Author: sh-rp

.github/workflows/test_common.yml

@@ -154,7 +154,7 @@ jobs:
       # Run pipeline dashboard e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
       - name: Run dashboard e2e
         run: |
-          marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines_dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
+          marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.os != 'windows-latest' && matrix.python-version != '3.14.0-beta.4'
 
   matrix_job_required_check:

Makefile

@@ -167,4 +167,4 @@ test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
 start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines_dir _storage/.dlt/pipelines --with_test_identifiers true
+	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true

dlt/cli/command_wrappers.py

@@ -154,10 +154,10 @@ def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool)
 
 
 @utils.track_command("dashboard", True)
-def dashboard_command_wrapper() -> None:
+def dashboard_command_wrapper(pipelines_dir: Optional[str], edit: bool) -> None:
     from dlt.helpers.dashboard.runner import run_dashboard
 
-    run_dashboard()
+    run_dashboard(pipelines_dir=pipelines_dir, edit=edit)
 
 
 @utils.track_command("telemetry", False)

dlt/cli/pipeline_command.py

@@ -52,6 +52,15 @@ def pipeline_command(
             fmt.secho(_dir, fg="green")
         return
 
+    # we may open the dashboard for a pipeline without checking if it exists
+    if operation == "show" and not command_kwargs.get("streamlit"):
+        from dlt.common.utils import custom_environ
+        from dlt.common.known_env import DLT_DATA_DIR
+
+        from dlt.helpers.dashboard.runner import run_dashboard
+
+        run_dashboard(pipeline_name, edit=command_kwargs.get("edit"), pipelines_dir=pipelines_dir)
+
     try:
         if verbosity > 0:
             fmt.echo("Attaching to pipeline %s" % fmt.bold(pipeline_name))
@@ -117,39 +126,30 @@ def _display_pending_packages() -> Tuple[Sequence[str], Sequence[str]]:
     fmt.echo("Found pipeline %s in %s" % (fmt.bold(p.pipeline_name), fmt.bold(p.pipelines_dir)))
 
     if operation == "show":
-        if command_kwargs.get("dashboard"):
-            from dlt.common.utils import custom_environ
-            from dlt.common.known_env import DLT_DATA_DIR
-
-            from dlt.helpers.dashboard.runner import run_dashboard
-
-            with custom_environ({DLT_DATA_DIR: os.path.dirname(p.pipelines_dir)}):
-                run_dashboard(pipeline_name, edit=command_kwargs.get("edit"))
-        else:
-            from dlt.common.runtime import signals
-            from dlt.helpers.streamlit_app import index
-
-            with signals.delayed_signals():
-                streamlit_cmd = [
-                    "streamlit",
-                    "run",
-                    index.__file__,
-                    "--client.showSidebarNavigation",
-                    "false",
-                ]
-
-                if hot_reload:
-                    streamlit_cmd.append("--server.runOnSave")
-                    streamlit_cmd.append("true")
-
-                streamlit_cmd.append("--")
-                streamlit_cmd.append(pipeline_name)
-                streamlit_cmd.append("--pipelines-dir")
-                streamlit_cmd.append(p.pipelines_dir)
-
-                venv = Venv.restore_current()
-                for line in iter_stdout(venv, *streamlit_cmd):
-                    fmt.echo(line)
+        from dlt.common.runtime import signals
+        from dlt.helpers.streamlit_app import index
+
+        with signals.delayed_signals():
+            streamlit_cmd = [
+                "streamlit",
+                "run",
+                index.__file__,
+                "--client.showSidebarNavigation",
+                "false",
+            ]
+
+            if hot_reload:
+                streamlit_cmd.append("--server.runOnSave")
+                streamlit_cmd.append("true")
+
+            streamlit_cmd.append("--")
+            streamlit_cmd.append(pipeline_name)
+            streamlit_cmd.append("--pipelines-dir")
+            streamlit_cmd.append(p.pipelines_dir)
+
+            venv = Venv.restore_current()
+            for line in iter_stdout(venv, *streamlit_cmd):
+                fmt.echo(line)
 
     if operation == "info":
         state: TSourceState = p.state  # type: ignore

dlt/cli/plugins.py

@@ -206,30 +206,29 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None:
                 "Generates and launches Streamlit app with the loading status and dataset explorer"
             ),
             description="""
-Generates and launches Streamlit (https://streamlit.io/) app with the loading status and dataset explorer.
+Launches the pipeline dashboard app with a comprehensive interface to inspect the pipeline state, schemas, and data in the destination.
 
-This is a simple app that you can use to inspect the schemas and data in the destination as well as your pipeline state and loading status/stats. It should be executed from the same folder from which you ran the pipeline script to access destination credentials.
+This app should be executed from the same folder from which you ran the pipeline script to be able access destination credentials.
 
-Requires `streamlit` to be installed in the current environment: `pip install streamlit`. Using --dashboard flag to launch pipeline dashboard preview instead of streamlit.
+If the --edit flag is used, will launch the editable version of the app if it exists in the current directory, or create this version and launch it in edit mode.
+
+Requires `marimo` to be installed in the current environment: `pip install marimo`. Use the --streamlit flag to launch the legacy streamlit app.
 """,
         )
         show_cmd.add_argument(
-            "--dashboard",
+            "--streamlit",
             default=False,
             action="store_true",
-            help=(
-                "Launch pipeline dashboard instead of streamlit. Will launch editable version of"
-                " app (created with the --edit flag) if it exists in the current directory."
-            ),
+            help="Launch the legacy Streamlit dashboard instead of the new pipeline dashboard. ",
         )
         show_cmd.add_argument(
             "--edit",
             default=False,
             action="store_true",
             help=(
                 "Creates editable version of pipeline dashboard in current directory if it does not"
-                " exist there yet and launches it in edit mode. Only works when using the pipeline"
-                " dashboard (--dashboard flag)."
+                " exist there yet and launches it in edit mode. Will have no effect when using the"
+                " streamlit flag."
             ),
         )
         pipeline_subparsers.add_parser(
@@ -494,9 +493,18 @@ class DashboardCommand(SupportsCliCommand):
 
     def configure_parser(self, parser: argparse.ArgumentParser) -> None:
         self.parser = parser
+        self.parser.add_argument(
+            "--pipelines-dir", help="Pipelines working directory", default=None
+        )
+        self.parser.add_argument(
+            "--edit",
+            action="store_true",
+            help="Eject Dashboard and start editable version",
+            default=None,
+        )
 
     def execute(self, args: argparse.Namespace) -> None:
-        dashboard_command_wrapper()
+        dashboard_command_wrapper(pipelines_dir=args.pipelines_dir, edit=args.edit)
 
 
 class TelemetryCommand(SupportsCliCommand):
@@ -744,9 +752,9 @@ def plug_cli_schema() -> Type[SupportsCliCommand]:
 
 
 # TODO: define actual command and re-enable
-# @plugins.hookimpl(specname="plug_cli")
-# def plug_cli_dashboard() -> Type[SupportsCliCommand]:
-#     return DashboardCommand
+@plugins.hookimpl(specname="plug_cli")
+def plug_cli_dashboard() -> Type[SupportsCliCommand]:
+    return DashboardCommand
 
 
 @plugins.hookimpl(specname="plug_cli")

dlt/common/destination/dataset.py

@@ -449,9 +449,12 @@ def __exit__(
         """Context manager to keep the connection to the destination open between queries"""
         ...
 
-    def ibis(self) -> IbisBackend:
+    def ibis(self, read_only: bool = False) -> IbisBackend:
         """Returns a connected ibis backend for the dataset. Not implemented for all destinations.
 
+        Args:
+            read_only (bool): Whether to open the connection in read only mode. Only used for duckdb.
+
         Returns:
             IbisBackend: The ibis backend for the dataset
         """

dlt/dataset/dataset.py

@@ -59,13 +59,14 @@ def __init__(
         self._opened_sql_client: SqlClientBase[Any] = None
         self._table_client: SupportsOpenTables = None
 
-    def ibis(self) -> IbisBackend:
-        """return a connected ibis backend"""
+    def ibis(self, read_only: bool = False) -> IbisBackend:
+        """return a connected ibis backend, read-only flag is only used for duckdb"""
         from dlt.helpers.ibis import create_ibis_backend
 
         return create_ibis_backend(
             self._destination,
             self._get_destination_client(self.schema),
+            read_only=read_only,
         )
 
     @property

dlt/dataset/relation.py

@@ -219,10 +219,10 @@ def cursor(self) -> Generator[SupportsDataAccess, Any, Any]:
     #
     # Query  / Expression Management
     #
-    def to_sql(self, pretty: bool = False) -> str:
+    def to_sql(self, pretty: bool = False, _raw_query: bool = False) -> str:
         """Returns an executable sql query string in the correct sql dialect for this relation"""
 
-        if self.__execute_raw_query:
+        if self.__execute_raw_query or _raw_query:
             query = self._sqlglot_expression
         else:
             query = self._normalized_query