feat/3198-add-workspace-info-and-profile-selection

.github/workflows/test_tools_dashboard.yml

@@ -107,14 +107,12 @@ jobs:
       # Run workspace dashboard e2e tests (does not pass with python 3.9
       - name: Run dashboard e2e
         run: |
-          marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
+          pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
 
       # note that this test will pass only when running from cmd shell (_storage\.dlt\pipelines\ must stay)
       - name: Run dashboard e2e windows
         run: |
-          start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
-          timeout /t 6 /nobreak
           pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
 

Makefile

@@ -173,9 +173,6 @@ test-e2e-dashboard:
 test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
-start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
-
 # creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
 create-test-pipelines:
 	uv run python tests/workspace/helpers/dashboard/example_pipelines.py

dlt/_workspace/helpers/dashboard/dlt_dashboard.py

@@ -17,10 +17,13 @@
     import pyarrow
     from dlt._workspace.helpers.dashboard import strings, utils, ui_elements as ui
     from dlt._workspace.helpers.dashboard.config import DashboardConfiguration
+    from dlt.common.configuration.specs.pluggable_run_context import ProfilesRunContext
+    from dlt._workspace.run_context import switch_profile
 
 
 @app.cell(hide_code=True)
 def home(
+    dlt_profile_select: mo.ui.dropdown,
     dlt_all_pipelines: List[Dict[str, Any]],
     dlt_pipeline_select: mo.ui.multiselect,
     dlt_pipelines_dir: str,
@@ -41,16 +44,44 @@ def home(
         dlt_pipeline = utils.get_pipeline(dlt_pipeline_name, dlt_pipelines_dir)
 
     dlt_config = utils.resolve_dashboard_config(dlt_pipeline)
-
+    _header_controls = (
+        [
+            dlt_profile_select,
+            mo.md(f"<small> Workspace: {getattr(dlt.current.run_context(), 'name', None)}</small>"),
+        ]
+        if isinstance(dlt.current.run_context(), ProfilesRunContext)
+        else None
+    )
     if not dlt_pipeline and not dlt_pipeline_name:
         _stack = [
             mo.hstack(
                 [
-                    mo.image(
-                        "https://dlthub.com/docs/img/dlthub-logo.png", width=100, alt="dltHub logo"
+                    mo.hstack(
+                        [
+                            mo.image(
+                                "https://dlthub.com/docs/img/dlthub-logo.png",
+                                width=100,
+                                alt="dltHub logo",
+                            ),
+                            _header_controls[0] if _header_controls else "",
+                        ],
+                        justify="start",
+                        gap=2,
+                    ),
+                    mo.hstack(
+                        [
+                            _header_controls[1] if _header_controls else "",
+                        ],
+                        justify="center",
+                    ),
+                    mo.hstack(
+                        [
+                            dlt_pipeline_select,
+                        ],
+                        justify="end",
                     ),
-                    dlt_pipeline_select,
                 ],
+                justify="space-between",
             ),
             mo.md(strings.app_title).center(),
             mo.md(strings.app_intro).center(),
@@ -91,14 +122,65 @@ def home(
                 [
                     mo.hstack(
                         [
-                            mo.image(
-                                "https://dlthub.com/docs/img/dlthub-logo.png",
-                                width=100,
-                                alt="dltHub logo",
-                            ).style(padding_bottom="1em"),
-                            mo.center(mo.md(strings.app_title_pipeline.format(dlt_pipeline_name))),
-                            dlt_pipeline_select,
+                            mo.vstack(
+                                [
+                                    mo.hstack(
+                                        [
+                                            mo.hstack(
+                                                [
+                                                    mo.hstack(
+                                                        [
+                                                            mo.image(
+                                                                "https://dlthub.com/docs/img/dlthub-logo.png",
+                                                                width=100,
+                                                                alt="dltHub logo",
+                                                            ),
+                                                            (
+                                                                _header_controls[0]
+                                                                if _header_controls
+                                                                else ""
+                                                            ),
+                                                        ],
+                                                        justify="start",
+                                                        gap=2,
+                                                    ),
+                                                    mo.hstack(
+                                                        [
+                                                            (
+                                                                _header_controls[1]
+                                                                if _header_controls
+                                                                else ""
+                                                            ),
+                                                        ],
+                                                        justify="center",
+                                                    ),
+                                                    mo.hstack(
+                                                        [
+                                                            dlt_pipeline_select,
+                                                        ],
+                                                        justify="end",
+                                                    ),
+                                                ],
+                                                justify="center",
+                                            ),
+                                        ],
+                                    ),
+                                    mo.center(
+                                        mo.hstack(
+                                            [
+                                                mo.md(
+                                                    strings.app_title_pipeline.format(
+                                                        dlt_pipeline_name
+                                                    )
+                                                ),
+                                            ],
+                                            align="center",
+                                        ),
+                                    ),
+                                ]
+                            ),
                         ],
+                        justify="space-between",
                     ),
                     mo.hstack(_buttons, justify="start"),
                 ]
@@ -785,6 +867,7 @@ def section_ibis_backend(
 
 @app.cell(hide_code=True)
 def utils_discover_pipelines(
+    dlt_profile_select: mo.ui.dropdown,
     mo_cli_arg_pipelines_dir: str,
     mo_cli_arg_pipeline: str,
     mo_query_var_pipeline_name: str,
@@ -793,6 +876,13 @@ def utils_discover_pipelines(
     Discovers local pipelines and returns a multiselect widget to select one of the pipelines
     """
 
+    _run_context = dlt.current.run_context()
+    if (
+        isinstance(_run_context, ProfilesRunContext)
+        and not _run_context.profile == dlt_profile_select.value
+    ):
+        switch_profile(dlt_profile_select.value)
+
     # discover pipelines and build selector
     dlt_pipelines_dir: str = ""
     dlt_all_pipelines: List[Dict[str, Any]] = []
@@ -816,6 +906,39 @@ def utils_discover_pipelines(
     return dlt_all_pipelines, dlt_pipeline_select, dlt_pipelines_dir
 
 
+@app.cell(hide_code=True)
+def utils_discover_profiles(mo_query_var_profile: str, mo_cli_arg_profile: str):
+    """Discover profiles and return a single-select multiselect, similar to pipelines."""
+    run_context = dlt.current.run_context()
+
+    # Default (non-profile-aware) output
+    dlt_profile_select = mo.ui.dropdown(options=[], value=None, label="Profile: ")
+    selected_profile = None
+
+    if isinstance(run_context, ProfilesRunContext):
+        options = run_context.available_profiles() or []
+        current = run_context.profile if options and run_context.profile in options else None
+
+        selected_profile = current
+        if mo_query_var_profile and mo_query_var_profile in options:
+            selected_profile = mo_query_var_profile
+        elif mo_cli_arg_profile and mo_cli_arg_profile in options:
+            selected_profile = mo_cli_arg_profile
+
+        def _on_profile_change(v: str) -> None:
+            mo.query_params().set("profile", v)
+
+        dlt_profile_select = mo.ui.dropdown(
+            options=options,
+            value=selected_profile,
+            label="Profile: ",
+            on_change=_on_profile_change,
+            searchable=True,
+        )
+
+    return dlt_profile_select, selected_profile
+
+
 @app.cell(hide_code=True)
 def utils_discover_schemas(dlt_pipeline: dlt.Pipeline):
     """
@@ -1040,25 +1163,40 @@ def utils_cli_args_and_query_vars_config():
     """
     Prepare cli args  as globals for the following cells
     """
-
+    _run_context = dlt.current.run_context()
+    mo_query_var_pipeline_name: str = None
+    mo_cli_arg_pipelines_dir: str = None
+    mo_cli_arg_with_test_identifiers: bool = False
+    mo_cli_arg_pipeline: str = None
+    mo_query_var_profile: str = None
+    mo_cli_arg_profile: str = None
     try:
-        mo_query_var_pipeline_name: str = cast(str, mo.query_params().get("pipeline")) or None
-        mo_cli_arg_pipeline: str = cast(str, mo.cli_args().get("pipeline")) or None
-        mo_cli_arg_pipelines_dir: str = cast(str, mo.cli_args().get("pipelines-dir")) or None
-        mo_cli_arg_with_test_identifiers: bool = (
+        mo_query_var_pipeline_name = cast(str, mo.query_params().get("pipeline")) or None
+        mo_cli_arg_pipeline = cast(str, mo.cli_args().get("pipeline")) or None
+        mo_cli_arg_pipelines_dir = cast(str, mo.cli_args().get("pipelines-dir")) or None
+        mo_cli_arg_with_test_identifiers = (
             cast(bool, mo.cli_args().get("with_test_identifiers")) or False
         )
+        mo_query_var_profile = (
+            cast(str, mo.query_params().get("profile")) or None
+            if isinstance(_run_context, ProfilesRunContext)
+            else None
+        )
+        mo_cli_arg_profile = (
+            cast(str, mo.cli_args().get("profile")) or None
+            if isinstance(_run_context, ProfilesRunContext)
+            else None
+        )
     except Exception:
-        mo_query_var_pipeline_name = None
-        mo_cli_arg_pipelines_dir = None
-        mo_cli_arg_with_test_identifiers = False
-        mo_cli_arg_pipeline = None
+        pass
 
     return (
         mo_cli_arg_pipelines_dir,
         mo_cli_arg_with_test_identifiers,
         mo_query_var_pipeline_name,
         mo_cli_arg_pipeline,
+        mo_query_var_profile,
+        mo_cli_arg_profile,
     )
 
 

dlt/_workspace/helpers/dashboard/runner.py

@@ -1,9 +1,13 @@
+import contextlib
 import os
 import sys
 import subprocess
 from importlib.resources import files
-from typing import Any
+import time
+from typing import Any, Iterator, List
 from pathlib import Path
+import urllib
+
 from dlt.common.exceptions import MissingDependencyException
 
 
@@ -39,7 +43,71 @@ def run_dashboard(
     pipelines_dir: str = None,
     port: int = None,
     host: str = None,
+    with_test_identifiers: bool = False,
+    headless: bool = False,
 ) -> None:
+    """Run dashboard blocked"""
+    try:
+        subprocess.run(
+            run_dashboard_command(
+                pipeline_name, edit, pipelines_dir, port, host, with_test_identifiers, headless
+            )
+        )
+    except KeyboardInterrupt:
+        pass
+
+
+def _wait_http_up(url: str, timeout_s: float = 15.0) -> None:
+    start = time.time()
+    while time.time() - start < timeout_s:
+        try:
+            with urllib.request.urlopen(url, timeout=1.0):
+                return
+        except Exception:
+            time.sleep(0.1)
+    raise TimeoutError(f"Server did not become ready: {url}")
+
+
+@contextlib.contextmanager
+def start_dashboard(
+    pipelines_dir: str = None,
+    port: int = 2718,
+    test_identifiers: bool = True,
+    headless: bool = True,
+) -> Iterator[subprocess.Popen[bytes]]:
+    """Launches dashboard in context manager that will kill it after use"""
+    command = run_dashboard_command(
+        pipeline_name=None,
+        edit=False,
+        pipelines_dir=pipelines_dir,
+        port=port,
+        with_test_identifiers=test_identifiers,
+        headless=headless,
+    )
+    # start the dashboard process using subprocess.Popen
+    proc = subprocess.Popen(command)
+    try:
+        _wait_http_up(f"http://localhost:{port}", timeout_s=60.0)
+        yield proc
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+
+
+def run_dashboard_command(
+    pipeline_name: str = None,
+    edit: bool = False,
+    pipelines_dir: str = None,
+    port: int = None,
+    host: str = None,
+    with_test_identifiers: bool = False,
+    headless: bool = False,
+) -> List[str]:
+    """Creates cli command to run workspace dashboard"""
     from dlt._workspace.helpers.dashboard import dlt_dashboard
 
     ejected_app_path = os.path.join(os.getcwd(), EJECTED_APP_FILE_NAME)
@@ -77,6 +145,9 @@ def run_dashboard(
         dashboard_cmd.append("--host")
         dashboard_cmd.append(host)
 
+    if headless:
+        dashboard_cmd.append("--headless")
+
     if pipeline_name:
         dashboard_cmd.append("--")
         dashboard_cmd.append("--pipeline")
@@ -85,8 +156,9 @@ def run_dashboard(
         dashboard_cmd.append("--")
         dashboard_cmd.append("--pipelines-dir")
         dashboard_cmd.append(pipelines_dir)
+    if with_test_identifiers:
+        dashboard_cmd.append("--")
+        dashboard_cmd.append("--with_test_identifiers")
+        dashboard_cmd.append("true")
 
-    try:
-        subprocess.run(dashboard_cmd)
-    except KeyboardInterrupt:
-        pass
+    return dashboard_cmd

dlt/_workspace/helpers/dashboard/ui_elements.py

@@ -1,4 +1,5 @@
 from typing import Any
+from dlt.common.configuration.specs.pluggable_run_context import ProfilesRunContext
 
 import dlt
 

docs/education/dlt-fundamentals-course/lesson_3_pagination_and_authentication_and_dlt_configuration.ipynb

@@ -560,6 +560,7 @@
     "from dlt.sources.helpers.rest_client.auth import BearerTokenAuth\n",
     "from dlt.common.typing import TDataItems\n",
     "\n",
+    "\n",
     "@dlt.source\n",
     "def github_source(\n",
     "    access_token=dlt.secrets.value,\n",