Merge branch 'devel' into feat/explains-partition-and-split-loading

Author: rudolfix

.github/workflows/lint.yml

@@ -38,6 +38,9 @@ jobs:
           activate-environment: true
           enable-cache: true
 
+      - name: Check lockfile
+        run: uv lock --check
+
       - name: Install dependencies
         run: uv sync --all-extras --group airflow --group providers --group pipeline --group sources --group sentry-sdk --group dbt --group streamlit
 

.github/workflows/test_docs_snippets.yml

@@ -87,4 +87,5 @@ jobs:
       - name: Test notebooks
         run: cd docs/notebooks && make test
 
-
+      - name: Lint education notebooks
+        run: make lint-notebooks

CONTRIBUTING.md

@@ -49,7 +49,8 @@ When you're ready to contribute, follow these steps:
 3. Write your code and tests.
 4. Lint your code by running `make lint` and test common modules with `make test-common`.
 5. If you're working on destination code, contact us to get access to test destinations.
-6. Create a pull request targeting the **devel** branch of the main repository.
+6. If you have added, removed or updated dependencies in the `pyproject.toml`, make sure `uv.lock` is up to date by running `uv lock`. If you are merging in upstream changes from the devel branch and get a conflict on the lockfile, it's best to take the devel version and re-run `uv lock` to re-apply your changes.
+7. Create a pull request targeting the **devel** branch of the main repository.
 
 **Note:** for some special cases, you'd need to contact us to create a branch in this repository (not fork). See below.
 

Makefile

@@ -64,6 +64,7 @@ lint:
 
 format:
 	uv run black dlt docs tests --extend-exclude='.*syntax_error.py|_storage/.*'
+	uv run black docs/education --ipynb --extend-exclude='.*syntax_error.py|_storage/.*'
 
 lint-snippets:
 	cd docs/tools && uv run python check_embedded_snippets.py full
@@ -91,6 +92,13 @@ lint-security:
 	# go for ll by cleaning up eval and SQL warnings.
 	uv run bandit -r dlt/ -n 3 -lll
 
+lint-notebooks:
+	uv run nbqa flake8 docs/education --extend-ignore=D,F704 --max-line-length=200
+	uv run nbqa mypy docs/education \
+	--ignore-missing-imports \
+	--disable-error-code=no-redef \
+	--disable-error-code=top-level-await
+
 # check docstrings for all important public classes and functions
 lint-docstrings:
 	uv run flake8 --count \
@@ -162,11 +170,18 @@ update-cli-docs:
 check-cli-docs:
 	uv run dlt --debug render-docs docs/website/docs/reference/command-line-interface.md --compare
 
+# Commands for running dashboard e2e tests
+# To run these tests locally, run `make start-dlt-dashboard-e2e` in one terminal and `make test-e2e-dashboard-headed` in another terminal
+
 test-e2e-dashboard:
 	uv run pytest --browser chromium tests/e2e
 
 test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
 start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
+	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
+
+# creates the dashboard test pipelines globally for manual testingn of the dashboard app and cli
+create-test-pipelines:
+	uv run python tests/helpers/dashboard/example_pipelines.py

dlt/__init__.py

@@ -29,7 +29,6 @@
 from dlt import sources
 from dlt.extract.decorators import source, resource, transformer, defer
 from dlt.destinations.decorators import destination
-from dlt.common.destination.dataset import SupportsDataset, SupportsRelation
 from dlt.dataset import dataset, Relation, Dataset
 
 from dlt.pipeline import (
@@ -80,8 +79,6 @@
     "TCredentials",
     "sources",
     "destinations",
-    "SupportsDataset",
-    "SupportsRelation",
     "dataset",
     "Relation",
     "Dataset",

dlt/cli/pipeline_command.py

@@ -37,7 +37,6 @@ def pipeline_command(
     verbosity: int,
     dataset_name: str = None,
     destination: TDestinationReferenceArg = None,
-    hot_reload: Optional[bool] = False,
     **command_kwargs: Any,
 ) -> None:
     if operation == "list":
@@ -54,12 +53,15 @@ def pipeline_command(
 
     # we may open the dashboard for a pipeline without checking if it exists
     if operation == "show" and not command_kwargs.get("streamlit"):
+        # TODO: why this is imported here
         from dlt.common.utils import custom_environ
         from dlt.common.known_env import DLT_DATA_DIR
 
         from dlt.helpers.dashboard.runner import run_dashboard
 
         run_dashboard(pipeline_name, edit=command_kwargs.get("edit"), pipelines_dir=pipelines_dir)
+        # return so streamlit does not run
+        return
 
     try:
         if verbosity > 0:
@@ -127,21 +129,16 @@ def _display_pending_packages() -> Tuple[Sequence[str], Sequence[str]]:
 
     if operation == "show":
         from dlt.common.runtime import signals
-        from dlt.helpers.streamlit_app import index
 
         with signals.delayed_signals():
             streamlit_cmd = [
                 "streamlit",
                 "run",
-                index.__file__,
+                os.path.join(os.path.dirname(dlt.__file__), "helpers", "streamlit_app", "index.py"),
                 "--client.showSidebarNavigation",
                 "false",
             ]
 
-            if hot_reload:
-                streamlit_cmd.append("--server.runOnSave")
-                streamlit_cmd.append("true")
-
             streamlit_cmd.append("--")
             streamlit_cmd.append(pipeline_name)
             streamlit_cmd.append("--pipelines-dir")

dlt/cli/plugins.py

@@ -162,12 +162,6 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None:
             action="store_true",
             help="List local pipelines",
         )
-        pipe_cmd.add_argument(
-            "--hot-reload",
-            default=False,
-            action="store_true",
-            help="Reload streamlit app (for core development)",
-        )
         pipe_cmd.add_argument("pipeline_name", nargs="?", help="Pipeline name")
         pipe_cmd.add_argument("--pipelines-dir", help="Pipelines working directory", default=None)
         pipe_cmd.add_argument(

dlt/common/data_types/type_helpers.py

@@ -18,7 +18,7 @@
     ensure_pendulum_date,
     ensure_pendulum_time,
 )
-from dlt.common.utils import map_nested_in_place, str2bool
+from dlt.common.utils import map_nested_values_in_place, str2bool
 
 
 def _py_type_to_sc_type(t: Type[Any]) -> TDataType:
@@ -75,7 +75,7 @@ def _py_type_to_sc_type(t: Type[Any]) -> TDataType:
 
 
 def json_to_str(value: Any) -> str:
-    return json.dumps(map_nested_in_place(custom_pua_remove, value))
+    return json.dumps(map_nested_values_in_place(custom_pua_remove, value))
 
 
 def coerce_from_date_types(
@@ -101,7 +101,7 @@ def coerce_value(to_type: TDataType, from_type: TDataType, value: Any) -> Any:
     if to_type == from_type:
         if to_type == "json":
             # nested types need custom encoding to be removed
-            return map_nested_in_place(custom_pua_remove, value)
+            return map_nested_values_in_place(custom_pua_remove, value)
         # Make sure we use enum value instead of the object itself
         # This check is faster than `isinstance(value, Enum)` for non-enum types
         if hasattr(value, "value"):

dlt/common/data_writers/writers.py

@@ -188,7 +188,7 @@ def write_header(self, columns_schema: TTableSchemaColumns) -> None:
     def write_data(self, items: Sequence[TDataItem]) -> None:
         super().write_data(items)
         for item in items:
-            dialect = item.query_dialect()
+            dialect = item.query_dialect
             query = item.to_sql()
             self._f.write("dialect: " + (dialect or "") + "\n" + query + "\n")
 

dlt/common/destination/capabilities.py

@@ -129,6 +129,9 @@ def ensure_supported_type(
         )
 
 
+TCasefoldIdentifier = Callable[[str], str]
+
+
 @configspec
 class DestinationCapabilitiesContext(ContainerInjectableContext):
     """Injectable destination capabilities required for many Pipeline stages ie. normalize"""
@@ -152,7 +155,7 @@ class DestinationCapabilitiesContext(ContainerInjectableContext):
     "Escapes table name, column name and other identifiers"
     escape_literal: Callable[[Any], Any] = None
     "Escapes string literal"
-    casefold_identifier: Callable[[str], str] = str
+    casefold_identifier: TCasefoldIdentifier = str
     """Casing function applied by destination to represent case insensitive identifiers."""
     has_case_sensitive_identifiers: bool = None
     """Tells if destination supports case sensitive identifiers"""