Merge branch 'devel' into feat/extend-table-reference

Author: rudolfix

.github/workflows/build_docs.yml

@@ -0,0 +1,36 @@
+name: docs | build docs
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build_docs:
+    name: docs | build docs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 9.13.2
+
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '22'
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install node dependencies
+        run: cd docs/website && npm install
+
+      - name: Install python dependencies
+        run: cd docs/website && pip install -r requirements.txt
+
+      - name: Build docs
+        run: cd docs/website && npm run build:cloudflare

.github/workflows/main.yml

@@ -30,6 +30,11 @@ jobs:
   test_docs_snippets:
     name: test snippets in docs
     uses: ./.github/workflows/test_docs_snippets.yml
+
+  # NOTE: we build docs the same way as on cloudflare, so we can catch problems early
+  build_docs:
+    name: build docs
+    uses: ./.github/workflows/build_docs.yml
 
   lint:
     name: lint on all python versions
@@ -61,10 +66,10 @@ jobs:
     needs: test_common
     uses: ./.github/workflows/test_sources_local.yml
 
-  test_hub:
-    name: test dlthub features
-    needs: lint
-    uses: ./.github/workflows/test_hub.yml
+  # test_hub:
+  #   name: test dlthub features
+  #   needs: lint
+  #   uses: ./.github/workflows/test_hub.yml
 
   test_tools_airflow:
     name: test airflow helpers

.github/workflows/test_common.yml

@@ -63,11 +63,11 @@ jobs:
           - os: windows-latest
             python-version: "3.11"
             shell: cmd
-            pytest_args: '-m "not forked"'
+            pytest_args: '-m "not forked and not rfam"'
           - os: windows-latest
             python-version: "3.13"
             shell: cmd
-            pytest_args: '-m "not forked"'
+            pytest_args: '-m "not forked and not rfam"'
 
     defaults:
       run:
@@ -121,13 +121,22 @@ jobs:
           pytest tests/pipeline/test_pipeline_extra.py -k arrow ${{ matrix.pytest_args }}
         if: matrix.python-version != '3.14.0-beta.4'
 
+      - name: Install workspace dependencies
+        run: uv sync ${{ matrix.uv_sync_args }} --extra workspace --extra cli --group sentry-sdk
+        if: matrix.python-version != '3.14.0-beta.4'
+
+      - name: Run workspace tests
+        run: |
+          pytest tests/workspace ${{ matrix.pytest_args }}
+        if: matrix.python-version != '3.14.0-beta.4'
+
       - name: Install pipeline and sources dependencies
         run: uv sync ${{ matrix.uv_sync_args }} --extra duckdb --extra cli --extra parquet --extra deltalake --extra sql_database --group sentry-sdk --group pipeline --group sources --group ibis
         if: matrix.python-version != '3.14.0-beta.4'
 
       - name: Run extract and pipeline tests
         run: |
-          pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources ${{ matrix.pytest_args }}
+          pytest tests/extract tests/pipeline tests/libs tests/destinations tests/sources ${{ matrix.pytest_args }}
         if: matrix.python-version != '3.14.0-beta.4'
 
       # here we upgrade sql alchemy to 2 an run the sql_database tests again

.github/workflows/test_destinations_local.yml

@@ -33,13 +33,11 @@ jobs:
             post_install_commands: "uv run pip install sqlalchemy==2.0.18" # minimum version required by `pyiceberg`
 
           # TODO: also test ducklake in remote mode with a buckets and remote postgres
-          - name: postgres, duckdb, ducklake, and dummy with cli commands
+          - name: postgres, duckdb, ducklake, and dummy
             destinations: "[\"postgres\", \"duckdb\", \"ducklake\", \"dummy\"]"
             filesystem_drivers: "[\"memory\", \"file\"]"
             extras: "--group adbc --extra postgres --extra postgis --extra parquet --extra duckdb --extra cli --extra filesystem"
             needs_postgres: true
-            additional_tests: "pytest tests/cli"
-
 
           # Clickhouse OSS (TODO: test with minio s3)
           - name: clickhouse

.github/workflows/test_docs_snippets.yml

@@ -71,8 +71,8 @@ jobs:
       - name: Install dependencies
         run: uv sync --extra duckdb --extra weaviate --extra parquet --extra qdrant --extra bigquery --extra postgres --extra lancedb  --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
 
-      - name: Install dlt-plus incl alpha releases
-        run: uv run pip install --pre dlt-plus
+      # - name: Install dlthub incl alpha releases
+      #   run: uv run pip install --pre dlthub
 
       - name: run docs preprocessor
         run: make preprocess-docs

.github/workflows/test_examples.yml

@@ -47,8 +47,8 @@ jobs:
       - name: Install dependencies
         run: uv sync --extra duckdb --extra weaviate --extra parquet --extra qdrant --extra bigquery --extra postgres --extra lancedb  --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
 
-      - name: Install dlt-plus incl alpha releases
-        run: uv run pip install --pre dlt-plus
+      # - name: Install dlthub incl alpha releases
+      #   run: uv run pip install --pre dlthub
 
       - name: create secrets.toml for examples
         run: pwd && echo "$DLT_SECRETS_TOML" > docs/examples/.dlt/secrets.toml

.github/workflows/test_hub.yml

@@ -1,7 +1,7 @@
 name: hub | dlthub features
 
 #
-# dlt-plus smoke tests against the nightly build.
+# dlthub smoke tests against the nightly build.
 #
 
 on:
@@ -19,7 +19,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         python-version: ["3.10", "3.11", "3.12"]
-        plus_dep: ["dlt-plus", "https://dlt-packages.fra1.digitaloceanspaces.com/dlt-plus/dlt_plus-0.0.0+nightly-py3-none-any.whl"]
+        dlthub_dep: ["dlthub", "https://dlt-packages.fra1.digitaloceanspaces.com/dlthub/dlthub-0.0.0+nightly-py3-none-any.whl"]
         # Test all python versions on ubuntu only
         exclude:
         - os: "macos-latest"
@@ -61,8 +61,8 @@ jobs:
       - name: Install all dependencies
         run: make dev
 
-      - name: Install dlt-plus nightly devel build without cache
-        run: uv run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.plus_dep }}
+      - name: Install dlthub nightly devel build without cache
+        run: uv run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.dlthub_dep }}
 
       - name: Run tests
         run: pytest tests/hub

.github/workflows/test_tools_dashboard.yml

@@ -102,18 +102,18 @@ jobs:
       # Run pipeline dashboard unit tests
       - name: Run pipeline dashboard unit tests
         run: |
-          pytest tests/helpers/dashboard
+          pytest tests/workspace/helpers/dashboard
 
       # Run pipeline dashboard e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
       # Mac is also disabled for the time being
       - name: Run dashboard e2e
         run: |
-          marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
+          marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
 
       - name: Run dashboard e2e windows
         run: |
-          start marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
+          start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
           pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
 

.github/workflows/tools_deploy_docs.yml

@@ -48,8 +48,10 @@ dev: has-uv
 
 dev-airflow: has-uv
 	uv sync --all-extras --group docs --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group airflow
-	
-lint:
+
+lint: lint-core lint-security lint-docstrings
+
+lint-core:
 	uv run mypy --config-file mypy.ini dlt tests
 	# NOTE: we need to make sure docstring_parser_fork is the only version of docstring_parser installed
 	uv pip uninstall docstring_parser
@@ -59,22 +61,20 @@ lint:
 	uv run flake8 --extend-ignore=D --max-line-length=200 dlt
 	uv run flake8 --extend-ignore=D --max-line-length=200 tests --exclude tests/reflection/module_cases,tests/common/reflection/cases/modules/
 	uv run black dlt docs tests --check --diff --color --extend-exclude=".*syntax_error.py"
-	$(MAKE) lint-security
-	$(MAKE) lint-docstrings
 
 format:
 	uv run black dlt docs tests --extend-exclude='.*syntax_error.py|_storage/.*'
 	uv run black docs/education --ipynb --extend-exclude='.*syntax_error.py|_storage/.*'
 
 lint-snippets:
 	cd docs/tools && uv run python check_embedded_snippets.py full
-
-lint-and-test-snippets: lint-snippets
-	# TODO: re-enable transformation snippets tests
+	# TODO: re-enable transformation snippets tests when dlthub dep is available
 	uv pip install docstring_parser_fork --reinstall
-	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/ --exclude docs/website/docs/general-usage/transformations/transformation-snippets.py
+	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/
 	uv run ruff check
-	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo --exclude docs/website/docs/hub/features/transformations/transformation-snippets.py
+	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo
+
+lint-and-test-snippets: lint-snippets
 	cd docs/website/docs && uv run pytest --ignore=node_modules --ignore hub/features/transformations/transformation-snippets.py
 
 lint-and-test-examples:
@@ -123,7 +123,7 @@ test-load-local-postgres:
 	DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data ACTIVE_DESTINATIONS='["postgres"]' ALL_FILESYSTEM_DRIVERS='["memory"]'  uv run pytest tests/load
 
 test-common:
-	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations
+	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/workspace tests/load/test_dummy_client.py tests/libs tests/destinations
 
 reset-test-storage:
 	-rm -r _storage
@@ -180,8 +180,8 @@ test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
 start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
+	uv run marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
 
 # creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
 create-test-pipelines:
-	uv run python tests/helpers/dashboard/example_pipelines.py
+	uv run python tests/workspace/helpers/dashboard/example_pipelines.py