Merge pull request #3267 from dlt-hub/devel

master merge for 1.18.1 release

Author: rudolfix

.github/workflows/main.yml

@@ -66,10 +66,10 @@ jobs:
     needs: test_common
     uses: ./.github/workflows/test_sources_local.yml
 
-  # test_hub:
-  #   name: test dlthub features
-  #   needs: lint
-  #   uses: ./.github/workflows/test_hub.yml
+  test_hub:
+    name: test dlthub features
+    needs: lint
+    uses: ./.github/workflows/test_hub.yml
 
   test_tools_airflow:
     name: test airflow helpers

.github/workflows/test_docs_snippets.yml

@@ -7,11 +7,8 @@ on:
 
 env:
 
-  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
 
-  # Path to local qdrant database
-  DESTINATION__QDRANT__CREDENTIALS__PATH: zendesk.qdb
   # detect if the workflow is executed in a repo fork
   IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
 
@@ -48,8 +45,8 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
-      - name: Start weaviate
-        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
+      # - name: Start weaviate
+      #   run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup node 20
         uses: actions/setup-node@v4
@@ -69,10 +66,10 @@ jobs:
           enable-cache: true
 
       - name: Install dependencies
-        run: uv sync --extra duckdb --extra weaviate --extra parquet --extra qdrant --extra bigquery --extra postgres --extra lancedb  --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
+        run: uv sync --extra duckdb --extra parquet --extra postgres --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
 
-      # - name: Install dlthub incl alpha releases
-      #   run: uv run pip install --pre dlthub
+      - name: Install dlthub incl alpha releases
+        run: uv run pip install --pre dlthub
 
       - name: run docs preprocessor
         run: make preprocess-docs

.github/workflows/test_hub.yml

@@ -18,18 +18,22 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         dlthub_dep: ["dlthub", "https://dlt-packages.fra1.digitaloceanspaces.com/dlthub/dlthub-0.0.0+nightly-py3-none-any.whl"]
         # Test all python versions on ubuntu only
         exclude:
         - os: "macos-latest"
-          python-version: "3.11"
+          python-version: "3.10"
         - os: "macos-latest"
           python-version: "3.12"
+        - os: "macos-latest"
+          python-version: "3.13"
         - os: "windows-latest"
-          python-version: "3.11"
+          python-version: "3.10"
         - os: "windows-latest"
           python-version: "3.12"
+        - os: "windows-latest"
+          python-version: "3.13"
 
     defaults:
       run:
@@ -58,19 +62,30 @@ jobs:
           sudo ACCEPT_EULA=Y apt-get install --yes msodbcsql18
         if: matrix.os == 'ubuntu-latest'
 
+      # still does not work in mac
+      # - name: Install ODBC driver for SQL Server (macOS)
+      #   if: matrix.os == 'macos-latest'
+      #   env:
+      #     HOMEBREW_NONINTERACTIVE: 1
+      #     HOMEBREW_NO_AUTO_UPDATE: 1
+      #     HOMEBREW_NO_ANALYTICS: 1
+      #     HOMEBREW_NO_INSTALL_CLEANUP: 1
+      #   run: |
+      #     brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release
+      #     printf "YES\n" | brew install msodbcsql18
+      #     # quick diagnostics (do not fail the build if unavailable)
+      #     odbcinst -j || true
+      #     odbcinst -q -d || true
+
       - name: Install all dependencies
         run: make dev
 
-      - name: Install dlthub nightly devel build without cache
+      - name: Install dlthub
         run: uv run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.dlthub_dep }}
 
       - name: Run tests
         run: pytest tests/hub
-        if: matrix.os == 'ubuntu-latest'
-
-      - name: Run tests on mac on win without mssql driver
-        run: pytest tests/hub -m "not mssql"
-        if: matrix.os == 'macos-latest' || matrix.os == 'windows-latest'
+        # if: matrix.os != 'macos-latest'
 
   matrix_job_required_check:
     name: hub | dlthub features tests

Makefile

@@ -75,7 +75,7 @@ lint-snippets:
 	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo,docs/website/node_modules
 
 lint-and-test-snippets: lint-snippets
-	cd docs/website/docs && uv run pytest --ignore=node_modules --ignore hub/features/transformations/transformation-snippets.py
+	cd docs/website/docs && uv run pytest --ignore=node_modules
 
 lint-and-test-examples:
 	uv pip install docstring_parser_fork --reinstall
@@ -134,18 +134,17 @@ clean-dist:
 
 publish-library: clean-dist build-library
 	ls -l dist/
-	@read -sp "Enter PyPI API token: " PYPI_API_TOKEN; echo ; \
-	uv publish --token "$$PYPI_API_TOKEN"
+	@bash -c 'read -s -p "Enter PyPI API token: " PYPI_API_TOKEN; echo; \
+	uv publish --token "$$PYPI_API_TOKEN"'
 
 test-build-images: build-library
 	# NOTE: uv export does not work with our many different deps, we install a subset and freeze
-	uv sync --extra gcp --extra redshift --extra duckdb
-	uv pip freeze > _gen_requirements.txt
+	# uv sync --extra gcp --extra redshift --extra duckdb
+	# uv pip freeze > _gen_requirements.txt
 	# filter out libs that need native compilation
-	grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
+	# grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
 	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell uv version --short)" .
-    # enable when we upgrade arrow to 20.x
-    # docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell uv version)" .
+	docker build -f deploy/dlt/Dockerfile.minimal --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell uv version --short)" .
 
 preprocess-docs:
 	# run docs preprocessing to run a few checks and ensure examples can be parsed

deploy/dlt/Dockerfile.minimal

@@ -0,0 +1,56 @@
+FROM python:3.11.11-alpine
+
+# Metadata
+LABEL org.label-schema.vendor="dltHub" \
+    org.label-schema.url="https://dlthub.com" \
+    org.label-schema.name="dlt" \
+    org.label-schema.description="**data load tool (dlt)** is a simple, open source Python library that makes data loading easy."
+
+# prepare dirs to install dlt
+RUN mkdir -p /tmp/pydlt
+
+WORKDIR /tmp/pydlt
+
+# install alpine deps
+RUN apk update &&\
+    apk add --no-cache ca-certificates curl &&\
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py &&\
+    rm get-pip.py &&\
+    pip install --upgrade setuptools wheel pip
+
+
+# add build labels and envs
+ARG COMMIT_SHA=""
+ARG IMAGE_VERSION=""
+LABEL commit_sha=${COMMIT_SHA}
+LABEL version=${IMAGE_VERSION}
+ENV COMMIT_SHA=${COMMIT_SHA}
+ENV IMAGE_VERSION=${IMAGE_VERSION}
+
+# install exactly the same version of the library we used to build
+COPY dist/dlt-${IMAGE_VERSION}.tar.gz .
+RUN pip install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz
+
+# create app dir to run simple test
+RUN mkdir -p /app
+WORKDIR /app
+RUN rm -r /tmp/pydlt
+
+# make sure dlt can be actually imported
+RUN python -c 'import dlt;import pendulum;'
+
+# check excluded imports
+COPY deploy/dlt/restrict_imports.py .
+RUN python restrict_imports.py
+
+# run simple pipeline
+COPY deploy/dlt/minimal_pipeline.py .
+RUN python minimal_pipeline.py
+RUN dlt pipeline fruit_pipeline info
+
+# enable workspace
+RUN mkdir -p .dlt && touch .dlt/.workspace
+# RUN dlt pipeline fruit_pipeline info
+RUN dlt workspace info
+RUN python minimal_pipeline.py
+RUN dlt pipeline fruit_pipeline info

deploy/dlt/minimal_pipeline.py

@@ -0,0 +1,7 @@
+import dlt
+from dlt._workspace._templates._single_file_templates.fruitshop_pipeline import (
+    fruitshop as fruitshop_source,
+)
+
+pf = dlt.pipeline(pipeline_name="fruit_pipeline", destination=dlt.destinations.filesystem("_data"))
+print(pf.run(fruitshop_source()))

deploy/dlt/restrict_imports.py

@@ -0,0 +1,16 @@
+print("this module checks if certain modules are not imported by default")
+import sys, dlt
+
+
+# plugin system should not be initialized on dlt import, workspace should not be imported
+excluded_imports = ["dlt._workspace._workspace_context", "pluggy", "git"]
+
+for excluded  in excluded_imports:
+    assert excluded not in sys.modules
+
+
+# will instantiate pluggy
+dlt.current.run_context()
+
+assert "pluggy" in sys.modules
+assert "dlt._workspace._workspace_context" not in sys.modules

dlt/_workspace/_plugins.py

@@ -1,21 +1,18 @@
 import os
 from typing import Any, Dict, Optional
 
-from dlt._workspace.exceptions import WorkspaceRunContextNotAvailable
-from dlt._workspace.profile import DEFAULT_PROFILE, read_profile_pin
 from dlt.common.configuration import plugins as _plugins
 from dlt.common.configuration.specs.pluggable_run_context import RunContextBase
-from dlt.common.runtime.run_context import RunContext
-
-from dlt._workspace.run_context import default_name
-from dlt._workspace._workspace_context import (
-    WorkspaceRunContext,
-    is_workspace_dir,
-)
+from dlt.common.runtime.run_context import DOT_DLT, RunContext
 
 __all__ = ["plug_workspace_context_impl"]
 
 
+def is_workspace_dir(run_dir: str) -> bool:
+    """Checks if `run_dir` contains dlt workspace, this is true if a config file is found"""
+    return os.path.isfile(os.path.join(run_dir, DOT_DLT, ".workspace"))
+
+
 @_plugins.hookimpl(specname="plug_run_context", trylast=False)
 def plug_workspace_context_impl(
     run_dir: Optional[str], runtime_kwargs: Optional[Dict[str, Any]]
@@ -24,12 +21,19 @@ def plug_workspace_context_impl(
     # if runtime_kwargs.get("_look_recursive")
     run_dir = os.path.abspath(run_dir or ".")
     if is_workspace_dir(run_dir):
+        # import workspace only when context detected
+        from dlt._workspace.profile import DEFAULT_PROFILE, read_profile_pin
+        from dlt._workspace.run_context import default_name
+        from dlt._workspace._workspace_context import WorkspaceRunContext
+
         profile: str = None
         if runtime_kwargs:
             profile = runtime_kwargs.get("profile")
         profile = profile or read_profile_pin(RunContext(run_dir)) or DEFAULT_PROFILE
         return WorkspaceRunContext(default_name(run_dir), run_dir, profile)
     elif runtime_kwargs and runtime_kwargs.get("_required") == "WorkspaceRunContext":
+        from dlt._workspace.exceptions import WorkspaceRunContextNotAvailable
+
         raise WorkspaceRunContextNotAvailable(run_dir)
 
     return None

dlt/_workspace/_workspace_context.py

@@ -187,22 +187,6 @@ def switch_profile(self, new_profile: str) -> "WorkspaceRunContext":
 switch_profile = copy_sig_ret(_switch_profile, WorkspaceRunContext)(_switch_profile)
 
 
-def is_workspace_dir(run_dir: str) -> bool:
-    """Checks if `project_dir` contains dlt project, this is true if a config file is found"""
-    if os.path.isfile(os.path.join(run_dir, DOT_DLT, ".workspace")):
-        return True
-    return False
-
-
-def is_workspace_active() -> bool:
-    try:
-        active()
-    except WorkspaceRunContextNotAvailable:
-        return False
-    else:
-        return True
-
-
 def active() -> WorkspaceRunContext:
     """Returns currently active Workspace"""
     ctx = Container()[PluggableRunContext].context

dlt/_workspace/cli/_plugins.py

@@ -10,8 +10,8 @@
 from typing import Type
 
 from dlt.common.configuration import plugins
+from dlt.common.runtime.run_context import active as run_context_active
 
-from dlt._workspace._workspace_context import is_workspace_active
 
 __all__ = [
     "plug_cli_init",
@@ -27,6 +27,13 @@
 ]
 
 
+def is_workspace_active() -> bool:
+    # verify run context type without importing
+
+    ctx = run_context_active()
+    return ctx.__class__.__name__ == "WorkspaceRunContext"
+
+
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_init() -> Type[plugins.SupportsCliCommand]:
     from dlt._workspace.cli.commands import InitCommand