Refactor transformations

.github/workflows/test_common.yml

@@ -121,7 +121,7 @@ jobs:
 
       - name: Run extract and pipeline tests
         run: |
-          pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources tests/transformations ${{ matrix.pytest_args }}
+          pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources ${{ matrix.pytest_args }}
         if: matrix.python-version != '3.14.0-beta.4'
 
       # here we upgrade sql alchemy to 2 an run the sql_database tests again

Makefile

@@ -69,10 +69,11 @@ lint-snippets:
 	cd docs/tools && uv run python check_embedded_snippets.py full
 
 lint-and-test-snippets: lint-snippets
+	# TODO: re-enable transformation snippets tests
 	uv pip install docstring_parser_fork --reinstall
-	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/
-	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo
-	cd docs/website/docs && uv run pytest --ignore=node_modules
+	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/ --exclude docs/website/docs/general-usage/transformations/transformation-snippets.py
+	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo --exclude docs/website/docs/general-usage/transformations/transformation-snippets.py
+	cd docs/website/docs && uv run pytest --ignore=node_modules --ignore general-usage/transformations/transformation-snippets.py
 
 lint-and-test-examples:
 	uv pip install docstring_parser_fork --reinstall
@@ -112,7 +113,7 @@ test-load-local-postgres:
 	DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data ACTIVE_DESTINATIONS='["postgres"]' ALL_FILESYSTEM_DRIVERS='["memory"]'  uv run pytest tests/load
 
 test-common:
-	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations tests/transformations
+	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations
 
 reset-test-storage:
 	-rm -r _storage

dlt/__init__.py

@@ -29,7 +29,6 @@
 from dlt import sources
 from dlt.extract.decorators import source, resource, transformer, defer
 from dlt.destinations.decorators import destination
-from dlt.transformations.decorators import transformation
 from dlt.common.destination.dataset import Dataset, Relation
 from dlt.destinations.dataset import dataset
 
@@ -84,7 +83,6 @@
     "Dataset",
     "Relation",
     "dataset",
-    "transformation",
 ]
 
 # verify that no injection context was created

dlt/destinations/dataset/dataset.py

@@ -25,7 +25,7 @@
 from dlt.destinations.dataset.utils import get_destination_clients
 from dlt.destinations.queries import build_row_counts_expr
 
-from dlt.transformations import lineage
+from dlt.destinations.dataset import lineage
 
 
 if TYPE_CHECKING:

dlt/destinations/dataset/exceptions.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from dlt.common.exceptions import DltException
 
 
@@ -20,3 +21,9 @@ def __init__(self, column_name: str) -> None:
             f"The selected column `{column_name}` is not known in the dlt schema for this relation."
         )
         super().__init__(msg)
+
+
+class LineageFailedException(DltException):
+    def __init__(self, msg: Optional[str] = None, *, resource_name: Optional[str] = None):
+        super().__init__(msg)
+        self.resource_name = resource_name

dlt/transformations/lineage.py → dlt/destinations/dataset/lineage.py

@@ -22,7 +22,7 @@
     TTableSchemaColumns,
     TColumnSchema,
 )
-from dlt.transformations.exceptions import LineageFailedException
+from dlt.destinations.dataset.exceptions import LineageFailedException
 
 
 logger = logging.getLogger(__file__)

dlt/destinations/dataset/relation.py

@@ -30,7 +30,7 @@
 from dlt.common.schema.typing import TTableSchemaColumns, TTableSchema
 from dlt.common.typing import Self, TSortOrder
 from dlt.common.exceptions import ValueErrorWithKnownValues
-from dlt.transformations import lineage
+from dlt.destinations.dataset import lineage
 from dlt.destinations.sql_client import SqlClientBase, WithSqlClient
 from dlt.destinations.queries import normalize_query, build_select_expr
 from dlt.common.exceptions import MissingDependencyException