🔥 Added API Docs

.gitignore

@@ -159,3 +159,13 @@ dev/cleanup.py
 .python-version
 .databricks-login.json
 .local-dev
+
+# Cursor IDE specific files (not all developers use Cursor)
+.cursorrules
+.cursor/
+
+
+# docgen
+docs/dqx/docs/reference/api
+!docs/dqx/docs/reference/api/index.mdx
+

Makefile

@@ -31,14 +31,17 @@ e2e:
 coverage:
 	hatch run coverage; open htmlcov/index.html
 
-docs-build:
+docs-build: 
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx build
 
 docs-serve-dev:
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx start
 
 docs-install:
 	yarn --cwd docs/dqx install
 
 docs-serve: docs-build
+	hatch run docs:pydoc-markdown
 	yarn --cwd docs/dqx serve

docs/dqx/docs/reference/api/index.mdx

@@ -0,0 +1,8 @@
+---
+title: API Reference
+sidebar_position: 8
+---
+
+# API Reference
+
+This section contains the API reference for the Databricks Labs DQX project.

docs/dqx/docs/reference/engine.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 1
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/profiler.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 2
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/quality_rules.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 3
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/docs/reference/testing.mdx

@@ -1,3 +1,7 @@
+---
+sidebar_position: 4
+---
+
 import Admonition from '@theme/Admonition';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';

docs/dqx/package.json

@@ -15,8 +15,8 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "3.7.0",
-    "@docusaurus/preset-classic": "3.7.0",
+    "@docusaurus/core": "^3.8.1",
+    "@docusaurus/preset-classic": "^3.8.1",
     "@mdx-js/react": "^3.0.0",
     "@radix-ui/react-slot": "^1.1.1",
     "class-variance-authority": "^0.7.1",
@@ -28,12 +28,13 @@
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "tailwind-merge": "^2.6.0",
-    "tailwindcss-animate": "^1.0.7"
+    "tailwindcss-animate": "^1.0.7",
+    "tsx": "^4.20.3"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.7.0",
-    "@docusaurus/tsconfig": "3.7.0",
-    "@docusaurus/types": "3.7.0",
+    "@docusaurus/module-type-aliases": "^3.8.1",
+    "@docusaurus/tsconfig": "^3.8.1",
+    "@docusaurus/types": "^3.8.1",
     "@tailwindcss/typography": "^0.5.16",
     "autoprefixer": "^10.4.20",
     "postcss": "^8.5.1",

docs/dqx/sidebars.ts

@@ -15,7 +15,6 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs';
 const sidebars: SidebarsConfig = {
   // By default, Docusaurus generates a sidebar from the docs folder structure
   tutorialSidebar: [{type: 'autogenerated', dirName: '.'}],
-
 };
 
 export default sidebars;

pyproject.toml

@@ -27,9 +27,10 @@ classifiers = [
     "Topic :: Software Development :: Libraries",
     "Topic :: Utilities",
 ]
-dependencies = ["databricks-labs-blueprint>=0.9.1,<0.10",
-                "databricks-sdk~=0.57",
-                "databricks-labs-lsql>=0.5,<0.15",
+dependencies = [
+    "databricks-labs-blueprint>=0.9.1,<0.10",
+    "databricks-sdk~=0.57",
+    "databricks-labs-lsql>=0.5,<0.15",
 ]
 
 [project.optional-dependencies]
@@ -85,27 +86,40 @@ dependencies = [
     "pyspark~=3.5.0"
 ]
 
-python="3.10"
+python = "3.10"
 
 # store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better
 path = ".venv"
 
 [tool.hatch.envs.default.scripts]
-test        = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20"
-coverage    = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20"
+test = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20"
+coverage = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20"
 integration = "pytest tests/integration/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20"
-e2e         = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20"
-fmt         = ["black .",
-               "ruff check . --fix",
-               "mypy .",
-               "pylint --output-format=colorized -j 0 src tests"]
-verify      = ["black --check .",
-               "ruff check .",
-               "mypy .",
-               "pylint --output-format=colorized -j 0 src tests"]
+e2e = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20"
+fmt = [
+    "black .",
+    "ruff check . --fix",
+    "mypy .",
+    "pylint --output-format=colorized -j 0 src tests",
+]
+verify = [
+    "black --check .",
+    "ruff check .",
+    "mypy .",
+    "pylint --output-format=colorized -j 0 src tests",
+]
 update_github_urls = "python docs/dqx/update_github_urls.py"
 extract_yaml_checks_examples = "python src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py"
 
+[tool.hatch.envs.docs]
+dependencies = ["pydoc-markdown>=4.8.2"]
+
+[tool.pydoc-markdown]
+loaders = [{ type = "python", search_path = ["./src"], packages = ["databricks.labs.dqx"] }]
+processors = [{ type = "smart" }, { type = "crossref" }, {type = "filter", skip_empty_modules = true}]
+renderer = { type = "docusaurus", docs_base_path = "docs/dqx/docs", relative_output_path = "reference/api", relative_sidebar_path = "sidebar.json" }
+hooks = {post-render = ["cp -r docs/dqx/docs/reference/api/databricks/labs/dqx/* docs/dqx/docs/reference/api", "rm -rf docs/dqx/docs/reference/api/databricks"]}
+
 [tool.isort]
 profile = "black"
 
@@ -138,11 +152,7 @@ parallel = true
 
 [tool.coverage.report]
 omit = ["*/working-copy/*", 'src/databricks/labs/dqx/__main__.py']
-exclude_lines = [
-    "no cov",
-    "if __name__ == .__main__.:",
-    "if TYPE_CHECKING:",
-]
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
 
 [tool.pylint.main]
 # PyLint configuration is adapted from Google Python Style Guide with modifications.
@@ -321,15 +331,20 @@ function-rgx = "[A-Za-z_][A-Za-z0-9_]{2,}$"
 
 # Good variable names which should always be accepted, separated by a comma.
 good-names = [
-    "f",            # use for file handles
-    "i", "j", "k",  # use for loops
-    "df",           # use for pyspark.sql.DataFrame
-    "ex", "e",      # use for exceptions
-    "fn", "cb",     # use for callbacks
-    "_",            # use for ignores
-    "a",            # use for databricks.sdk.AccountClient
-    "w", "ws",      # use for databricks.sdk.WorkspaceClient
-    "me"            # use for current user
+    "f",  # use for file handles
+    "i",
+    "j",
+    "k",  # use for loops
+    "df", # use for pyspark.sql.DataFrame
+    "ex",
+    "e",  # use for exceptions
+    "fn",
+    "cb", # use for callbacks
+    "_",  # use for ignores
+    "a",  # use for databricks.sdk.AccountClient
+    "w",
+    "ws", # use for databricks.sdk.WorkspaceClient
+    "me", # use for current user
 ]
 
 # Good variable names regexes, separated by a comma. If names match any regex,
@@ -531,7 +546,13 @@ logging-modules = ["logging"]
 [tool.pylint."messages control"]
 # Only show warnings with the listed confidence levels. Leave empty to show all.
 # Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
-confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"]
+confidence = [
+    "HIGH",
+    "CONTROL_FLOW",
+    "INFERENCE",
+    "INFERENCE_FAILURE",
+    "UNDEFINED",
+]
 
 # Disable the message, report, category or checker with the given id(s). You can
 # either give multiple identifiers separated by comma (,) or put this option
@@ -563,7 +584,7 @@ disable = [
     "unnecessary-default-type-args",
     "mock-no-usage",
     "broad-exception-caught",
-    "rewrite-as-for-loop"
+    "rewrite-as-for-loop",
 ]
 
 # Enable the message, report, category or checker with the given id(s). You can
@@ -575,7 +596,16 @@ enable = ["useless-suppression", "use-symbolic-message-instead"]
 [tool.pylint.method_args]
 # List of qualified names (i.e., library.method) which require a timeout
 # parameter e.g. 'requests.api.get,requests.api.post'
-timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"]
+timeout-methods = [
+    "requests.api.delete",
+    "requests.api.get",
+    "requests.api.head",
+    "requests.api.options",
+    "requests.api.patch",
+    "requests.api.post",
+    "requests.api.put",
+    "requests.api.request",
+]
 
 [tool.pylint.miscellaneous]
 # List of note tags to take in consideration, separated by a comma.
@@ -701,12 +731,22 @@ ignore-none = true
 ignore-on-opaque-inference = true
 
 # List of symbolic message names to ignore for Mixin members.
-ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"]
+ignored-checks-for-mixins = [
+    "no-member",
+    "not-async-context-manager",
+    "not-context-manager",
+    "attribute-defined-outside-init",
+]
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes = ["SQLObject", "optparse.Values", "thread._local", "_thread._local"]
+ignored-classes = [
+    "SQLObject",
+    "optparse.Values",
+    "thread._local",
+    "_thread._local",
+]
 
 # Show a hint with possible names when a member name was not found. The aspect of
 # finding the hint is based on edit distance.
@@ -753,7 +793,13 @@ ignored-argument-names = "_.*|^ignored_|^unused_"
 
 # List of qualified module names which can have objects that can redefine
 # builtins.
-redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"]
+redefining-builtins-modules = [
+    "six.moves",
+    "past.builtins",
+    "future.builtins",
+    "builtins",
+    "io",
+]
 
 [tool.hatch.metadata]
 allow-direct-references = true

src/databricks/labs/dqx/base.py

@@ -45,7 +45,6 @@ def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient:
 
 
 class DQEngineCoreBase(DQEngineBase):
-
     @abc.abstractmethod
     def apply_checks(
         self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None

src/databricks/labs/dqx/check_funcs.py

@@ -868,7 +868,7 @@ def sql_query(
     :param msg: Optional custom message or Column expression.
     :param name: Optional name for the result.
     :param negate: If True, the condition is negated (i.e., the check fails when the condition is False).
-    :param input_placeholder: Name to be used in the sql query as {{ input_placeholder }} to refer to the
+    :param input_placeholder: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the
      input DataFrame on which the checks are applied.
     :param row_filter: Optional SQL expression for filtering rows before checking the foreign key.
     Auto-injected from the check filter.
@@ -890,7 +890,7 @@ def sql_query(
 
     def _replace_template(sql: str, replacements: dict[str, str]) -> str:
         """
-        Replace {{ template }} placeholders in sql with actual names, allowing for whitespace between braces.
+        Replace `{{ template }}` placeholders in sql with actual names, allowing for whitespace between braces.
         """
         for key, val in replacements.items():
             pattern = r"\{\{\s*" + re.escape(key) + r"\s*\}\}"
@@ -1117,7 +1117,7 @@ def compare_datasets(
     The comparison does not support Map types (any column comparison on map type is skipped automatically).
 
     The log containing detailed differences is written to the message field of the check result as JSON string.
-    Example: {\"row_missing\":false,\"row_extra\":true,\"changed\":{\"val\":{\"df\":\"val1\"}}}
+    Example: `{"row_missing":false,"row_extra":true,"changed":{"val":{"df":"val1"}}}`
 
     :param columns: List of columns to use for row matching with the reference DataFrame
     (can be a list of string column names or column expressions).
@@ -1336,8 +1336,8 @@ def _match_rows(
 ) -> DataFrame:
     """
     Perform a null-safe join between two DataFrames based on primary key columns.
-    Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols
-    Use eq null safe join to ensure that: 1 == 1 matches; NULL <=> NULL matches; 1 <=> NULL does not match
+    Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols.
+    Use eq null safe join to ensure that: `1 == 1 matches; NULL <=> NULL matches; 1 <=> NULL` does not match
 
     :param df: The input DataFrame to join.
     :param ref_df: The reference DataFrame to join against.

src/databricks/labs/dqx/contexts/workflows.py

@@ -13,7 +13,6 @@
 
 
 class RuntimeContext(GlobalContext):
-
     @cached_property
     def _config_path(self) -> Path:
         config = self.named_parameters.get("config")

src/databricks/labs/dqx/installer/workflows_installer.py

@@ -573,7 +573,6 @@ def _job_clusters(self, job_clusters: set[str], spark_conf: dict[str, str] | Non
 
 
 class MaxedStreamHandler(logging.StreamHandler):
-
     MAX_STREAM_SIZE = 2**20 - 2**6  # 1 Mb minus some buffer
     _installed_handlers: dict[str, tuple[logging.Logger, MaxedStreamHandler]] = {}
     _sent_bytes = 0

src/databricks/labs/dqx/profiler/dlt_generator.py

@@ -11,7 +11,6 @@
 
 
 class DQDltGenerator(DQEngineBase):
-
     def generate_dlt_rules(
         self, rules: list[DQProfile], action: str | None = None, language: str = "SQL"
     ) -> list[str] | str | dict:

src/databricks/labs/dqx/profiler/generator.py

@@ -9,7 +9,6 @@
 
 
 class DQGenerator(DQEngineBase):
-
     def generate_dq_rules(self, profiles: list[DQProfile] | None = None, level: str = "error") -> list[dict]:
         """
         Generates a list of data quality rules based on the provided dq profiles.

src/databricks/labs/dqx/rule.py

@@ -107,7 +107,6 @@ def _build_columns_args(self, columns: list[str | Column] | None, valid_params:
 
 
 class DQRuleTypeMixin:
-
     _expected_rule_type: str  # to be defined in subclasses
     _alternative_rules: list[str]  # e.g., "DQRowRule" or "DQDatasetRule"