From d373ea214a782652bd221f1b29fa7fa60dfbcde3 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 13 Aug 2025 10:26:32 +0200 Subject: [PATCH 01/37] add minimal impl --- .gitignore | 11 +- docs/dqx/docs/reference/api.md | 4 + docs/dqx/package.json | 16 +- docs/dqx/yarn.lock | 1736 ++++++++++++++++---------------- pyproject.toml | 5 + scripts/docgen.py | 668 ++++++++++++ 6 files changed, 1555 insertions(+), 885 deletions(-) create mode 100644 docs/dqx/docs/reference/api.md create mode 100644 scripts/docgen.py diff --git a/.gitignore b/.gitignore index f3299b645..0b1d5ddf4 100644 --- a/.gitignore +++ b/.gitignore @@ -153,4 +153,13 @@ dev/cleanup.py .python-version .databricks-login.json -.local-dev \ No newline at end of file +.local-dev + +# Cursor IDE specific files (not all developers use Cursor) +.cursorrules +.cursor/ + + +# docgen +docs/dqx/generated/** +!docs/dqx/generated/.gitkeep \ No newline at end of file diff --git a/docs/dqx/docs/reference/api.md b/docs/dqx/docs/reference/api.md new file mode 100644 index 000000000..158946bc1 --- /dev/null +++ b/docs/dqx/docs/reference/api.md @@ -0,0 +1,4 @@ + + +# databricks.labs.dqx + diff --git a/docs/dqx/package.json b/docs/dqx/package.json index 6caa19ced..e8e2e09af 100644 --- a/docs/dqx/package.json +++ b/docs/dqx/package.json @@ -12,11 +12,12 @@ "serve": "docusaurus serve", "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", - "typecheck": "tsc" + "typecheck": "tsc", + "generate-api-docs": "tsx src/scripts/generate-api-docs.ts" }, "dependencies": { - "@docusaurus/core": "3.7.0", - "@docusaurus/preset-classic": "3.7.0", + "@docusaurus/core": "^3.8.1", + "@docusaurus/preset-classic": "^3.8.1", "@mdx-js/react": "^3.0.0", "@radix-ui/react-slot": "^1.1.1", "class-variance-authority": "^0.7.1", @@ -28,12 +29,13 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "tailwind-merge": "^2.6.0", - "tailwindcss-animate": "^1.0.7" + "tailwindcss-animate": "^1.0.7", + "tsx": "^4.20.3" }, "devDependencies": { - "@docusaurus/module-type-aliases": "3.7.0", - "@docusaurus/tsconfig": "3.7.0", - "@docusaurus/types": "3.7.0", + "@docusaurus/module-type-aliases": "^3.8.1", + "@docusaurus/tsconfig": "^3.8.1", + "@docusaurus/types": "^3.8.1", "@tailwindcss/typography": "^0.5.16", "autoprefixer": "^10.4.20", "postcss": "^8.5.1", diff --git a/docs/dqx/yarn.lock b/docs/dqx/yarn.lock index 9d42d6c79..788575dc1 100644 --- a/docs/dqx/yarn.lock +++ b/docs/dqx/yarn.lock @@ -163,7 +163,7 @@ "@jridgewell/gen-mapping" "^0.3.5" "@jridgewell/trace-mapping" "^0.3.24" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.25.9", "@babel/code-frame@^7.26.0", "@babel/code-frame@^7.26.2", "@babel/code-frame@^7.8.3": +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.25.9", "@babel/code-frame@^7.26.0", "@babel/code-frame@^7.26.2": version "7.26.2" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.26.2.tgz#4b5fab97d33338eff916235055f0ebc21e573a85" integrity sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ== @@ -1056,92 +1056,103 @@ resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== -"@csstools/cascade-layer-name-parser@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@csstools/cascade-layer-name-parser/-/cascade-layer-name-parser-2.0.4.tgz#64d128529397aa1e1c986f685713363b262b81b1" - integrity sha512-7DFHlPuIxviKYZrOiwVU/PiHLm3lLUR23OMuEEtfEOQTOp9hzQ2JjdY6X5H18RVuUPJqSCI+qNnD5iOLMVE0bA== +"@csstools/cascade-layer-name-parser@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@csstools/cascade-layer-name-parser/-/cascade-layer-name-parser-2.0.5.tgz#43f962bebead0052a9fed1a2deeb11f85efcbc72" + integrity sha512-p1ko5eHgV+MgXFVa4STPKpvPxr6ReS8oS2jzTukjR74i5zJNyWO1ZM1m8YKBXnzDKWfBN1ztLYlHxbVemDD88A== -"@csstools/color-helpers@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@csstools/color-helpers/-/color-helpers-5.0.1.tgz#829f1c76f5800b79c51c709e2f36821b728e0e10" - integrity sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA== +"@csstools/color-helpers@^5.0.2": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@csstools/color-helpers/-/color-helpers-5.0.2.tgz#82592c9a7c2b83c293d9161894e2a6471feb97b8" + integrity sha512-JqWH1vsgdGcw2RR6VliXXdA0/59LttzlU8UlRT/iUUsEeWfYq8I+K0yhihEUTTHLRm1EXvpsCx3083EU15ecsA== -"@csstools/css-calc@^2.1.1": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@csstools/css-calc/-/css-calc-2.1.1.tgz#a7dbc66627f5cf458d42aed14bda0d3860562383" - integrity sha512-rL7kaUnTkL9K+Cvo2pnCieqNpTKgQzy5f+N+5Iuko9HAoasP+xgprVh7KN/MaJVvVL1l0EzQq2MoqBHKSrDrag== +"@csstools/css-calc@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@csstools/css-calc/-/css-calc-2.1.4.tgz#8473f63e2fcd6e459838dd412401d5948f224c65" + integrity sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ== -"@csstools/css-color-parser@^3.0.7": - version "3.0.7" - resolved "https://registry.yarnpkg.com/@csstools/css-color-parser/-/css-color-parser-3.0.7.tgz#442d61d58e54ad258d52c309a787fceb33906484" - integrity sha512-nkMp2mTICw32uE5NN+EsJ4f5N+IGFeCFu4bGpiKgb2Pq/7J/MpyLBeQ5ry4KKtRFZaYs6sTmcMYrSRIyj5DFKA== +"@csstools/css-color-parser@^3.0.10": + version "3.0.10" + resolved "https://registry.yarnpkg.com/@csstools/css-color-parser/-/css-color-parser-3.0.10.tgz#79fc68864dd43c3b6782d2b3828bc0fa9d085c10" + integrity sha512-TiJ5Ajr6WRd1r8HSiwJvZBiJOqtH86aHpUjq5aEKWHiII2Qfjqd/HCWKPOW8EP4vcspXbHnXrwIDlu5savQipg== dependencies: - "@csstools/color-helpers" "^5.0.1" - "@csstools/css-calc" "^2.1.1" + "@csstools/color-helpers" "^5.0.2" + "@csstools/css-calc" "^2.1.4" -"@csstools/css-parser-algorithms@^3.0.4": - version "3.0.4" - resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.4.tgz#74426e93bd1c4dcab3e441f5cc7ba4fb35d94356" - integrity sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A== +"@csstools/css-parser-algorithms@^3.0.5": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz#5755370a9a29abaec5515b43c8b3f2cf9c2e3076" + integrity sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ== -"@csstools/css-tokenizer@^3.0.3": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@csstools/css-tokenizer/-/css-tokenizer-3.0.3.tgz#a5502c8539265fecbd873c1e395a890339f119c2" - integrity sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw== +"@csstools/css-tokenizer@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz#333fedabc3fd1a8e5d0100013731cf19e6a8c5d3" + integrity sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw== -"@csstools/media-query-list-parser@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@csstools/media-query-list-parser/-/media-query-list-parser-4.0.2.tgz#e80e17eba1693fceafb8d6f2cfc68c0e7a9ab78a" - integrity sha512-EUos465uvVvMJehckATTlNqGj4UJWkTmdWuDMjqvSUkjGpmOyFZBVwb4knxCm/k2GMTXY+c/5RkdndzFYWeX5A== +"@csstools/media-query-list-parser@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@csstools/media-query-list-parser/-/media-query-list-parser-4.0.3.tgz#7aec77bcb89c2da80ef207e73f474ef9e1b3cdf1" + integrity sha512-HAYH7d3TLRHDOUQK4mZKf9k9Ph/m8Akstg66ywKR4SFAigjs3yBiUeZtFxywiTm5moZMAp/5W/ZuFnNXXYLuuQ== -"@csstools/postcss-cascade-layers@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-cascade-layers/-/postcss-cascade-layers-5.0.1.tgz#9640313e64b5e39133de7e38a5aa7f40dc259597" - integrity sha512-XOfhI7GShVcKiKwmPAnWSqd2tBR0uxt+runAxttbSp/LY2U16yAVPmAf7e9q4JJ0d+xMNmpwNDLBXnmRCl3HMQ== +"@csstools/postcss-cascade-layers@^5.0.2": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@csstools/postcss-cascade-layers/-/postcss-cascade-layers-5.0.2.tgz#dd2c70db3867b88975f2922da3bfbae7d7a2cae7" + integrity sha512-nWBE08nhO8uWl6kSAeCx4im7QfVko3zLrtgWZY4/bP87zrSPpSyN/3W3TDqz1jJuH+kbKOHXg5rJnK+ZVYcFFg== dependencies: "@csstools/selector-specificity" "^5.0.0" postcss-selector-parser "^7.0.0" -"@csstools/postcss-color-function@^4.0.7": - version "4.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-color-function/-/postcss-color-function-4.0.7.tgz#d31d2044d8a4f8b3154ac54ac77014879eae9f56" - integrity sha512-aDHYmhNIHR6iLw4ElWhf+tRqqaXwKnMl0YsQ/X105Zc4dQwe6yJpMrTN6BwOoESrkDjOYMOfORviSSLeDTJkdQ== +"@csstools/postcss-color-function@^4.0.10": + version "4.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-color-function/-/postcss-color-function-4.0.10.tgz#11ad43a66ef2cc794ab826a07df8b5fa9fb47a3a" + integrity sha512-4dY0NBu7NVIpzxZRgh/Q/0GPSz/jLSw0i/u3LTUor0BkQcz/fNhN10mSWBDsL0p9nDb0Ky1PD6/dcGbhACuFTQ== dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-color-mix-function@^3.0.7": - version "3.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-color-mix-function/-/postcss-color-mix-function-3.0.7.tgz#39735bbc84dc173061e4c2842ec656bb9bc6ed2e" - integrity sha512-e68Nev4CxZYCLcrfWhHH4u/N1YocOfTmw67/kVX5Rb7rnguqqLyxPjhHWjSBX8o4bmyuukmNf3wrUSU3//kT7g== +"@csstools/postcss-color-mix-function@^3.0.10": + version "3.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-color-mix-function/-/postcss-color-mix-function-3.0.10.tgz#8c9d0ccfae5c45a9870dd84807ea2995c7a3a514" + integrity sha512-P0lIbQW9I4ShE7uBgZRib/lMTf9XMjJkFl/d6w4EMNHu2qvQ6zljJGEcBkw/NsBtq/6q3WrmgxSS8kHtPMkK4Q== dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-content-alt-text@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@csstools/postcss-content-alt-text/-/postcss-content-alt-text-2.0.4.tgz#76f4687fb15ed45bc1139bb71e5775779762897a" - integrity sha512-YItlZUOuZJCBlRaCf8Aucc1lgN41qYGALMly0qQllrxYJhiyzlI6RxOTMUvtWk+KhS8GphMDsDhKQ7KTPfEMSw== +"@csstools/postcss-color-mix-variadic-function-arguments@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@csstools/postcss-color-mix-variadic-function-arguments/-/postcss-color-mix-variadic-function-arguments-1.0.0.tgz#0b29cb9b4630d7ed68549db265662d41554a17ed" + integrity sha512-Z5WhouTyD74dPFPrVE7KydgNS9VvnjB8qcdes9ARpCOItb4jTnm7cHp4FhxCRUoyhabD0WVv43wbkJ4p8hLAlQ== dependencies: - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-exponential-functions@^2.0.6": +"@csstools/postcss-content-alt-text@^2.0.6": version "2.0.6" - resolved "https://registry.yarnpkg.com/@csstools/postcss-exponential-functions/-/postcss-exponential-functions-2.0.6.tgz#dcee86d22102576b13d8bea059125fbcf98e83cc" - integrity sha512-IgJA5DQsQLu/upA3HcdvC6xEMR051ufebBTIXZ5E9/9iiaA7juXWz1ceYj814lnDYP/7eWjZnw0grRJlX4eI6g== + resolved "https://registry.yarnpkg.com/@csstools/postcss-content-alt-text/-/postcss-content-alt-text-2.0.6.tgz#548862226eac54bab0ee5f1bf3a9981393ab204b" + integrity sha512-eRjLbOjblXq+byyaedQRSrAejKGNAFued+LcbzT+LCL78fabxHkxYjBbxkroONxHHYu2qxhFK2dBStTLPG3jpQ== dependencies: - "@csstools/css-calc" "^2.1.1" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" + "@csstools/utilities" "^2.0.0" + +"@csstools/postcss-exponential-functions@^2.0.9": + version "2.0.9" + resolved "https://registry.yarnpkg.com/@csstools/postcss-exponential-functions/-/postcss-exponential-functions-2.0.9.tgz#fc03d1272888cb77e64cc1a7d8a33016e4f05c69" + integrity sha512-abg2W/PI3HXwS/CZshSa79kNWNZHdJPMBXeZNyPQFbbj8sKO3jXxOt/wF7juJVjyDTc6JrvaUZYFcSBZBhaxjw== + dependencies: + "@csstools/css-calc" "^2.1.4" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" "@csstools/postcss-font-format-keywords@^4.0.0": version "4.0.0" @@ -1151,67 +1162,67 @@ "@csstools/utilities" "^2.0.0" postcss-value-parser "^4.2.0" -"@csstools/postcss-gamut-mapping@^2.0.7": - version "2.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-gamut-mapping/-/postcss-gamut-mapping-2.0.7.tgz#8aaa4b6ffb6e2187379a83d253607f988533be25" - integrity sha512-gzFEZPoOkY0HqGdyeBXR3JP218Owr683u7KOZazTK7tQZBE8s2yhg06W1tshOqk7R7SWvw9gkw2TQogKpIW8Xw== - dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - -"@csstools/postcss-gradients-interpolation-method@^5.0.7": - version "5.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-gradients-interpolation-method/-/postcss-gradients-interpolation-method-5.0.7.tgz#57e19d25e98aa028b98e22ef392ea24c3e61c568" - integrity sha512-WgEyBeg6glUeTdS2XT7qeTFBthTJuXlS9GFro/DVomj7W7WMTamAwpoP4oQCq/0Ki2gvfRYFi/uZtmRE14/DFA== - dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" +"@csstools/postcss-gamut-mapping@^2.0.10": + version "2.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-gamut-mapping/-/postcss-gamut-mapping-2.0.10.tgz#f518d941231d721dbecf5b41e3c441885ff2f28b" + integrity sha512-QDGqhJlvFnDlaPAfCYPsnwVA6ze+8hhrwevYWlnUeSjkkZfBpcCO42SaUD8jiLlq7niouyLgvup5lh+f1qessg== + dependencies: + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + +"@csstools/postcss-gradients-interpolation-method@^5.0.10": + version "5.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-gradients-interpolation-method/-/postcss-gradients-interpolation-method-5.0.10.tgz#3146da352c31142a721fdba062ac3a6d11dbbec3" + integrity sha512-HHPauB2k7Oits02tKFUeVFEU2ox/H3OQVrP3fSOKDxvloOikSal+3dzlyTZmYsb9FlY9p5EUpBtz0//XBmy+aw== + dependencies: + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-hwb-function@^4.0.7": - version "4.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-hwb-function/-/postcss-hwb-function-4.0.7.tgz#d09528098c4b99c49c76de686a4ae35585acc691" - integrity sha512-LKYqjO+wGwDCfNIEllessCBWfR4MS/sS1WXO+j00KKyOjm7jDW2L6jzUmqASEiv/kkJO39GcoIOvTTfB3yeBUA== +"@csstools/postcss-hwb-function@^4.0.10": + version "4.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-hwb-function/-/postcss-hwb-function-4.0.10.tgz#f93f3c457e6440ac37ef9b908feb5d901b417d50" + integrity sha512-nOKKfp14SWcdEQ++S9/4TgRKchooLZL0TUFdun3nI4KPwCjETmhjta1QT4ICQcGVWQTvrsgMM/aLB5We+kMHhQ== dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-ic-unit@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@csstools/postcss-ic-unit/-/postcss-ic-unit-4.0.0.tgz#b60ec06500717c337447c39ae7fe7952eeb9d48f" - integrity sha512-9QT5TDGgx7wD3EEMN3BSUG6ckb6Eh5gSPT5kZoVtUuAonfPmLDJyPhqR4ntPpMYhUKAMVKAg3I/AgzqHMSeLhA== +"@csstools/postcss-ic-unit@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@csstools/postcss-ic-unit/-/postcss-ic-unit-4.0.2.tgz#7561e09db65fac8304ceeab9dd3e5c6e43414587" + integrity sha512-lrK2jjyZwh7DbxaNnIUjkeDmU8Y6KyzRBk91ZkI5h8nb1ykEfZrtIVArdIjX4DHMIBGpdHrgP0n4qXDr7OHaKA== dependencies: - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" postcss-value-parser "^4.2.0" -"@csstools/postcss-initial@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@csstools/postcss-initial/-/postcss-initial-2.0.0.tgz#a86f5fc59ab9f16f1422dade4c58bd941af5df22" - integrity sha512-dv2lNUKR+JV+OOhZm9paWzYBXOCi+rJPqJ2cJuhh9xd8USVrd0cBEPczla81HNOyThMQWeCcdln3gZkQV2kYxA== +"@csstools/postcss-initial@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@csstools/postcss-initial/-/postcss-initial-2.0.1.tgz#c385bd9d8ad31ad159edd7992069e97ceea4d09a" + integrity sha512-L1wLVMSAZ4wovznquK0xmC7QSctzO4D0Is590bxpGqhqjboLXYA16dWZpfwImkdOgACdQ9PqXsuRroW6qPlEsg== -"@csstools/postcss-is-pseudo-class@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-is-pseudo-class/-/postcss-is-pseudo-class-5.0.1.tgz#12041448fedf01090dd4626022c28b7f7623f58e" - integrity sha512-JLp3POui4S1auhDR0n8wHd/zTOWmMsmK3nQd3hhL6FhWPaox5W7j1se6zXOG/aP07wV2ww0lxbKYGwbBszOtfQ== +"@csstools/postcss-is-pseudo-class@^5.0.3": + version "5.0.3" + resolved "https://registry.yarnpkg.com/@csstools/postcss-is-pseudo-class/-/postcss-is-pseudo-class-5.0.3.tgz#d34e850bcad4013c2ed7abe948bfa0448aa8eb74" + integrity sha512-jS/TY4SpG4gszAtIg7Qnf3AS2pjcUM5SzxpApOrlndMeGhIbaTzWBzzP/IApXoNWEW7OhcjkRT48jnAUIFXhAQ== dependencies: "@csstools/selector-specificity" "^5.0.0" postcss-selector-parser "^7.0.0" -"@csstools/postcss-light-dark-function@^2.0.7": - version "2.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-light-dark-function/-/postcss-light-dark-function-2.0.7.tgz#807c170cd28eebb0c00e64dfc6ab0bf418f19209" - integrity sha512-ZZ0rwlanYKOHekyIPaU+sVm3BEHCe+Ha0/px+bmHe62n0Uc1lL34vbwrLYn6ote8PHlsqzKeTQdIejQCJ05tfw== +"@csstools/postcss-light-dark-function@^2.0.9": + version "2.0.9" + resolved "https://registry.yarnpkg.com/@csstools/postcss-light-dark-function/-/postcss-light-dark-function-2.0.9.tgz#9fb080188907539734a9d5311d2a1cb82531ef38" + integrity sha512-1tCZH5bla0EAkFAI2r0H33CDnIBeLUaJh1p+hvvsylJ4svsv2wOmJjJn+OXwUZLXef37GYbRIVKX+X+g6m+3CQ== dependencies: - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" "@csstools/postcss-logical-float-and-clear@^3.0.0": @@ -1236,32 +1247,32 @@ dependencies: postcss-value-parser "^4.2.0" -"@csstools/postcss-logical-viewport-units@^3.0.3": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@csstools/postcss-logical-viewport-units/-/postcss-logical-viewport-units-3.0.3.tgz#f6cc63520ca2a6eb76b9cd946070c38dda66d733" - integrity sha512-OC1IlG/yoGJdi0Y+7duz/kU/beCwO+Gua01sD6GtOtLi7ByQUpcIqs7UE/xuRPay4cHgOMatWdnDdsIDjnWpPw== +"@csstools/postcss-logical-viewport-units@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@csstools/postcss-logical-viewport-units/-/postcss-logical-viewport-units-3.0.4.tgz#016d98a8b7b5f969e58eb8413447eb801add16fc" + integrity sha512-q+eHV1haXA4w9xBwZLKjVKAWn3W2CMqmpNpZUk5kRprvSiBEGMgrNH3/sJZ8UA3JgyHaOt3jwT9uFa4wLX4EqQ== dependencies: - "@csstools/css-tokenizer" "^3.0.3" + "@csstools/css-tokenizer" "^3.0.4" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-media-minmax@^2.0.6": - version "2.0.6" - resolved "https://registry.yarnpkg.com/@csstools/postcss-media-minmax/-/postcss-media-minmax-2.0.6.tgz#427921c0f08033203810af16dfed0baedc538eab" - integrity sha512-J1+4Fr2W3pLZsfxkFazK+9kr96LhEYqoeBszLmFjb6AjYs+g9oDAw3J5oQignLKk3rC9XHW+ebPTZ9FaW5u5pg== +"@csstools/postcss-media-minmax@^2.0.9": + version "2.0.9" + resolved "https://registry.yarnpkg.com/@csstools/postcss-media-minmax/-/postcss-media-minmax-2.0.9.tgz#184252d5b93155ae526689328af6bdf3fc113987" + integrity sha512-af9Qw3uS3JhYLnCbqtZ9crTvvkR+0Se+bBqSr7ykAnl9yKhk6895z9rf+2F4dClIDJWxgn0iZZ1PSdkhrbs2ig== dependencies: - "@csstools/css-calc" "^2.1.1" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/media-query-list-parser" "^4.0.2" + "@csstools/css-calc" "^2.1.4" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/media-query-list-parser" "^4.0.3" -"@csstools/postcss-media-queries-aspect-ratio-number-values@^3.0.4": - version "3.0.4" - resolved "https://registry.yarnpkg.com/@csstools/postcss-media-queries-aspect-ratio-number-values/-/postcss-media-queries-aspect-ratio-number-values-3.0.4.tgz#d71102172c74baf3f892fac88cf1ea46a961600d" - integrity sha512-AnGjVslHMm5xw9keusQYvjVWvuS7KWK+OJagaG0+m9QnIjZsrysD2kJP/tr/UJIyYtMCtu8OkUd+Rajb4DqtIQ== +"@csstools/postcss-media-queries-aspect-ratio-number-values@^3.0.5": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@csstools/postcss-media-queries-aspect-ratio-number-values/-/postcss-media-queries-aspect-ratio-number-values-3.0.5.tgz#f485c31ec13d6b0fb5c528a3474334a40eff5f11" + integrity sha512-zhAe31xaaXOY2Px8IYfoVTB3wglbJUVigGphFLj6exb7cjZRH9A6adyE22XfFK3P2PzwRk0VDeTJmaxpluyrDg== dependencies: - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/media-query-list-parser" "^4.0.2" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/media-query-list-parser" "^4.0.3" "@csstools/postcss-nested-calc@^4.0.0": version "4.0.0" @@ -1278,42 +1289,42 @@ dependencies: postcss-value-parser "^4.2.0" -"@csstools/postcss-oklab-function@^4.0.7": - version "4.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-oklab-function/-/postcss-oklab-function-4.0.7.tgz#33b3322dfb27b0b5eb83a7ad36e67f08bc4e66cd" - integrity sha512-I6WFQIbEKG2IO3vhaMGZDkucbCaUSXMxvHNzDdnfsTCF5tc0UlV3Oe2AhamatQoKFjBi75dSEMrgWq3+RegsOQ== +"@csstools/postcss-oklab-function@^4.0.10": + version "4.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-oklab-function/-/postcss-oklab-function-4.0.10.tgz#d4c23c51dd0be45e6dedde22432d7d0003710780" + integrity sha512-ZzZUTDd0fgNdhv8UUjGCtObPD8LYxMH+MJsW9xlZaWTV8Ppr4PtxlHYNMmF4vVWGl0T6f8tyWAKjoI6vePSgAg== dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" -"@csstools/postcss-progressive-custom-properties@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@csstools/postcss-progressive-custom-properties/-/postcss-progressive-custom-properties-4.0.0.tgz#ecdb85bcdb1852d73970a214a376684a91f82bdc" - integrity sha512-XQPtROaQjomnvLUSy/bALTR5VCtTVUFwYs1SblvYgLSeTo2a/bMNwUwo2piXw5rTv/FEYiy5yPSXBqg9OKUx7Q== +"@csstools/postcss-progressive-custom-properties@^4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@csstools/postcss-progressive-custom-properties/-/postcss-progressive-custom-properties-4.1.0.tgz#70c8d41b577f4023633b7e3791604e0b7f3775bc" + integrity sha512-YrkI9dx8U4R8Sz2EJaoeD9fI7s7kmeEBfmO+UURNeL6lQI7VxF6sBE+rSqdCBn4onwqmxFdBU3lTwyYb/lCmxA== dependencies: postcss-value-parser "^4.2.0" -"@csstools/postcss-random-function@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@csstools/postcss-random-function/-/postcss-random-function-1.0.2.tgz#699702820f19bb6b9632966ff44d8957db6889d2" - integrity sha512-vBCT6JvgdEkvRc91NFoNrLjgGtkLWt47GKT6E2UDn3nd8ZkMBiziQ1Md1OiKoSsgzxsSnGKG3RVdhlbdZEkHjA== - dependencies: - "@csstools/css-calc" "^2.1.1" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - -"@csstools/postcss-relative-color-syntax@^3.0.7": - version "3.0.7" - resolved "https://registry.yarnpkg.com/@csstools/postcss-relative-color-syntax/-/postcss-relative-color-syntax-3.0.7.tgz#862f8c6a2bbbab1a46aff8265b6a095fd267a3a6" - integrity sha512-apbT31vsJVd18MabfPOnE977xgct5B1I+Jpf+Munw3n6kKb1MMuUmGGH+PT9Hm/fFs6fe61Q/EWnkrb4bNoNQw== - dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" +"@csstools/postcss-random-function@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@csstools/postcss-random-function/-/postcss-random-function-2.0.1.tgz#3191f32fe72936e361dadf7dbfb55a0209e2691e" + integrity sha512-q+FQaNiRBhnoSNo+GzqGOIBKoHQ43lYz0ICrV+UudfWnEF6ksS6DsBIJSISKQT2Bvu3g4k6r7t0zYrk5pDlo8w== + dependencies: + "@csstools/css-calc" "^2.1.4" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + +"@csstools/postcss-relative-color-syntax@^3.0.10": + version "3.0.10" + resolved "https://registry.yarnpkg.com/@csstools/postcss-relative-color-syntax/-/postcss-relative-color-syntax-3.0.10.tgz#daa840583969461e1e06b12e9c591e52a790ec86" + integrity sha512-8+0kQbQGg9yYG8hv0dtEpOMLwB9M+P7PhacgIzVzJpixxV4Eq9AUQtQw8adMmAJU1RBBmIlpmtmm3XTRd/T00g== + dependencies: + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" "@csstools/postcss-scope-pseudo-class@^4.0.1": @@ -1323,50 +1334,50 @@ dependencies: postcss-selector-parser "^7.0.0" -"@csstools/postcss-sign-functions@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-sign-functions/-/postcss-sign-functions-1.1.1.tgz#eb8e4a5ac637982aeb9264cb99f85817612ad3e8" - integrity sha512-MslYkZCeMQDxetNkfmmQYgKCy4c+w9pPDfgOBCJOo/RI1RveEUdZQYtOfrC6cIZB7sD7/PHr2VGOcMXlZawrnA== +"@csstools/postcss-sign-functions@^1.1.4": + version "1.1.4" + resolved "https://registry.yarnpkg.com/@csstools/postcss-sign-functions/-/postcss-sign-functions-1.1.4.tgz#a9ac56954014ae4c513475b3f1b3e3424a1e0c12" + integrity sha512-P97h1XqRPcfcJndFdG95Gv/6ZzxUBBISem0IDqPZ7WMvc/wlO+yU0c5D/OCpZ5TJoTt63Ok3knGk64N+o6L2Pg== dependencies: - "@csstools/css-calc" "^2.1.1" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" + "@csstools/css-calc" "^2.1.4" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" -"@csstools/postcss-stepped-value-functions@^4.0.6": - version "4.0.6" - resolved "https://registry.yarnpkg.com/@csstools/postcss-stepped-value-functions/-/postcss-stepped-value-functions-4.0.6.tgz#ee88c6122daf58a1b8641f462e8e33427c60b1f1" - integrity sha512-/dwlO9w8vfKgiADxpxUbZOWlL5zKoRIsCymYoh1IPuBsXODKanKnfuZRr32DEqT0//3Av1VjfNZU9yhxtEfIeA== +"@csstools/postcss-stepped-value-functions@^4.0.9": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@csstools/postcss-stepped-value-functions/-/postcss-stepped-value-functions-4.0.9.tgz#36036f1a0e5e5ee2308e72f3c9cb433567c387b9" + integrity sha512-h9btycWrsex4dNLeQfyU3y3w40LMQooJWFMm/SK9lrKguHDcFl4VMkncKKoXi2z5rM9YGWbUQABI8BT2UydIcA== dependencies: - "@csstools/css-calc" "^2.1.1" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" + "@csstools/css-calc" "^2.1.4" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" -"@csstools/postcss-text-decoration-shorthand@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@csstools/postcss-text-decoration-shorthand/-/postcss-text-decoration-shorthand-4.0.1.tgz#251fab0939d50c6fd73bb2b830b2574188efa087" - integrity sha512-xPZIikbx6jyzWvhms27uugIc0I4ykH4keRvoa3rxX5K7lEhkbd54rjj/dv60qOCTisoS+3bmwJTeyV1VNBrXaw== +"@csstools/postcss-text-decoration-shorthand@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@csstools/postcss-text-decoration-shorthand/-/postcss-text-decoration-shorthand-4.0.2.tgz#a3bcf80492e6dda36477538ab8e8943908c9f80a" + integrity sha512-8XvCRrFNseBSAGxeaVTaNijAu+FzUvjwFXtcrynmazGb/9WUdsPCpBX+mHEHShVRq47Gy4peYAoxYs8ltUnmzA== dependencies: - "@csstools/color-helpers" "^5.0.1" + "@csstools/color-helpers" "^5.0.2" postcss-value-parser "^4.2.0" -"@csstools/postcss-trigonometric-functions@^4.0.6": - version "4.0.6" - resolved "https://registry.yarnpkg.com/@csstools/postcss-trigonometric-functions/-/postcss-trigonometric-functions-4.0.6.tgz#fc5c5f4c9bd0fd796b58b9a14d5d663be76d19fa" - integrity sha512-c4Y1D2Why/PeccaSouXnTt6WcNHJkoJRidV2VW9s5gJ97cNxnLgQ4Qj8qOqkIR9VmTQKJyNcbF4hy79ZQnWD7A== +"@csstools/postcss-trigonometric-functions@^4.0.9": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@csstools/postcss-trigonometric-functions/-/postcss-trigonometric-functions-4.0.9.tgz#3f94ed2e319b57f2c59720b64e4d0a8a6fb8c3b2" + integrity sha512-Hnh5zJUdpNrJqK9v1/E3BbrQhaDTj5YiX7P61TOvUhoDHnUmsNNxcDAgkQ32RrcWx9GVUvfUNPcUkn8R3vIX6A== dependencies: - "@csstools/css-calc" "^2.1.1" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" + "@csstools/css-calc" "^2.1.4" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" "@csstools/postcss-unset-value@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@csstools/postcss-unset-value/-/postcss-unset-value-4.0.0.tgz#7caa981a34196d06a737754864baf77d64de4bba" integrity sha512-cBz3tOCI5Fw6NIFEwU3RiwK6mn3nKegjpJuzCndoGq3BZPkUjnsq7uQmIeMNeMbMk7YD2MfKcgCpZwX5jyXqCA== -"@csstools/selector-resolve-nested@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@csstools/selector-resolve-nested/-/selector-resolve-nested-3.0.0.tgz#704a9b637975680e025e069a4c58b3beb3e2752a" - integrity sha512-ZoK24Yku6VJU1gS79a5PFmC8yn3wIapiKmPgun0hZgEI5AOqgH2kiPRsPz1qkGv4HL+wuDLH83yQyk6inMYrJQ== +"@csstools/selector-resolve-nested@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@csstools/selector-resolve-nested/-/selector-resolve-nested-3.1.0.tgz#848c6f44cb65e3733e478319b9342b7aa436fac7" + integrity sha512-mf1LEW0tJLKfWyvn5KdDrhpxHyuxpbNwTIwOYLIvsTffeyOf85j5oIzfG0yosxDgx/sswlqBnESYUcQH0vgZ0g== "@csstools/selector-specificity@^5.0.0": version "5.0.0" @@ -1383,25 +1394,25 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== -"@docsearch/css@3.8.3": - version "3.8.3" - resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.8.3.tgz#12f377cf8c14b687042273f920efdfdb794e9fcf" - integrity sha512-1nELpMV40JDLJ6rpVVFX48R1jsBFIQ6RnEQDsLFGmzOjPWTOMlZqUcXcvRx8VmYV/TqnS1l784Ofz+ZEb+wEOQ== +"@docsearch/css@3.9.0": + version "3.9.0" + resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.9.0.tgz#3bc29c96bf024350d73b0cfb7c2a7b71bf251cd5" + integrity sha512-cQbnVbq0rrBwNAKegIac/t6a8nWoUAn8frnkLFW6YARaRmAQr5/Eoe6Ln2fqkUCZ40KpdrKbpSAmgrkviOxuWA== -"@docsearch/react@^3.8.1": - version "3.8.3" - resolved "https://registry.yarnpkg.com/@docsearch/react/-/react-3.8.3.tgz#72f6bcbbda6cd07f23398af641e483c27d16e00a" - integrity sha512-6UNrg88K7lJWmuS6zFPL/xgL+n326qXqZ7Ybyy4E8P/6Rcblk3GE8RXxeol4Pd5pFpKMhOhBhzABKKwHtbJCIg== +"@docsearch/react@^3.9.0": + version "3.9.0" + resolved "https://registry.yarnpkg.com/@docsearch/react/-/react-3.9.0.tgz#d0842b700c3ee26696786f3c8ae9f10c1a3f0db3" + integrity sha512-mb5FOZYZIkRQ6s/NWnM98k879vu5pscWqTLubLFBO87igYYT4VzVazh4h5o/zCvTIZgEt3PvsCOMOswOUo9yHQ== dependencies: "@algolia/autocomplete-core" "1.17.9" "@algolia/autocomplete-preset-algolia" "1.17.9" - "@docsearch/css" "3.8.3" + "@docsearch/css" "3.9.0" algoliasearch "^5.14.2" -"@docusaurus/babel@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/babel/-/babel-3.7.0.tgz#770dd5da525a9d6a2fee7d3212ec62040327f776" - integrity sha512-0H5uoJLm14S/oKV3Keihxvh8RV+vrid+6Gv+2qhuzbqHanawga8tYnsdpjEyt36ucJjqlby2/Md2ObWjA02UXQ== +"@docusaurus/babel@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/babel/-/babel-3.8.1.tgz#db329ac047184214e08e2dbc809832c696c18506" + integrity sha512-3brkJrml8vUbn9aeoZUlJfsI/GqyFcDgQJwQkmBtclJgWDEQBKKeagZfOgx0WfUQhagL1sQLNW0iBdxnI863Uw== dependencies: "@babel/core" "^7.25.9" "@babel/generator" "^7.25.9" @@ -1413,55 +1424,54 @@ "@babel/runtime" "^7.25.9" "@babel/runtime-corejs3" "^7.25.9" "@babel/traverse" "^7.25.9" - "@docusaurus/logger" "3.7.0" - "@docusaurus/utils" "3.7.0" + "@docusaurus/logger" "3.8.1" + "@docusaurus/utils" "3.8.1" babel-plugin-dynamic-import-node "^2.3.3" fs-extra "^11.1.1" tslib "^2.6.0" -"@docusaurus/bundler@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/bundler/-/bundler-3.7.0.tgz#d8e7867b3b2c43a1e320ed429f8dfe873c38506d" - integrity sha512-CUUT9VlSGukrCU5ctZucykvgCISivct+cby28wJwCC/fkQFgAHRp/GKv2tx38ZmXb7nacrKzFTcp++f9txUYGg== +"@docusaurus/bundler@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/bundler/-/bundler-3.8.1.tgz#e2b11d615f09a6e470774bb36441b8d06736b94c" + integrity sha512-/z4V0FRoQ0GuSLToNjOSGsk6m2lQUG4FRn8goOVoZSRsTrU8YR2aJacX5K3RG18EaX9b+52pN4m1sL3MQZVsQA== dependencies: "@babel/core" "^7.25.9" - "@docusaurus/babel" "3.7.0" - "@docusaurus/cssnano-preset" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" + "@docusaurus/babel" "3.8.1" + "@docusaurus/cssnano-preset" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" babel-loader "^9.2.1" - clean-css "^5.3.2" + clean-css "^5.3.3" copy-webpack-plugin "^11.0.0" - css-loader "^6.8.1" + css-loader "^6.11.0" css-minimizer-webpack-plugin "^5.0.1" cssnano "^6.1.2" file-loader "^6.2.0" html-minifier-terser "^7.2.0" - mini-css-extract-plugin "^2.9.1" + mini-css-extract-plugin "^2.9.2" null-loader "^4.0.1" - postcss "^8.4.26" - postcss-loader "^7.3.3" - postcss-preset-env "^10.1.0" - react-dev-utils "^12.0.1" + postcss "^8.5.4" + postcss-loader "^7.3.4" + postcss-preset-env "^10.2.1" terser-webpack-plugin "^5.3.9" tslib "^2.6.0" url-loader "^4.1.1" webpack "^5.95.0" webpackbar "^6.0.1" -"@docusaurus/core@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-3.7.0.tgz#e871586d099093723dfe6de81c1ce610aeb20292" - integrity sha512-b0fUmaL+JbzDIQaamzpAFpTviiaU4cX3Qz8cuo14+HGBCwa0evEK0UYCBFY3n4cLzL8Op1BueeroUD2LYAIHbQ== - dependencies: - "@docusaurus/babel" "3.7.0" - "@docusaurus/bundler" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/mdx-loader" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" +"@docusaurus/core@3.8.1", "@docusaurus/core@^3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-3.8.1.tgz#c22e47c16a22cb7d245306c64bc54083838ff3db" + integrity sha512-ENB01IyQSqI2FLtOzqSI3qxG2B/jP4gQPahl2C3XReiLebcVh5B5cB9KYFvdoOqOWPyr5gXK4sjgTKv7peXCrA== + dependencies: + "@docusaurus/babel" "3.8.1" + "@docusaurus/bundler" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/mdx-loader" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" boxen "^6.2.1" chalk "^4.1.2" chokidar "^3.5.3" @@ -1469,19 +1479,19 @@ combine-promises "^1.1.0" commander "^5.1.0" core-js "^3.31.1" - del "^6.1.1" detect-port "^1.5.1" escape-html "^1.0.3" eta "^2.2.0" eval "^0.1.8" + execa "5.1.1" fs-extra "^11.1.1" html-tags "^3.3.1" html-webpack-plugin "^5.6.0" leven "^3.1.0" lodash "^4.17.21" + open "^8.4.0" p-map "^4.0.0" prompts "^2.4.2" - react-dev-utils "^12.0.1" react-helmet-async "npm:@slorber/react-helmet-async@1.3.0" react-loadable "npm:@docusaurus/react-loadable@6.0.0" react-loadable-ssr-addon-v5-slorber "^1.0.1" @@ -1490,7 +1500,7 @@ react-router-dom "^5.3.4" semver "^7.5.4" serve-handler "^6.1.6" - shelljs "^0.8.5" + tinypool "^1.0.2" tslib "^2.6.0" update-notifier "^6.0.2" webpack "^5.95.0" @@ -1498,39 +1508,39 @@ webpack-dev-server "^4.15.2" webpack-merge "^6.0.1" -"@docusaurus/cssnano-preset@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-3.7.0.tgz#8fe8f2c3acbd32384b69e14983b9a63c98cae34e" - integrity sha512-X9GYgruZBSOozg4w4dzv9uOz8oK/EpPVQXkp0MM6Tsgp/nRIU9hJzJ0Pxg1aRa3xCeEQTOimZHcocQFlLwYajQ== +"@docusaurus/cssnano-preset@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-3.8.1.tgz#bd55026251a6ab8e2194839a2042458ef9880c44" + integrity sha512-G7WyR2N6SpyUotqhGznERBK+x84uyhfMQM2MmDLs88bw4Flom6TY46HzkRkSEzaP9j80MbTN8naiL1fR17WQug== dependencies: cssnano-preset-advanced "^6.1.2" - postcss "^8.4.38" + postcss "^8.5.4" postcss-sort-media-queries "^5.2.0" tslib "^2.6.0" -"@docusaurus/logger@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/logger/-/logger-3.7.0.tgz#07ecc2f460c4d2382df4991f9ce4e348e90af04c" - integrity sha512-z7g62X7bYxCYmeNNuO9jmzxLQG95q9QxINCwpboVcNff3SJiHJbGrarxxOVMVmAh1MsrSfxWkVGv4P41ktnFsA== +"@docusaurus/logger@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/logger/-/logger-3.8.1.tgz#45321b2e2e14695d0dbd8b4104ea7b0fbaa98700" + integrity sha512-2wjeGDhKcExEmjX8k1N/MRDiPKXGF2Pg+df/bDDPnnJWHXnVEZxXj80d6jcxp1Gpnksl0hF8t/ZQw9elqj2+ww== dependencies: chalk "^4.1.2" tslib "^2.6.0" -"@docusaurus/mdx-loader@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-3.7.0.tgz#5890c6e7a5b68cb1d066264ac5290cdcd59d4ecc" - integrity sha512-OFBG6oMjZzc78/U3WNPSHs2W9ZJ723ewAcvVJaqS0VgyeUfmzUV8f1sv+iUHA0DtwiR5T5FjOxj6nzEE8LY6VA== +"@docusaurus/mdx-loader@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-3.8.1.tgz#74309b3614bbcef1d55fb13e6cc339b7fb000b5f" + integrity sha512-DZRhagSFRcEq1cUtBMo4TKxSNo/W6/s44yhr8X+eoXqCLycFQUylebOMPseHi5tc4fkGJqwqpWJLz6JStU9L4w== dependencies: - "@docusaurus/logger" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" + "@docusaurus/logger" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" "@mdx-js/mdx" "^3.0.0" "@slorber/remark-comment" "^1.0.0" escape-html "^1.0.3" estree-util-value-to-estree "^3.0.1" file-loader "^6.2.0" fs-extra "^11.1.1" - image-size "^1.0.2" + image-size "^2.0.2" mdast-util-mdx "^3.0.0" mdast-util-to-string "^4.0.0" rehype-raw "^7.0.0" @@ -1546,197 +1556,210 @@ vfile "^6.0.1" webpack "^5.88.1" -"@docusaurus/module-type-aliases@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/module-type-aliases/-/module-type-aliases-3.7.0.tgz#15c0745b829c6966c5b3b2c2527c72b54830b0e5" - integrity sha512-g7WdPqDNaqA60CmBrr0cORTrsOit77hbsTj7xE2l71YhBn79sxdm7WMK7wfhcaafkbpIh7jv5ef5TOpf1Xv9Lg== +"@docusaurus/module-type-aliases@3.8.1", "@docusaurus/module-type-aliases@^3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/module-type-aliases/-/module-type-aliases-3.8.1.tgz#454de577bd7f50b5eae16db0f76b49ca5e4e281a" + integrity sha512-6xhvAJiXzsaq3JdosS7wbRt/PwEPWHr9eM4YNYqVlbgG1hSK3uQDXTVvQktasp3VO6BmfYWPozueLWuj4gB+vg== dependencies: - "@docusaurus/types" "3.7.0" + "@docusaurus/types" "3.8.1" "@types/history" "^4.7.11" "@types/react" "*" "@types/react-router-config" "*" "@types/react-router-dom" "*" - react-helmet-async "npm:@slorber/react-helmet-async@*" + react-helmet-async "npm:@slorber/react-helmet-async@1.3.0" react-loadable "npm:@docusaurus/react-loadable@6.0.0" -"@docusaurus/plugin-content-blog@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.7.0.tgz#7bd69de87a1f3adb652e1473ef5b7ccc9468f47e" - integrity sha512-EFLgEz6tGHYWdPU0rK8tSscZwx+AsyuBW/r+tNig2kbccHYGUJmZtYN38GjAa3Fda4NU+6wqUO5kTXQSRBQD3g== - dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/mdx-loader" "3.7.0" - "@docusaurus/theme-common" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" +"@docusaurus/plugin-content-blog@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.8.1.tgz#88d842b562b04cf59df900d9f6984b086f821525" + integrity sha512-vNTpMmlvNP9n3hGEcgPaXyvTljanAKIUkuG9URQ1DeuDup0OR7Ltvoc8yrmH+iMZJbcQGhUJF+WjHLwuk8HSdw== + dependencies: + "@docusaurus/core" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/mdx-loader" "3.8.1" + "@docusaurus/theme-common" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" cheerio "1.0.0-rc.12" feed "^4.2.2" fs-extra "^11.1.1" lodash "^4.17.21" - reading-time "^1.5.0" + schema-dts "^1.1.2" srcset "^4.0.0" tslib "^2.6.0" unist-util-visit "^5.0.0" utility-types "^3.10.0" webpack "^5.88.1" -"@docusaurus/plugin-content-docs@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.7.0.tgz#297a549e926ee2b1147b5242af6f21532c7b107c" - integrity sha512-GXg5V7kC9FZE4FkUZA8oo/NrlRb06UwuICzI6tcbzj0+TVgjq/mpUXXzSgKzMS82YByi4dY2Q808njcBCyy6tQ== - dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/mdx-loader" "3.7.0" - "@docusaurus/module-type-aliases" "3.7.0" - "@docusaurus/theme-common" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" +"@docusaurus/plugin-content-docs@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.8.1.tgz#40686a206abb6373bee5638de100a2c312f112a4" + integrity sha512-oByRkSZzeGNQByCMaX+kif5Nl2vmtj2IHQI2fWjCfCootsdKZDPFLonhIp5s3IGJO7PLUfe0POyw0Xh/RrGXJA== + dependencies: + "@docusaurus/core" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/mdx-loader" "3.8.1" + "@docusaurus/module-type-aliases" "3.8.1" + "@docusaurus/theme-common" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" "@types/react-router-config" "^5.0.7" combine-promises "^1.1.0" fs-extra "^11.1.1" js-yaml "^4.1.0" lodash "^4.17.21" + schema-dts "^1.1.2" tslib "^2.6.0" utility-types "^3.10.0" webpack "^5.88.1" -"@docusaurus/plugin-content-pages@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.7.0.tgz#c4a8f7237872236aacb77665822c474c0a00e91a" - integrity sha512-YJSU3tjIJf032/Aeao8SZjFOrXJbz/FACMveSMjLyMH4itQyZ2XgUIzt4y+1ISvvk5zrW4DABVT2awTCqBkx0Q== +"@docusaurus/plugin-content-pages@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.8.1.tgz#41b684dbd15390b7bb6a627f78bf81b6324511ac" + integrity sha512-a+V6MS2cIu37E/m7nDJn3dcxpvXb6TvgdNI22vJX8iUTp8eoMoPa0VArEbWvCxMY/xdC26WzNv4wZ6y0iIni/w== dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/mdx-loader" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/mdx-loader" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" fs-extra "^11.1.1" tslib "^2.6.0" webpack "^5.88.1" -"@docusaurus/plugin-debug@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-3.7.0.tgz#a4fd45132e40cffe96bb51f48e89982a1cb8e194" - integrity sha512-Qgg+IjG/z4svtbCNyTocjIwvNTNEwgRjSXXSJkKVG0oWoH0eX/HAPiu+TS1HBwRPQV+tTYPWLrUypYFepfujZA== +"@docusaurus/plugin-css-cascade-layers@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-css-cascade-layers/-/plugin-css-cascade-layers-3.8.1.tgz#cb414b4a82aa60fc64ef2a435ad0105e142a6c71" + integrity sha512-VQ47xRxfNKjHS5ItzaVXpxeTm7/wJLFMOPo1BkmoMG4Cuz4nuI+Hs62+RMk1OqVog68Swz66xVPK8g9XTrBKRw== + dependencies: + "@docusaurus/core" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" + tslib "^2.6.0" + +"@docusaurus/plugin-debug@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-3.8.1.tgz#45b107e46b627caaae66995f53197ace78af3491" + integrity sha512-nT3lN7TV5bi5hKMB7FK8gCffFTBSsBsAfV84/v293qAmnHOyg1nr9okEw8AiwcO3bl9vije5nsUvP0aRl2lpaw== dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" fs-extra "^11.1.1" - react-json-view-lite "^1.2.0" + react-json-view-lite "^2.3.0" tslib "^2.6.0" -"@docusaurus/plugin-google-analytics@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.7.0.tgz#d20f665e810fb2295d1c1bbfe13398c5ff42eb24" - integrity sha512-otIqiRV/jka6Snjf+AqB360XCeSv7lQC+DKYW+EUZf6XbuE8utz5PeUQ8VuOcD8Bk5zvT1MC4JKcd5zPfDuMWA== +"@docusaurus/plugin-google-analytics@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.8.1.tgz#64a302e62fe5cb6e007367c964feeef7b056764a" + integrity sha512-Hrb/PurOJsmwHAsfMDH6oVpahkEGsx7F8CWMjyP/dw1qjqmdS9rcV1nYCGlM8nOtD3Wk/eaThzUB5TSZsGz+7Q== dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" tslib "^2.6.0" -"@docusaurus/plugin-google-gtag@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.7.0.tgz#a48638dfd132858060458b875a440b6cbda6bf8f" - integrity sha512-M3vrMct1tY65ModbyeDaMoA+fNJTSPe5qmchhAbtqhDD/iALri0g9LrEpIOwNaoLmm6lO88sfBUADQrSRSGSWA== +"@docusaurus/plugin-google-gtag@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.8.1.tgz#8c76f8a1d96448f2f0f7b10e6bde451c40672b95" + integrity sha512-tKE8j1cEZCh8KZa4aa80zpSTxsC2/ZYqjx6AAfd8uA8VHZVw79+7OTEP2PoWi0uL5/1Is0LF5Vwxd+1fz5HlKg== dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" "@types/gtag.js" "^0.0.12" tslib "^2.6.0" -"@docusaurus/plugin-google-tag-manager@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.7.0.tgz#0a4390f4b0e760d073bdb1905436bfa7bd71356b" - integrity sha512-X8U78nb8eiMiPNg3jb9zDIVuuo/rE1LjGDGu+5m5CX4UBZzjMy+klOY2fNya6x8ACyE/L3K2erO1ErheP55W/w== +"@docusaurus/plugin-google-tag-manager@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.8.1.tgz#88241ffd06369f4a4d5fb982ff3ac2777561ae37" + integrity sha512-iqe3XKITBquZq+6UAXdb1vI0fPY5iIOitVjPQ581R1ZKpHr0qe+V6gVOrrcOHixPDD/BUKdYwkxFjpNiEN+vBw== dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" tslib "^2.6.0" -"@docusaurus/plugin-sitemap@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.7.0.tgz#2c1bf9de26aeda455df6f77748e5887ace39b2d7" - integrity sha512-bTRT9YLZ/8I/wYWKMQke18+PF9MV8Qub34Sku6aw/vlZ/U+kuEuRpQ8bTcNOjaTSfYsWkK4tTwDMHK2p5S86cA== - dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" +"@docusaurus/plugin-sitemap@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.8.1.tgz#3aebd39186dc30e53023f1aab44625bc0bdac892" + integrity sha512-+9YV/7VLbGTq8qNkjiugIelmfUEVkTyLe6X8bWq7K5qPvGXAjno27QAfFq63mYfFFbJc7z+pudL63acprbqGzw== + dependencies: + "@docusaurus/core" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" fs-extra "^11.1.1" sitemap "^7.1.1" tslib "^2.6.0" -"@docusaurus/plugin-svgr@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-svgr/-/plugin-svgr-3.7.0.tgz#018e89efd615d5fde77b891a8c2aadf203013f5d" - integrity sha512-HByXIZTbc4GV5VAUkZ2DXtXv1Qdlnpk3IpuImwSnEzCDBkUMYcec5282hPjn6skZqB25M1TYCmWS91UbhBGxQg== +"@docusaurus/plugin-svgr@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-svgr/-/plugin-svgr-3.8.1.tgz#6f340be8eae418a2cce540d8ece096ffd9c9b6ab" + integrity sha512-rW0LWMDsdlsgowVwqiMb/7tANDodpy1wWPwCcamvhY7OECReN3feoFwLjd/U4tKjNY3encj0AJSTxJA+Fpe+Gw== dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" "@svgr/core" "8.1.0" "@svgr/webpack" "^8.1.0" tslib "^2.6.0" webpack "^5.88.1" -"@docusaurus/preset-classic@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-3.7.0.tgz#f6656a04ae6a4877523dbd04f7c491632e4003b9" - integrity sha512-nPHj8AxDLAaQXs+O6+BwILFuhiWbjfQWrdw2tifOClQoNfuXDjfjogee6zfx6NGHWqshR23LrcN115DmkHC91Q== - dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/plugin-content-blog" "3.7.0" - "@docusaurus/plugin-content-docs" "3.7.0" - "@docusaurus/plugin-content-pages" "3.7.0" - "@docusaurus/plugin-debug" "3.7.0" - "@docusaurus/plugin-google-analytics" "3.7.0" - "@docusaurus/plugin-google-gtag" "3.7.0" - "@docusaurus/plugin-google-tag-manager" "3.7.0" - "@docusaurus/plugin-sitemap" "3.7.0" - "@docusaurus/plugin-svgr" "3.7.0" - "@docusaurus/theme-classic" "3.7.0" - "@docusaurus/theme-common" "3.7.0" - "@docusaurus/theme-search-algolia" "3.7.0" - "@docusaurus/types" "3.7.0" - -"@docusaurus/theme-classic@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-3.7.0.tgz#b483bd8e2923b6994b5f47238884b9f8984222c5" - integrity sha512-MnLxG39WcvLCl4eUzHr0gNcpHQfWoGqzADCly54aqCofQX6UozOS9Th4RK3ARbM9m7zIRv3qbhggI53dQtx/hQ== - dependencies: - "@docusaurus/core" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/mdx-loader" "3.7.0" - "@docusaurus/module-type-aliases" "3.7.0" - "@docusaurus/plugin-content-blog" "3.7.0" - "@docusaurus/plugin-content-docs" "3.7.0" - "@docusaurus/plugin-content-pages" "3.7.0" - "@docusaurus/theme-common" "3.7.0" - "@docusaurus/theme-translations" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" +"@docusaurus/preset-classic@^3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-3.8.1.tgz#bb79fd12f3211363720c569a526c7e24d3aa966b" + integrity sha512-yJSjYNHXD8POMGc2mKQuj3ApPrN+eG0rO1UPgSx7jySpYU+n4WjBikbrA2ue5ad9A7aouEtMWUoiSRXTH/g7KQ== + dependencies: + "@docusaurus/core" "3.8.1" + "@docusaurus/plugin-content-blog" "3.8.1" + "@docusaurus/plugin-content-docs" "3.8.1" + "@docusaurus/plugin-content-pages" "3.8.1" + "@docusaurus/plugin-css-cascade-layers" "3.8.1" + "@docusaurus/plugin-debug" "3.8.1" + "@docusaurus/plugin-google-analytics" "3.8.1" + "@docusaurus/plugin-google-gtag" "3.8.1" + "@docusaurus/plugin-google-tag-manager" "3.8.1" + "@docusaurus/plugin-sitemap" "3.8.1" + "@docusaurus/plugin-svgr" "3.8.1" + "@docusaurus/theme-classic" "3.8.1" + "@docusaurus/theme-common" "3.8.1" + "@docusaurus/theme-search-algolia" "3.8.1" + "@docusaurus/types" "3.8.1" + +"@docusaurus/theme-classic@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-3.8.1.tgz#1e45c66d89ded359225fcd29bf3258d9205765c1" + integrity sha512-bqDUCNqXeYypMCsE1VcTXSI1QuO4KXfx8Cvl6rYfY0bhhqN6d2WZlRkyLg/p6pm+DzvanqHOyYlqdPyP0iz+iw== + dependencies: + "@docusaurus/core" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/mdx-loader" "3.8.1" + "@docusaurus/module-type-aliases" "3.8.1" + "@docusaurus/plugin-content-blog" "3.8.1" + "@docusaurus/plugin-content-docs" "3.8.1" + "@docusaurus/plugin-content-pages" "3.8.1" + "@docusaurus/theme-common" "3.8.1" + "@docusaurus/theme-translations" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" "@mdx-js/react" "^3.0.0" clsx "^2.0.0" copy-text-to-clipboard "^3.2.0" infima "0.2.0-alpha.45" lodash "^4.17.21" nprogress "^0.2.0" - postcss "^8.4.26" + postcss "^8.5.4" prism-react-renderer "^2.3.0" prismjs "^1.29.0" react-router-dom "^5.3.4" @@ -1744,15 +1767,15 @@ tslib "^2.6.0" utility-types "^3.10.0" -"@docusaurus/theme-common@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-3.7.0.tgz#18bf5c6b149a701f4bd865715ee8b595aa40b354" - integrity sha512-8eJ5X0y+gWDsURZnBfH0WabdNm8XMCXHv8ENy/3Z/oQKwaB/EHt5lP9VsTDTf36lKEp0V6DjzjFyFIB+CetL0A== +"@docusaurus/theme-common@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-3.8.1.tgz#17c23316fbe3ee3f7e707c7298cb59a0fff38b4b" + integrity sha512-UswMOyTnPEVRvN5Qzbo+l8k4xrd5fTFu2VPPfD6FcW/6qUtVLmJTQCktbAL3KJ0BVXGm5aJXz/ZrzqFuZERGPw== dependencies: - "@docusaurus/mdx-loader" "3.7.0" - "@docusaurus/module-type-aliases" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" + "@docusaurus/mdx-loader" "3.8.1" + "@docusaurus/module-type-aliases" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" "@types/history" "^4.7.11" "@types/react" "*" "@types/react-router-config" "*" @@ -1762,19 +1785,19 @@ tslib "^2.6.0" utility-types "^3.10.0" -"@docusaurus/theme-search-algolia@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.7.0.tgz#2108ddf0b300b82de7c2b9ff9fcf62121b66ea37" - integrity sha512-Al/j5OdzwRU1m3falm+sYy9AaB93S1XF1Lgk9Yc6amp80dNxJVplQdQTR4cYdzkGtuQqbzUA8+kaoYYO0RbK6g== - dependencies: - "@docsearch/react" "^3.8.1" - "@docusaurus/core" "3.7.0" - "@docusaurus/logger" "3.7.0" - "@docusaurus/plugin-content-docs" "3.7.0" - "@docusaurus/theme-common" "3.7.0" - "@docusaurus/theme-translations" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-validation" "3.7.0" +"@docusaurus/theme-search-algolia@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.8.1.tgz#3aa3d99c35cc2d4b709fcddd4df875a9b536e29b" + integrity sha512-NBFH5rZVQRAQM087aYSRKQ9yGEK9eHd+xOxQjqNpxMiV85OhJDD4ZGz6YJIod26Fbooy54UWVdzNU0TFeUUUzQ== + dependencies: + "@docsearch/react" "^3.9.0" + "@docusaurus/core" "3.8.1" + "@docusaurus/logger" "3.8.1" + "@docusaurus/plugin-content-docs" "3.8.1" + "@docusaurus/theme-common" "3.8.1" + "@docusaurus/theme-translations" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-validation" "3.8.1" algoliasearch "^5.17.1" algoliasearch-helper "^3.22.6" clsx "^2.0.0" @@ -1784,23 +1807,23 @@ tslib "^2.6.0" utility-types "^3.10.0" -"@docusaurus/theme-translations@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-translations/-/theme-translations-3.7.0.tgz#0891aedc7c7040afcb3a1b34051d3a69096d0d25" - integrity sha512-Ewq3bEraWDmienM6eaNK7fx+/lHMtGDHQyd1O+4+3EsDxxUmrzPkV7Ct3nBWTuE0MsoZr3yNwQVKjllzCMuU3g== +"@docusaurus/theme-translations@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/theme-translations/-/theme-translations-3.8.1.tgz#4b1d76973eb53861e167c7723485e059ba4ffd0a" + integrity sha512-OTp6eebuMcf2rJt4bqnvuwmm3NVXfzfYejL+u/Y1qwKhZPrjPoKWfk1CbOP5xH5ZOPkiAsx4dHdQBRJszK3z2g== dependencies: fs-extra "^11.1.1" tslib "^2.6.0" -"@docusaurus/tsconfig@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/tsconfig/-/tsconfig-3.7.0.tgz#654dcc524e25b8809af0f1b0b42485c18c047ab5" - integrity sha512-vRsyj3yUZCjscgfgcFYjIsTcAru/4h4YH2/XAE8Rs7wWdnng98PgWKvP5ovVc4rmRpRg2WChVW0uOy2xHDvDBQ== +"@docusaurus/tsconfig@^3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/tsconfig/-/tsconfig-3.8.1.tgz#a1f7daadfc93455289200647f4ee10cdca540f7b" + integrity sha512-XBWCcqhRHhkhfolnSolNL+N7gj3HVE3CoZVqnVjfsMzCoOsuQw2iCLxVVHtO+rePUUfouVZHURDgmqIySsF66A== -"@docusaurus/types@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-3.7.0.tgz#3f5a68a60f80ecdcb085666da1d68f019afda943" - integrity sha512-kOmZg5RRqJfH31m+6ZpnwVbkqMJrPOG5t0IOl4i/+3ruXyNfWzZ0lVtVrD0u4ONc/0NOsS9sWYaxxWNkH1LdLQ== +"@docusaurus/types@3.8.1", "@docusaurus/types@^3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-3.8.1.tgz#83ab66c345464e003b576a49f78897482061fc26" + integrity sha512-ZPdW5AB+pBjiVrcLuw3dOS6BFlrG0XkS2lDGsj8TizcnREQg3J8cjsgfDviszOk4CweNfwo1AEELJkYaMUuOPg== dependencies: "@mdx-js/mdx" "^3.0.0" "@types/history" "^4.7.11" @@ -1812,37 +1835,38 @@ webpack "^5.95.0" webpack-merge "^5.9.0" -"@docusaurus/utils-common@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/utils-common/-/utils-common-3.7.0.tgz#1bef52837d321db5dd2361fc07f3416193b5d029" - integrity sha512-IZeyIfCfXy0Mevj6bWNg7DG7B8G+S6o6JVpddikZtWyxJguiQ7JYr0SIZ0qWd8pGNuMyVwriWmbWqMnK7Y5PwA== +"@docusaurus/utils-common@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/utils-common/-/utils-common-3.8.1.tgz#c369b8c3041afb7dcd595d4172beb1cc1015c85f" + integrity sha512-zTZiDlvpvoJIrQEEd71c154DkcriBecm4z94OzEE9kz7ikS3J+iSlABhFXM45mZ0eN5pVqqr7cs60+ZlYLewtg== dependencies: - "@docusaurus/types" "3.7.0" + "@docusaurus/types" "3.8.1" tslib "^2.6.0" -"@docusaurus/utils-validation@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-3.7.0.tgz#dc0786fb633ae5cef8e93337bf21c2a826c7ecbd" - integrity sha512-w8eiKk8mRdN+bNfeZqC4nyFoxNyI1/VExMKAzD9tqpJfLLbsa46Wfn5wcKH761g9WkKh36RtFV49iL9lh1DYBA== +"@docusaurus/utils-validation@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-3.8.1.tgz#0499c0d151a4098a0963237057993282cfbd538e" + integrity sha512-gs5bXIccxzEbyVecvxg6upTwaUbfa0KMmTj7HhHzc016AGyxH2o73k1/aOD0IFrdCsfJNt37MqNI47s2MgRZMA== dependencies: - "@docusaurus/logger" "3.7.0" - "@docusaurus/utils" "3.7.0" - "@docusaurus/utils-common" "3.7.0" + "@docusaurus/logger" "3.8.1" + "@docusaurus/utils" "3.8.1" + "@docusaurus/utils-common" "3.8.1" fs-extra "^11.2.0" joi "^17.9.2" js-yaml "^4.1.0" lodash "^4.17.21" tslib "^2.6.0" -"@docusaurus/utils@3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-3.7.0.tgz#dfdebd63524c52b498f36b2907a3b2261930b9bb" - integrity sha512-e7zcB6TPnVzyUaHMJyLSArKa2AG3h9+4CfvKXKKWNx6hRs+p0a+u7HHTJBgo6KW2m+vqDnuIHK4X+bhmoghAFA== +"@docusaurus/utils@3.8.1": + version "3.8.1" + resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-3.8.1.tgz#2ac1e734106e2f73dbd0f6a8824d525f9064e9f0" + integrity sha512-P1ml0nvOmEFdmu0smSXOqTS1sxU5tqvnc0dA4MTKV39kye+bhQnjkIKEE18fNOvxjyB86k8esoCIFM3x4RykOQ== dependencies: - "@docusaurus/logger" "3.7.0" - "@docusaurus/types" "3.7.0" - "@docusaurus/utils-common" "3.7.0" + "@docusaurus/logger" "3.8.1" + "@docusaurus/types" "3.8.1" + "@docusaurus/utils-common" "3.8.1" escape-string-regexp "^4.0.0" + execa "5.1.1" file-loader "^6.2.0" fs-extra "^11.1.1" github-slugger "^1.5.0" @@ -1852,14 +1876,144 @@ js-yaml "^4.1.0" lodash "^4.17.21" micromatch "^4.0.5" + p-queue "^6.6.2" prompts "^2.4.2" resolve-pathname "^3.0.0" - shelljs "^0.8.5" tslib "^2.6.0" url-loader "^4.1.1" utility-types "^3.10.0" webpack "^5.88.1" +"@esbuild/aix-ppc64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.25.6.tgz#164b19122e2ed54f85469df9dea98ddb01d5e79e" + integrity sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw== + +"@esbuild/android-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.25.6.tgz#8f539e7def848f764f6432598e51cc3820fde3a5" + integrity sha512-hd5zdUarsK6strW+3Wxi5qWws+rJhCCbMiC9QZyzoxfk5uHRIE8T287giQxzVpEvCwuJ9Qjg6bEjcRJcgfLqoA== + +"@esbuild/android-arm@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.25.6.tgz#4ceb0f40113e9861169be83e2a670c260dd234ff" + integrity sha512-S8ToEOVfg++AU/bHwdksHNnyLyVM+eMVAOf6yRKFitnwnbwwPNqKr3srzFRe7nzV69RQKb5DgchIX5pt3L53xg== + +"@esbuild/android-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.25.6.tgz#ad4f280057622c25fe985c08999443a195dc63a8" + integrity sha512-0Z7KpHSr3VBIO9A/1wcT3NTy7EB4oNC4upJ5ye3R7taCc2GUdeynSLArnon5G8scPwaU866d3H4BCrE5xLW25A== + +"@esbuild/darwin-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.25.6.tgz#d1f04027396b3d6afc96bacd0d13167dfd9f01f7" + integrity sha512-FFCssz3XBavjxcFxKsGy2DYK5VSvJqa6y5HXljKzhRZ87LvEi13brPrf/wdyl/BbpbMKJNOr1Sd0jtW4Ge1pAA== + +"@esbuild/darwin-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.25.6.tgz#2b4a6cedb799f635758d7832d75b23772c8ef68f" + integrity sha512-GfXs5kry/TkGM2vKqK2oyiLFygJRqKVhawu3+DOCk7OxLy/6jYkWXhlHwOoTb0WqGnWGAS7sooxbZowy+pK9Yg== + +"@esbuild/freebsd-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.6.tgz#a26266cc97dd78dc3c3f3d6788b1b83697b1055d" + integrity sha512-aoLF2c3OvDn2XDTRvn8hN6DRzVVpDlj2B/F66clWd/FHLiHaG3aVZjxQX2DYphA5y/evbdGvC6Us13tvyt4pWg== + +"@esbuild/freebsd-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.25.6.tgz#9feb8e826735c568ebfd94859b22a3fbb6a9bdd2" + integrity sha512-2SkqTjTSo2dYi/jzFbU9Plt1vk0+nNg8YC8rOXXea+iA3hfNJWebKYPs3xnOUf9+ZWhKAaxnQNUf2X9LOpeiMQ== + +"@esbuild/linux-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.25.6.tgz#c07cbed8e249f4c28e7f32781d36fc4695293d28" + integrity sha512-b967hU0gqKd9Drsh/UuAm21Khpoh6mPBSgz8mKRq4P5mVK8bpA+hQzmm/ZwGVULSNBzKdZPQBRT3+WuVavcWsQ== + +"@esbuild/linux-arm@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.25.6.tgz#d6e2cd8ef3196468065d41f13fa2a61aaa72644a" + integrity sha512-SZHQlzvqv4Du5PrKE2faN0qlbsaW/3QQfUUc6yO2EjFcA83xnwm91UbEEVx4ApZ9Z5oG8Bxz4qPE+HFwtVcfyw== + +"@esbuild/linux-ia32@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.25.6.tgz#3e682bd47c4eddcc4b8f1393dfc8222482f17997" + integrity sha512-aHWdQ2AAltRkLPOsKdi3xv0mZ8fUGPdlKEjIEhxCPm5yKEThcUjHpWB1idN74lfXGnZ5SULQSgtr5Qos5B0bPw== + +"@esbuild/linux-loong64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.25.6.tgz#473f5ea2e52399c08ad4cd6b12e6dbcddd630f05" + integrity sha512-VgKCsHdXRSQ7E1+QXGdRPlQ/e08bN6WMQb27/TMfV+vPjjTImuT9PmLXupRlC90S1JeNNW5lzkAEO/McKeJ2yg== + +"@esbuild/linux-mips64el@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.25.6.tgz#9960631c9fd61605b0939c19043acf4ef2b51718" + integrity sha512-WViNlpivRKT9/py3kCmkHnn44GkGXVdXfdc4drNmRl15zVQ2+D2uFwdlGh6IuK5AAnGTo2qPB1Djppj+t78rzw== + +"@esbuild/linux-ppc64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.25.6.tgz#477cbf8bb04aa034b94f362c32c86b5c31db8d3e" + integrity sha512-wyYKZ9NTdmAMb5730I38lBqVu6cKl4ZfYXIs31Baf8aoOtB4xSGi3THmDYt4BTFHk7/EcVixkOV2uZfwU3Q2Jw== + +"@esbuild/linux-riscv64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.25.6.tgz#bcdb46c8fb8e93aa779e9a0a62cd4ac00dcac626" + integrity sha512-KZh7bAGGcrinEj4qzilJ4hqTY3Dg2U82c8bv+e1xqNqZCrCyc+TL9AUEn5WGKDzm3CfC5RODE/qc96OcbIe33w== + +"@esbuild/linux-s390x@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.25.6.tgz#f412cf5fdf0aea849ff51c73fd817c6c0234d46d" + integrity sha512-9N1LsTwAuE9oj6lHMyyAM+ucxGiVnEqUdp4v7IaMmrwb06ZTEVCIs3oPPplVsnjPfyjmxwHxHMF8b6vzUVAUGw== + +"@esbuild/linux-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.25.6.tgz#d8233c09b5ebc0c855712dc5eeb835a3a3341108" + integrity sha512-A6bJB41b4lKFWRKNrWoP2LHsjVzNiaurf7wyj/XtFNTsnPuxwEBWHLty+ZE0dWBKuSK1fvKgrKaNjBS7qbFKig== + +"@esbuild/netbsd-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.6.tgz#f51ae8dd1474172e73cf9cbaf8a38d1c72dd8f1a" + integrity sha512-IjA+DcwoVpjEvyxZddDqBY+uJ2Snc6duLpjmkXm/v4xuS3H+3FkLZlDm9ZsAbF9rsfP3zeA0/ArNDORZgrxR/Q== + +"@esbuild/netbsd-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.25.6.tgz#a267538602c0e50a858cf41dcfe5d8036f8da8e7" + integrity sha512-dUXuZr5WenIDlMHdMkvDc1FAu4xdWixTCRgP7RQLBOkkGgwuuzaGSYcOpW4jFxzpzL1ejb8yF620UxAqnBrR9g== + +"@esbuild/openbsd-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.6.tgz#a51be60c425b85c216479b8c344ad0511635f2d2" + integrity sha512-l8ZCvXP0tbTJ3iaqdNf3pjaOSd5ex/e6/omLIQCVBLmHTlfXW3zAxQ4fnDmPLOB1x9xrcSi/xtCWFwCZRIaEwg== + +"@esbuild/openbsd-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.25.6.tgz#7e4a743c73f75562e29223ba69d0be6c9c9008da" + integrity sha512-hKrmDa0aOFOr71KQ/19JC7az1P0GWtCN1t2ahYAf4O007DHZt/dW8ym5+CUdJhQ/qkZmI1HAF8KkJbEFtCL7gw== + +"@esbuild/openharmony-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.6.tgz#2087a5028f387879154ebf44bdedfafa17682e5b" + integrity sha512-+SqBcAWoB1fYKmpWoQP4pGtx+pUUC//RNYhFdbcSA16617cchuryuhOCRpPsjCblKukAckWsV+aQ3UKT/RMPcA== + +"@esbuild/sunos-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.25.6.tgz#56531f861723ea0dc6283a2bb8837304223cb736" + integrity sha512-dyCGxv1/Br7MiSC42qinGL8KkG4kX0pEsdb0+TKhmJZgCUDBGmyo1/ArCjNGiOLiIAgdbWgmWgib4HoCi5t7kA== + +"@esbuild/win32-arm64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.25.6.tgz#f4989f033deac6fae323acff58764fa8bc01436e" + integrity sha512-42QOgcZeZOvXfsCBJF5Afw73t4veOId//XD3i+/9gSkhSV6Gk3VPlWncctI+JcOyERv85FUo7RxuxGy+z8A43Q== + +"@esbuild/win32-ia32@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.25.6.tgz#b260e9df71e3939eb33925076d39f63cec7d1525" + integrity sha512-4AWhgXmDuYN7rJI6ORB+uU9DHLq/erBbuMoAuB4VWJTu5KtCgcKYPynF0YI1VkBNuEfjNlLrFr9KZPJzrtLkrQ== + +"@esbuild/win32-x64@0.25.6": + version "0.25.6" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.25.6.tgz#4276edd5c105bc28b11c6a1f76fb9d29d1bd25c1" + integrity sha512-NgJPHHbEpLQgDH2MjQu90pzW/5vvXIZ7KOnPyNBm92A6WgZ/7b6fJyUBjoumLqeOQQGqY2QjQxRo97ah4Sj0cA== + "@hapi/hoek@^9.0.0", "@hapi/hoek@^9.3.0": version "9.3.0" resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" @@ -2395,7 +2549,7 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/json-schema@*", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": +"@types/json-schema@*", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== @@ -2441,11 +2595,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190" integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw== -"@types/parse-json@^4.0.0": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.2.tgz#5950e50960793055845e956c427fc2b0d70c5239" - integrity sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw== - "@types/parse5@^5.0.0": version "5.0.3" resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.3.tgz#e7b5aebbac150f8b5fdd4a46e7f0bd8e65e19109" @@ -2737,7 +2886,7 @@ acorn@^8.0.0, acorn@^8.0.4, acorn@^8.11.0, acorn@^8.14.0, acorn@^8.8.2: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.14.0.tgz#063e2c70cac5fb4f6467f0b11152e04c682795b0" integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA== -address@^1.0.1, address@^1.1.2: +address@^1.0.1: version "1.2.2" resolved "https://registry.yarnpkg.com/address/-/address-1.2.2.tgz#2b5248dac5485a6390532c6a517fda2e3faac89e" integrity sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA== @@ -2757,7 +2906,7 @@ ajv-formats@^2.1.1: dependencies: ajv "^8.0.0" -ajv-keywords@^3.4.1, ajv-keywords@^3.5.2: +ajv-keywords@^3.5.2: version "3.5.2" resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== @@ -2769,7 +2918,7 @@ ajv-keywords@^5.1.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^6.12.2, ajv@^6.12.5: +ajv@^6.12.5: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -2906,11 +3055,6 @@ astring@^1.8.0: resolved "https://registry.yarnpkg.com/astring/-/astring-1.9.0.tgz#cc73e6062a7eb03e7d19c22d8b0b3451fd9bfeef" integrity sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg== -at-least-node@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" - integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== - autocomplete.js@^0.37.1: version "0.37.1" resolved "https://registry.yarnpkg.com/autocomplete.js/-/autocomplete.js-0.37.1.tgz#a29a048d827e7d2bf8f7df8b831766e5cc97df01" @@ -2930,6 +3074,18 @@ autoprefixer@^10.4.19, autoprefixer@^10.4.20: picocolors "^1.0.1" postcss-value-parser "^4.2.0" +autoprefixer@^10.4.21: + version "10.4.21" + resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-10.4.21.tgz#77189468e7a8ad1d9a37fbc08efc9f480cf0a95d" + integrity sha512-O+A6LWV5LDHSJD3LjHYoNi4VLsj/Whi7k6zG12xTYaU4cQ8oxQGckXNX8cRHK5yOZ/ppVHe0ZBXGzSV9jXdVbQ== + dependencies: + browserslist "^4.24.4" + caniuse-lite "^1.0.30001702" + fraction.js "^4.3.7" + normalize-range "^0.1.2" + picocolors "^1.1.1" + postcss-value-parser "^4.2.0" + babel-loader@^9.2.1: version "9.2.1" resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-9.2.1.tgz#04c7835db16c246dd19ba0914418f3937797587b" @@ -3085,7 +3241,7 @@ braces@^3.0.3, braces@~3.0.2: dependencies: fill-range "^7.1.1" -browserslist@^4.0.0, browserslist@^4.18.1, browserslist@^4.23.0, browserslist@^4.23.1, browserslist@^4.23.3, browserslist@^4.24.0, browserslist@^4.24.3: +browserslist@^4.0.0, browserslist@^4.23.0, browserslist@^4.23.3, browserslist@^4.24.0, browserslist@^4.24.3: version "4.24.4" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.24.4.tgz#c6b2865a3f08bcb860a0e827389003b9fe686e4b" integrity sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A== @@ -3095,6 +3251,16 @@ browserslist@^4.0.0, browserslist@^4.18.1, browserslist@^4.23.0, browserslist@^4 node-releases "^2.0.19" update-browserslist-db "^1.1.1" +browserslist@^4.24.4, browserslist@^4.25.0: + version "4.25.1" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.25.1.tgz#ba9e8e6f298a1d86f829c9b975e07948967bb111" + integrity sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw== + dependencies: + caniuse-lite "^1.0.30001726" + electron-to-chromium "^1.5.173" + node-releases "^2.0.19" + update-browserslist-db "^1.1.3" + buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" @@ -3192,17 +3358,17 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001646, caniuse-lite@^1.0.30001688: - version "1.0.30001695" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001695.tgz#39dfedd8f94851132795fdf9b79d29659ad9c4d4" - integrity sha512-vHyLade6wTgI2u1ec3WQBxv+2BrTERV28UXQu9LO6lZ9pYeMk34vjXFLOxo1A4UBA8XTL4njRQZdno/yYaSmWw== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001646, caniuse-lite@^1.0.30001688, caniuse-lite@^1.0.30001702, caniuse-lite@^1.0.30001726: + version "1.0.30001727" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001727.tgz" + integrity sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q== ccount@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/ccount/-/ccount-2.0.1.tgz#17a3bf82302e0870d6da43a01311a8bc02a3ecf5" integrity sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg== -chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: +chalk@^4.0.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -3265,7 +3431,7 @@ cheerio@1.0.0-rc.12: parse5 "^7.0.0" parse5-htmlparser2-tree-adapter "^7.0.0" -chokidar@^3.4.2, chokidar@^3.5.3, chokidar@^3.6.0: +chokidar@^3.5.3, chokidar@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== @@ -3297,7 +3463,7 @@ class-variance-authority@^0.7.1: dependencies: clsx "^2.1.1" -clean-css@^5.2.2, clean-css@^5.3.2, clean-css@~5.3.2: +clean-css@^5.2.2, clean-css@^5.3.3, clean-css@~5.3.2: version "5.3.3" resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-5.3.3.tgz#b330653cd3bd6b75009cc25c714cae7b93351ccd" integrity sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg== @@ -3549,17 +3715,6 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== -cosmiconfig@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982" - integrity sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg== - dependencies: - "@types/parse-json" "^4.0.0" - import-fresh "^3.1.0" - parse-json "^5.0.0" - path-type "^4.0.0" - yaml "^1.7.2" - cosmiconfig@^8.1.3, cosmiconfig@^8.3.5: version "8.3.6" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.3.6.tgz#060a2b871d66dba6c8538ea1118ba1ac16f5fae3" @@ -3607,7 +3762,7 @@ css-has-pseudo@^7.0.2: postcss-selector-parser "^7.0.0" postcss-value-parser "^4.2.0" -css-loader@^6.8.1: +css-loader@^6.11.0: version "6.11.0" resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-6.11.0.tgz#33bae3bf6363d0a7c2cf9031c96c744ff54d85ba" integrity sha512-CTJ+AEQJjq5NzLga5pE39qdiSV56F8ywCIsqNIRF0r7BDgWsN25aazToqAFg7ZrtA/U016xudB3ffgweORxX7g== @@ -3686,10 +3841,10 @@ css-what@^6.0.1, css-what@^6.1.0: resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== -cssdb@^8.2.3: - version "8.2.3" - resolved "https://registry.yarnpkg.com/cssdb/-/cssdb-8.2.3.tgz#7e6980bb5a785a9b4eb2a21bd38d50624b56cb46" - integrity sha512-9BDG5XmJrJQQnJ51VFxXCAtpZ5ebDlAREmO8sxMOVU0aSxN/gocbctjIG5LMh3WBUq+xTlb/jw2LoljBEqraTA== +cssdb@^8.3.0: + version "8.3.1" + resolved "https://registry.yarnpkg.com/cssdb/-/cssdb-8.3.1.tgz#0ac96395b7092ffee14563e948cf43c2019b051e" + integrity sha512-XnDRQMXucLueX92yDe0LPKupXetWoFOgawr4O4X41l5TltgK2NVbJJVDnnOywDYfW1sTJ28AcXGKOqdRKwCcmQ== cssesc@^3.0.0: version "3.0.0" @@ -3775,7 +3930,7 @@ debounce@^1.2.1: resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.1.tgz#38881d8f4166a5c5848020c11827b834bcb3e0a5" integrity sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug== -debug@2.6.9, debug@^2.6.0: +debug@2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -3808,7 +3963,7 @@ deep-extend@^0.6.0: resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== -deepmerge@^4.2.2, deepmerge@^4.3.1: +deepmerge@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== @@ -3848,20 +4003,6 @@ define-properties@^1.2.1: has-property-descriptors "^1.0.0" object-keys "^1.1.1" -del@^6.1.1: - version "6.1.1" - resolved "https://registry.yarnpkg.com/del/-/del-6.1.1.tgz#3b70314f1ec0aa325c6b14eb36b95786671edb7a" - integrity sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg== - dependencies: - globby "^11.0.1" - graceful-fs "^4.2.4" - is-glob "^4.0.1" - is-path-cwd "^2.2.0" - is-path-inside "^3.0.2" - p-map "^4.0.0" - rimraf "^3.0.2" - slash "^3.0.0" - depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -3887,14 +4028,6 @@ detect-node@^2.0.4: resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1" integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== -detect-port-alt@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/detect-port-alt/-/detect-port-alt-1.1.6.tgz#24707deabe932d4a3cf621302027c2b266568275" - integrity sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q== - dependencies: - address "^1.0.1" - debug "^2.6.0" - detect-port@^1.5.1: version "1.6.1" resolved "https://registry.yarnpkg.com/detect-port/-/detect-port-1.6.1.tgz#45e4073997c5f292b957cb678fb0bb8ed4250a67" @@ -4068,6 +4201,11 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== +electron-to-chromium@^1.5.173: + version "1.5.180" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.180.tgz#3e4f6e7494d6371e014af176dfdfd43c8a4b56df" + integrity sha512-ED+GEyEh3kYMwt2faNmgMB0b8O5qtATGgR4RmRsIp4T6p7B8vdMbIedYndnvZfsaXvSzegtpfqRMDNCjjiSduA== + electron-to-chromium@^1.5.73: version "1.5.84" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.84.tgz#8e334ca206bb293a20b16418bf454783365b0a95" @@ -4175,6 +4313,38 @@ esast-util-from-js@^2.0.0: esast-util-from-estree "^2.0.0" vfile-message "^4.0.0" +esbuild@~0.25.0: + version "0.25.6" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.25.6.tgz#9b82a3db2fa131aec069ab040fd57ed0a880cdcd" + integrity sha512-GVuzuUwtdsghE3ocJ9Bs8PNoF13HNQ5TXbEi2AhvVb8xU1Iwt9Fos9FEamfoee+u/TOsn7GUWc04lz46n2bbTg== + optionalDependencies: + "@esbuild/aix-ppc64" "0.25.6" + "@esbuild/android-arm" "0.25.6" + "@esbuild/android-arm64" "0.25.6" + "@esbuild/android-x64" "0.25.6" + "@esbuild/darwin-arm64" "0.25.6" + "@esbuild/darwin-x64" "0.25.6" + "@esbuild/freebsd-arm64" "0.25.6" + "@esbuild/freebsd-x64" "0.25.6" + "@esbuild/linux-arm" "0.25.6" + "@esbuild/linux-arm64" "0.25.6" + "@esbuild/linux-ia32" "0.25.6" + "@esbuild/linux-loong64" "0.25.6" + "@esbuild/linux-mips64el" "0.25.6" + "@esbuild/linux-ppc64" "0.25.6" + "@esbuild/linux-riscv64" "0.25.6" + "@esbuild/linux-s390x" "0.25.6" + "@esbuild/linux-x64" "0.25.6" + "@esbuild/netbsd-arm64" "0.25.6" + "@esbuild/netbsd-x64" "0.25.6" + "@esbuild/openbsd-arm64" "0.25.6" + "@esbuild/openbsd-x64" "0.25.6" + "@esbuild/openharmony-arm64" "0.25.6" + "@esbuild/sunos-x64" "0.25.6" + "@esbuild/win32-arm64" "0.25.6" + "@esbuild/win32-ia32" "0.25.6" + "@esbuild/win32-x64" "0.25.6" + escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" @@ -4319,7 +4489,7 @@ eval@^0.1.8: "@types/node" "*" require-like ">= 0.1.1" -eventemitter3@^4.0.0: +eventemitter3@^4.0.0, eventemitter3@^4.0.4: version "4.0.7" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== @@ -4329,7 +4499,7 @@ events@^3.2.0: resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== -execa@^5.0.0: +execa@5.1.1, execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== @@ -4462,11 +4632,6 @@ file-loader@^6.2.0: loader-utils "^2.0.0" schema-utils "^3.0.0" -filesize@^8.0.6: - version "8.0.7" - resolved "https://registry.yarnpkg.com/filesize/-/filesize-8.0.7.tgz#695e70d80f4e47012c132d57a059e80c6b580bd8" - integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ== - fill-range@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" @@ -4495,21 +4660,6 @@ find-cache-dir@^4.0.0: common-path-prefix "^3.0.0" pkg-dir "^7.0.0" -find-up@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" - integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg== - dependencies: - locate-path "^3.0.0" - -find-up@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" - integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== - dependencies: - locate-path "^6.0.0" - path-exists "^4.0.0" - find-up@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-6.3.0.tgz#2abab3d3280b2dc7ac10199ef324c4e002c8c790" @@ -4536,25 +4686,6 @@ foreground-child@^3.1.0: cross-spawn "^7.0.0" signal-exit "^4.0.1" -fork-ts-checker-webpack-plugin@^6.5.0: - version "6.5.3" - resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-6.5.3.tgz#eda2eff6e22476a2688d10661688c47f611b37f3" - integrity sha512-SbH/l9ikmMWycd5puHJKTkZJKddF4iRLyW3DeZ08HTI7NGyLS38MXd/KGgeWumQO7YNQbW2u/NtPT2YowbPaGQ== - dependencies: - "@babel/code-frame" "^7.8.3" - "@types/json-schema" "^7.0.5" - chalk "^4.1.0" - chokidar "^3.4.2" - cosmiconfig "^6.0.0" - deepmerge "^4.2.2" - fs-extra "^9.0.0" - glob "^7.1.6" - memfs "^3.1.2" - minimatch "^3.0.4" - schema-utils "2.7.0" - semver "^7.3.2" - tapable "^1.0.0" - form-data-encoder@^2.1.2: version "2.1.4" resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-2.1.4.tgz#261ea35d2a70d48d30ec7a9603130fa5515e9cd5" @@ -4589,16 +4720,6 @@ fs-extra@^11.1.1, fs-extra@^11.2.0: jsonfile "^6.0.1" universalify "^2.0.0" -fs-extra@^9.0.0: - version "9.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" - integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== - dependencies: - at-least-node "^1.0.0" - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs-monkey@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.6.tgz#8ead082953e88d992cf3ff844faa907b26756da2" @@ -4609,7 +4730,7 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@~2.3.2: +fsevents@~2.3.2, fsevents@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== @@ -4673,6 +4794,13 @@ get-stream@^6.0.0, get-stream@^6.0.1: resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== +get-tsconfig@^4.7.5: + version "4.10.1" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.10.1.tgz#d34c1c01f47d65a606c37aa7a177bc3e56ab4b2e" + integrity sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ== + dependencies: + resolve-pkg-maps "^1.0.0" + github-slugger@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/github-slugger/-/github-slugger-1.5.0.tgz#17891bbc73232051474d68bd867a34625c955f7d" @@ -4709,7 +4837,7 @@ glob@^10.3.10: package-json-from-dist "^1.0.0" path-scurry "^1.11.1" -glob@^7.0.0, glob@^7.1.3, glob@^7.1.6: +glob@^7.1.3: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -4728,28 +4856,12 @@ global-dirs@^3.0.0: dependencies: ini "2.0.0" -global-modules@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" - integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== - dependencies: - global-prefix "^3.0.0" - -global-prefix@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-3.0.0.tgz#fc85f73064df69f50421f47f883fe5b913ba9b97" - integrity sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg== - dependencies: - ini "^1.3.5" - kind-of "^6.0.2" - which "^1.3.1" - globals@^11.1.0: version "11.12.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== -globby@^11.0.1, globby@^11.0.4, globby@^11.1.0: +globby@^11.1.0: version "11.1.0" resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== @@ -5256,24 +5368,17 @@ ignore@^5.2.0, ignore@^5.2.4: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== -image-size@^1.0.2: - version "1.2.0" - resolved "https://registry.yarnpkg.com/image-size/-/image-size-1.2.0.tgz#312af27a2ff4ff58595ad00b9344dd684c910df6" - integrity sha512-4S8fwbO6w3GeCVN6OPtA9I5IGKkcDMPcKndtUlpJuCwu7JLjtj7JZpwqLuyY2nrmQT3AWsCJLSKPsc2mPBSl3w== - dependencies: - queue "6.0.2" +image-size@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/image-size/-/image-size-2.0.2.tgz#84a7b43704db5736f364bf0d1b029821299b4bdc" + integrity sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w== immediate@^3.2.3: version "3.3.0" resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.3.0.tgz#1aef225517836bcdf7f2a2de2600c79ff0269266" integrity sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q== -immer@^9.0.7: - version "9.0.21" - resolved "https://registry.yarnpkg.com/immer/-/immer-9.0.21.tgz#1e025ea31a40f24fb064f1fef23e931496330176" - integrity sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA== - -import-fresh@^3.1.0, import-fresh@^3.3.0: +import-fresh@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -5324,7 +5429,7 @@ ini@2.0.0: resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== -ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: +ini@^1.3.4, ini@~1.3.0: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== @@ -5334,11 +5439,6 @@ inline-style-parser@0.2.4: resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.2.4.tgz#f4af5fe72e612839fcd453d989a586566d695f22" integrity sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q== -interpret@^1.0.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" - integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== - invariant@^2.2.4: version "2.2.4" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" @@ -5465,11 +5565,6 @@ is-obj@^2.0.0: resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982" integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w== -is-path-cwd@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz#67d43b82664a7b5191fd9119127eb300048a9fdb" - integrity sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ== - is-path-inside@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" @@ -5502,11 +5597,6 @@ is-regexp@^1.0.0: resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" integrity sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA== -is-root@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-root/-/is-root-2.1.0.tgz#809e18129cf1129644302a4f8544035d51984a9c" - integrity sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg== - is-stream@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" @@ -5730,26 +5820,6 @@ loader-utils@^2.0.0: emojis-list "^3.0.0" json5 "^2.1.2" -loader-utils@^3.2.0: - version "3.3.1" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-3.3.1.tgz#735b9a19fd63648ca7adbd31c2327dfe281304e5" - integrity sha512-FMJTLMXfCLMLfJxcX9PFqX5qD88Z5MRGaZCVzfuqeZSPsyiBzs+pahDQjbIWz2QIzPZz0NX9Zy4FX3lmK6YHIg== - -locate-path@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e" - integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A== - dependencies: - p-locate "^3.0.0" - path-exists "^3.0.0" - -locate-path@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" - integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== - dependencies: - p-locate "^5.0.0" - locate-path@^7.1.0: version "7.2.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-7.2.0.tgz#69cb1779bd90b35ab1e771e1f2f89a202c2a8a8a" @@ -6107,7 +6177,7 @@ medium-zoom@^1.0.8: resolved "https://registry.yarnpkg.com/medium-zoom/-/medium-zoom-1.1.0.tgz#6efb6bbda861a02064ee71a2617a8dc4381ecc71" integrity sha512-ewyDsp7k4InCUp3jRmwHBRFGyjBimKps/AJLjRSox+2q/2H4p/PNpQf+pwONWlJiOudkBXtbdmVbFjqyybfTmQ== -memfs@^3.1.2, memfs@^3.4.3: +memfs@^3.4.3: version "3.6.0" resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.6.0.tgz#d7a2110f86f79dd950a8b6df6d57bc984aa185f6" integrity sha512-EGowvkkgbMcIChjMTMkESFDbZeSh8xZ7kNSF0hAiAN4Jh6jgHCRS0Ga/+C8y6Au+oqpezRHCfPsmJ2+DwAgiwQ== @@ -6609,7 +6679,7 @@ mimic-response@^4.0.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-4.0.0.tgz#35468b19e7c75d10f5165ea25e75a5ceea7cf70f" integrity sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg== -mini-css-extract-plugin@^2.9.1: +mini-css-extract-plugin@^2.9.2: version "2.9.2" resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-2.9.2.tgz#966031b468917a5446f4c24a80854b2947503c5b" integrity sha512-GJuACcS//jtq4kCtd5ii/M0SZf7OZRH+BxdqXZHaJfb8TJiVl+NgQRPwiYt2EuqeSkNydn/7vP+bcE27C5mb9w== @@ -6622,7 +6692,7 @@ minimalistic-assert@^1.0.0: resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz#2e194de044626d4a10e7f7fbc00ce73e83e4d5c7" integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== -minimatch@3.1.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: +minimatch@3.1.2, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -6683,6 +6753,11 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" +nanoid@^3.3.11: + version "3.3.11" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.11.tgz#4f4f112cefbe303202f2199838128936266d185b" + integrity sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w== + nanoid@^3.3.8: version "3.3.8" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" @@ -6867,19 +6942,10 @@ p-cancelable@^3.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-3.0.0.tgz#63826694b54d61ca1c20ebcb6d3ecf5e14cd8050" integrity sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw== -p-limit@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" - integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== - dependencies: - p-try "^2.0.0" - -p-limit@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" - integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== - dependencies: - yocto-queue "^0.1.0" +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== p-limit@^4.0.0: version "4.0.0" @@ -6888,20 +6954,6 @@ p-limit@^4.0.0: dependencies: yocto-queue "^1.0.0" -p-locate@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" - integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ== - dependencies: - p-limit "^2.0.0" - -p-locate@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" - integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== - dependencies: - p-limit "^3.0.2" - p-locate@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-6.0.0.tgz#3da9a49d4934b901089dca3302fa65dc5a05c04f" @@ -6916,6 +6968,14 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" +p-queue@^6.6.2: + version "6.6.2" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" + integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== + dependencies: + eventemitter3 "^4.0.4" + p-timeout "^3.2.0" + p-retry@^4.5.0: version "4.6.2" resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16" @@ -6924,10 +6984,12 @@ p-retry@^4.5.0: "@types/retry" "0.12.0" retry "^0.13.1" -p-try@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" - integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== +p-timeout@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" package-json-from-dist@^1.0.0: version "1.0.1" @@ -6972,7 +7034,7 @@ parse-entities@^4.0.0: is-decimal "^2.0.0" is-hexadecimal "^2.0.0" -parse-json@^5.0.0, parse-json@^5.2.0: +parse-json@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== @@ -7020,16 +7082,6 @@ pascal-case@^3.1.2: no-case "^3.0.4" tslib "^2.0.3" -path-exists@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" - integrity sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ== - -path-exists@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" - integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== - path-exists@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-5.0.0.tgz#a6aad9489200b21fab31e49cf09277e5116fb9e7" @@ -7124,13 +7176,6 @@ pkg-dir@^7.0.0: dependencies: find-up "^6.3.0" -pkg-up@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" - integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA== - dependencies: - find-up "^3.0.0" - postcss-attribute-case-insensitive@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-7.0.1.tgz#0c4500e3bcb2141848e89382c05b5a31c23033a3" @@ -7153,15 +7198,15 @@ postcss-clamp@^4.1.0: dependencies: postcss-value-parser "^4.2.0" -postcss-color-functional-notation@^7.0.7: - version "7.0.7" - resolved "https://registry.yarnpkg.com/postcss-color-functional-notation/-/postcss-color-functional-notation-7.0.7.tgz#c5362df010926f902ce4e7fb3da2a46cff175d1b" - integrity sha512-EZvAHsvyASX63vXnyXOIynkxhaHRSsdb7z6yiXKIovGXAolW4cMZ3qoh7k3VdTsLBS6VGdksGfIo3r6+waLoOw== +postcss-color-functional-notation@^7.0.10: + version "7.0.10" + resolved "https://registry.yarnpkg.com/postcss-color-functional-notation/-/postcss-color-functional-notation-7.0.10.tgz#f1e9c3e4371889dcdfeabfa8515464fd8338cedc" + integrity sha512-k9qX+aXHBiLTRrWoCJuUFI6F1iF6QJQUXNVWJVSbqZgj57jDhBlOvD8gNUGl35tgqDivbGLhZeW3Ongz4feuKA== dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" postcss-color-hex-alpha@^10.0.0: @@ -7198,35 +7243,35 @@ postcss-convert-values@^6.1.0: browserslist "^4.23.0" postcss-value-parser "^4.2.0" -postcss-custom-media@^11.0.5: - version "11.0.5" - resolved "https://registry.yarnpkg.com/postcss-custom-media/-/postcss-custom-media-11.0.5.tgz#2fcd88a9b1d4da41c67dac6f2def903063a3377d" - integrity sha512-SQHhayVNgDvSAdX9NQ/ygcDQGEY+aSF4b/96z7QUX6mqL5yl/JgG/DywcF6fW9XbnCRE+aVYk+9/nqGuzOPWeQ== - dependencies: - "@csstools/cascade-layer-name-parser" "^2.0.4" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/media-query-list-parser" "^4.0.2" - -postcss-custom-properties@^14.0.4: - version "14.0.4" - resolved "https://registry.yarnpkg.com/postcss-custom-properties/-/postcss-custom-properties-14.0.4.tgz#de9c663285a98833a946d7003a34369d3ce373a9" - integrity sha512-QnW8FCCK6q+4ierwjnmXF9Y9KF8q0JkbgVfvQEMa93x1GT8FvOiUevWCN2YLaOWyByeDX8S6VFbZEeWoAoXs2A== - dependencies: - "@csstools/cascade-layer-name-parser" "^2.0.4" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" +postcss-custom-media@^11.0.6: + version "11.0.6" + resolved "https://registry.yarnpkg.com/postcss-custom-media/-/postcss-custom-media-11.0.6.tgz#6b450e5bfa209efb736830066682e6567bd04967" + integrity sha512-C4lD4b7mUIw+RZhtY7qUbf4eADmb7Ey8BFA2px9jUbwg7pjTZDl4KY4bvlUV+/vXQvzQRfiGEVJyAbtOsCMInw== + dependencies: + "@csstools/cascade-layer-name-parser" "^2.0.5" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/media-query-list-parser" "^4.0.3" + +postcss-custom-properties@^14.0.6: + version "14.0.6" + resolved "https://registry.yarnpkg.com/postcss-custom-properties/-/postcss-custom-properties-14.0.6.tgz#1af73a650bf115ba052cf915287c9982825fc90e" + integrity sha512-fTYSp3xuk4BUeVhxCSJdIPhDLpJfNakZKoiTDx7yRGCdlZrSJR7mWKVOBS4sBF+5poPQFMj2YdXx1VHItBGihQ== + dependencies: + "@csstools/cascade-layer-name-parser" "^2.0.5" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" "@csstools/utilities" "^2.0.0" postcss-value-parser "^4.2.0" -postcss-custom-selectors@^8.0.4: - version "8.0.4" - resolved "https://registry.yarnpkg.com/postcss-custom-selectors/-/postcss-custom-selectors-8.0.4.tgz#95ef8268fdbbbd84f34cf84a4517c9d99d419c5a" - integrity sha512-ASOXqNvDCE0dAJ/5qixxPeL1aOVGHGW2JwSy7HyjWNbnWTQCl+fDc968HY1jCmZI0+BaYT5CxsOiUhavpG/7eg== +postcss-custom-selectors@^8.0.5: + version "8.0.5" + resolved "https://registry.yarnpkg.com/postcss-custom-selectors/-/postcss-custom-selectors-8.0.5.tgz#9448ed37a12271d7ab6cb364b6f76a46a4a323e8" + integrity sha512-9PGmckHQswiB2usSO6XMSswO2yFWVoCAuih1yl9FVcwkscLjRKjwsjM3t+NIWpSU2Jx3eOiK2+t4vVTQaoCHHg== dependencies: - "@csstools/cascade-layer-name-parser" "^2.0.4" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" + "@csstools/cascade-layer-name-parser" "^2.0.5" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" postcss-selector-parser "^7.0.0" postcss-dir-pseudo-class@^9.0.1: @@ -7263,12 +7308,12 @@ postcss-discard-unused@^6.0.5: dependencies: postcss-selector-parser "^6.0.16" -postcss-double-position-gradients@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/postcss-double-position-gradients/-/postcss-double-position-gradients-6.0.0.tgz#eddd424ec754bb543d057d4d2180b1848095d4d2" - integrity sha512-JkIGah3RVbdSEIrcobqj4Gzq0h53GG4uqDPsho88SgY84WnpkTpI0k50MFK/sX7XqVisZ6OqUfFnoUO6m1WWdg== +postcss-double-position-gradients@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/postcss-double-position-gradients/-/postcss-double-position-gradients-6.0.2.tgz#185f8eab2db9cf4e34be69b5706c905895bb52ae" + integrity sha512-7qTqnL7nfLRyJK/AHSVrrXOuvDDzettC+wGoienURV8v2svNbu6zJC52ruZtHaO6mfcagFmuTGFdzRsJKB3k5Q== dependencies: - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" postcss-value-parser "^4.2.0" @@ -7320,15 +7365,15 @@ postcss-js@^4.0.1: dependencies: camelcase-css "^2.0.1" -postcss-lab-function@^7.0.7: - version "7.0.7" - resolved "https://registry.yarnpkg.com/postcss-lab-function/-/postcss-lab-function-7.0.7.tgz#9c87c21ce5132c55824190b75d7d7adede9c2fac" - integrity sha512-+ONj2bpOQfsCKZE2T9VGMyVVdGcGUpr7u3SVfvkJlvhTRmDCfY25k4Jc8fubB9DclAPR4+w8uVtDZmdRgdAHig== +postcss-lab-function@^7.0.10: + version "7.0.10" + resolved "https://registry.yarnpkg.com/postcss-lab-function/-/postcss-lab-function-7.0.10.tgz#0537bd7245b935fc133298c8896bcbd160540cae" + integrity sha512-tqs6TCEv9tC1Riq6fOzHuHcZyhg4k3gIAMB8GGY/zA1ssGdm6puHMVE7t75aOSoFg7UD2wyrFFhbldiCMyyFTQ== dependencies: - "@csstools/css-color-parser" "^3.0.7" - "@csstools/css-parser-algorithms" "^3.0.4" - "@csstools/css-tokenizer" "^3.0.3" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" + "@csstools/css-color-parser" "^3.0.10" + "@csstools/css-parser-algorithms" "^3.0.5" + "@csstools/css-tokenizer" "^3.0.4" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" "@csstools/utilities" "^2.0.0" postcss-load-config@^4.0.2: @@ -7339,7 +7384,7 @@ postcss-load-config@^4.0.2: lilconfig "^3.0.0" yaml "^2.3.4" -postcss-loader@^7.3.3: +postcss-loader@^7.3.4: version "7.3.4" resolved "https://registry.yarnpkg.com/postcss-loader/-/postcss-loader-7.3.4.tgz#aed9b79ce4ed7e9e89e56199d25ad1ec8f606209" integrity sha512-iW5WTTBSC5BfsBJ9daFMPVrLT36MrNiC6fqOZTTaHjBNX6Pfd5p+hSBqe/fEeNd7pc13QiAyGt7VdGMw4eRC4A== @@ -7348,10 +7393,10 @@ postcss-loader@^7.3.3: jiti "^1.20.0" semver "^7.5.4" -postcss-logical@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/postcss-logical/-/postcss-logical-8.0.0.tgz#0db0b90c2dc53b485a8074a4b7a906297544f58d" - integrity sha512-HpIdsdieClTjXLOyYdUPAX/XQASNIwdKt5hoZW08ZOAiI+tbV0ta1oclkpVkW5ANU+xJvk3KkA0FejkjGLXUkg== +postcss-logical@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/postcss-logical/-/postcss-logical-8.1.0.tgz#4092b16b49e3ecda70c4d8945257da403d167228" + integrity sha512-pL1hXFQ2fEXNKiNiAgtfA005T9FBxky5zkX6s4GZM2D8RkVgRqz3f4g1JUoq925zXv495qk8UNldDwh8uGEDoA== dependencies: postcss-value-parser "^4.2.0" @@ -7448,12 +7493,12 @@ postcss-nested@^6.2.0: dependencies: postcss-selector-parser "^6.1.1" -postcss-nesting@^13.0.1: - version "13.0.1" - resolved "https://registry.yarnpkg.com/postcss-nesting/-/postcss-nesting-13.0.1.tgz#c405796d7245a3e4c267a9956cacfe9670b5d43e" - integrity sha512-VbqqHkOBOt4Uu3G8Dm8n6lU5+9cJFxiuty9+4rcoyRPO9zZS1JIs6td49VIoix3qYqELHlJIn46Oih9SAKo+yQ== +postcss-nesting@^13.0.2: + version "13.0.2" + resolved "https://registry.yarnpkg.com/postcss-nesting/-/postcss-nesting-13.0.2.tgz#fde0d4df772b76d03b52eccc84372e8d1ca1402e" + integrity sha512-1YCI290TX+VP0U/K/aFxzHzQWHWURL+CtHMSbex1lCdpXD1SoR2sYuxDu5aNI9lPoXpKTCggFZiDJbwylU0LEQ== dependencies: - "@csstools/selector-resolve-nested" "^3.0.0" + "@csstools/selector-resolve-nested" "^3.1.0" "@csstools/selector-specificity" "^5.0.0" postcss-selector-parser "^7.0.0" @@ -7551,67 +7596,68 @@ postcss-place@^10.0.0: dependencies: postcss-value-parser "^4.2.0" -postcss-preset-env@^10.1.0: - version "10.1.3" - resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-10.1.3.tgz#7d07adef2237a643162e751b00eb1e339aa3b82e" - integrity sha512-9qzVhcMFU/MnwYHyYpJz4JhGku/4+xEiPTmhn0hj3IxnUYlEF9vbh7OC1KoLAnenS6Fgg43TKNp9xcuMeAi4Zw== - dependencies: - "@csstools/postcss-cascade-layers" "^5.0.1" - "@csstools/postcss-color-function" "^4.0.7" - "@csstools/postcss-color-mix-function" "^3.0.7" - "@csstools/postcss-content-alt-text" "^2.0.4" - "@csstools/postcss-exponential-functions" "^2.0.6" +postcss-preset-env@^10.2.1: + version "10.2.4" + resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-10.2.4.tgz#17d386b5a86b136dfbca89b52ef03a95ad9e32fa" + integrity sha512-q+lXgqmTMdB0Ty+EQ31SuodhdfZetUlwCA/F0zRcd/XdxjzI+Rl2JhZNz5US2n/7t9ePsvuhCnEN4Bmu86zXlA== + dependencies: + "@csstools/postcss-cascade-layers" "^5.0.2" + "@csstools/postcss-color-function" "^4.0.10" + "@csstools/postcss-color-mix-function" "^3.0.10" + "@csstools/postcss-color-mix-variadic-function-arguments" "^1.0.0" + "@csstools/postcss-content-alt-text" "^2.0.6" + "@csstools/postcss-exponential-functions" "^2.0.9" "@csstools/postcss-font-format-keywords" "^4.0.0" - "@csstools/postcss-gamut-mapping" "^2.0.7" - "@csstools/postcss-gradients-interpolation-method" "^5.0.7" - "@csstools/postcss-hwb-function" "^4.0.7" - "@csstools/postcss-ic-unit" "^4.0.0" - "@csstools/postcss-initial" "^2.0.0" - "@csstools/postcss-is-pseudo-class" "^5.0.1" - "@csstools/postcss-light-dark-function" "^2.0.7" + "@csstools/postcss-gamut-mapping" "^2.0.10" + "@csstools/postcss-gradients-interpolation-method" "^5.0.10" + "@csstools/postcss-hwb-function" "^4.0.10" + "@csstools/postcss-ic-unit" "^4.0.2" + "@csstools/postcss-initial" "^2.0.1" + "@csstools/postcss-is-pseudo-class" "^5.0.3" + "@csstools/postcss-light-dark-function" "^2.0.9" "@csstools/postcss-logical-float-and-clear" "^3.0.0" "@csstools/postcss-logical-overflow" "^2.0.0" "@csstools/postcss-logical-overscroll-behavior" "^2.0.0" "@csstools/postcss-logical-resize" "^3.0.0" - "@csstools/postcss-logical-viewport-units" "^3.0.3" - "@csstools/postcss-media-minmax" "^2.0.6" - "@csstools/postcss-media-queries-aspect-ratio-number-values" "^3.0.4" + "@csstools/postcss-logical-viewport-units" "^3.0.4" + "@csstools/postcss-media-minmax" "^2.0.9" + "@csstools/postcss-media-queries-aspect-ratio-number-values" "^3.0.5" "@csstools/postcss-nested-calc" "^4.0.0" "@csstools/postcss-normalize-display-values" "^4.0.0" - "@csstools/postcss-oklab-function" "^4.0.7" - "@csstools/postcss-progressive-custom-properties" "^4.0.0" - "@csstools/postcss-random-function" "^1.0.2" - "@csstools/postcss-relative-color-syntax" "^3.0.7" + "@csstools/postcss-oklab-function" "^4.0.10" + "@csstools/postcss-progressive-custom-properties" "^4.1.0" + "@csstools/postcss-random-function" "^2.0.1" + "@csstools/postcss-relative-color-syntax" "^3.0.10" "@csstools/postcss-scope-pseudo-class" "^4.0.1" - "@csstools/postcss-sign-functions" "^1.1.1" - "@csstools/postcss-stepped-value-functions" "^4.0.6" - "@csstools/postcss-text-decoration-shorthand" "^4.0.1" - "@csstools/postcss-trigonometric-functions" "^4.0.6" + "@csstools/postcss-sign-functions" "^1.1.4" + "@csstools/postcss-stepped-value-functions" "^4.0.9" + "@csstools/postcss-text-decoration-shorthand" "^4.0.2" + "@csstools/postcss-trigonometric-functions" "^4.0.9" "@csstools/postcss-unset-value" "^4.0.0" - autoprefixer "^10.4.19" - browserslist "^4.23.1" + autoprefixer "^10.4.21" + browserslist "^4.25.0" css-blank-pseudo "^7.0.1" css-has-pseudo "^7.0.2" css-prefers-color-scheme "^10.0.0" - cssdb "^8.2.3" + cssdb "^8.3.0" postcss-attribute-case-insensitive "^7.0.1" postcss-clamp "^4.1.0" - postcss-color-functional-notation "^7.0.7" + postcss-color-functional-notation "^7.0.10" postcss-color-hex-alpha "^10.0.0" postcss-color-rebeccapurple "^10.0.0" - postcss-custom-media "^11.0.5" - postcss-custom-properties "^14.0.4" - postcss-custom-selectors "^8.0.4" + postcss-custom-media "^11.0.6" + postcss-custom-properties "^14.0.6" + postcss-custom-selectors "^8.0.5" postcss-dir-pseudo-class "^9.0.1" - postcss-double-position-gradients "^6.0.0" + postcss-double-position-gradients "^6.0.2" postcss-focus-visible "^10.0.1" postcss-focus-within "^9.0.1" postcss-font-variant "^5.0.0" postcss-gap-properties "^6.0.0" postcss-image-set-function "^7.0.0" - postcss-lab-function "^7.0.7" - postcss-logical "^8.0.0" - postcss-nesting "^13.0.1" + postcss-lab-function "^7.0.10" + postcss-logical "^8.1.0" + postcss-nesting "^13.0.2" postcss-opacity-percentage "^3.0.0" postcss-overflow-shorthand "^6.0.0" postcss-page-break "^3.0.4" @@ -7717,7 +7763,7 @@ postcss-zindex@^6.0.2: resolved "https://registry.yarnpkg.com/postcss-zindex/-/postcss-zindex-6.0.2.tgz#e498304b83a8b165755f53db40e2ea65a99b56e1" integrity sha512-5BxW9l1evPB/4ZIc+2GobEBoKC+h8gPGCMi+jxsYvd2x0mjq7wazk6DrP71pStqxE9Foxh5TVnonbWpFZzXaYg== -postcss@^8.4.21, postcss@^8.4.24, postcss@^8.4.26, postcss@^8.4.33, postcss@^8.4.38, postcss@^8.4.47, postcss@^8.5.1: +postcss@^8.4.21, postcss@^8.4.24, postcss@^8.4.33, postcss@^8.4.47, postcss@^8.5.1: version "8.5.1" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.1.tgz#e2272a1f8a807fafa413218245630b5db10a3214" integrity sha512-6oz2beyjc5VMn/KV1pPw8fliQkhBXrVn1Z3TVyqZxU8kZpzEKhBdmCFqI6ZbmGtamQvQGuU1sgPTk8ZrXDD7jQ== @@ -7726,6 +7772,15 @@ postcss@^8.4.21, postcss@^8.4.24, postcss@^8.4.26, postcss@^8.4.33, postcss@^8.4 picocolors "^1.1.1" source-map-js "^1.2.1" +postcss@^8.5.4: + version "8.5.6" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.6.tgz#2825006615a619b4f62a9e7426cc120b349a8f3c" + integrity sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg== + dependencies: + nanoid "^3.3.11" + picocolors "^1.1.1" + source-map-js "^1.2.1" + pretty-error@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-4.0.0.tgz#90a703f46dd7234adb46d0f84823e9d1cb8f10d6" @@ -7823,13 +7878,6 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== -queue@6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/queue/-/queue-6.0.2.tgz#b91525283e2315c7553d2efa18d83e76432fed65" - integrity sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA== - dependencies: - inherits "~2.0.3" - quick-lru@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" @@ -7872,36 +7920,6 @@ rc@1.2.8: minimist "^1.2.0" strip-json-comments "~2.0.1" -react-dev-utils@^12.0.1: - version "12.0.1" - resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-12.0.1.tgz#ba92edb4a1f379bd46ccd6bcd4e7bc398df33e73" - integrity sha512-84Ivxmr17KjUupyqzFode6xKhjwuEJDROWKJy/BthkL7Wn6NJ8h4WE6k/exAv6ImS+0oZLRRW5j/aINMHyeGeQ== - dependencies: - "@babel/code-frame" "^7.16.0" - address "^1.1.2" - browserslist "^4.18.1" - chalk "^4.1.2" - cross-spawn "^7.0.3" - detect-port-alt "^1.1.6" - escape-string-regexp "^4.0.0" - filesize "^8.0.6" - find-up "^5.0.0" - fork-ts-checker-webpack-plugin "^6.5.0" - global-modules "^2.0.0" - globby "^11.0.4" - gzip-size "^6.0.0" - immer "^9.0.7" - is-root "^2.1.0" - loader-utils "^3.2.0" - open "^8.4.0" - pkg-up "^3.1.0" - prompts "^2.4.2" - react-error-overlay "^6.0.11" - recursive-readdir "^2.2.2" - shell-quote "^1.7.3" - strip-ansi "^6.0.1" - text-table "^0.2.0" - react-dom@^19.0.0: version "19.0.0" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-19.0.0.tgz#43446f1f01c65a4cd7f7588083e686a6726cfb57" @@ -7909,17 +7927,12 @@ react-dom@^19.0.0: dependencies: scheduler "^0.25.0" -react-error-overlay@^6.0.11: - version "6.0.11" - resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.11.tgz#92835de5841c5cf08ba00ddd2d677b6d17ff9adb" - integrity sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg== - react-fast-compare@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-3.2.2.tgz#929a97a532304ce9fee4bcae44234f1ce2c21d49" integrity sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ== -"react-helmet-async@npm:@slorber/react-helmet-async@*", "react-helmet-async@npm:@slorber/react-helmet-async@1.3.0": +"react-helmet-async@npm:@slorber/react-helmet-async@1.3.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@slorber/react-helmet-async/-/react-helmet-async-1.3.0.tgz#11fbc6094605cf60aa04a28c17e0aab894b4ecff" integrity sha512-e9/OK8VhwUSc67diWI8Rb3I0YgI9/SBQtnhe9aEuK6MhZm7ntZZimXgwXnd8W96YTmSOb9M4d8LwhRZyhWr/1A== @@ -7935,10 +7948,10 @@ react-is@^16.13.1, react-is@^16.6.0, react-is@^16.7.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== -react-json-view-lite@^1.2.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/react-json-view-lite/-/react-json-view-lite-1.5.0.tgz#377cc302821717ac79a1b6d099e1891df54c8662" - integrity sha512-nWqA1E4jKPklL2jvHWs6s+7Na0qNgw9HCP6xehdQJeg6nPBTFZgGwyko9Q0oj+jQWKTTVRS30u0toM5wiuL3iw== +react-json-view-lite@^2.3.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/react-json-view-lite/-/react-json-view-lite-2.4.1.tgz#0d06696a06aaf4a74e890302b76cf8cddcc45d60" + integrity sha512-fwFYknRIBxjbFm0kBDrzgBy1xa5tDg2LyXXBepC5f1b+MY3BUClMCsvanMPn089JbV1Eg3nZcrp0VCuH43aXnA== react-loadable-ssr-addon-v5-slorber@^1.0.1: version "1.0.1" @@ -8030,18 +8043,6 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" -reading-time@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/reading-time/-/reading-time-1.5.0.tgz#d2a7f1b6057cb2e169beaf87113cc3411b5bc5bb" - integrity sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg== - -rechoir@^0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" - integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw== - dependencies: - resolve "^1.1.6" - recma-build-jsx@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/recma-build-jsx/-/recma-build-jsx-1.0.0.tgz#c02f29e047e103d2fab2054954e1761b8ea253c4" @@ -8082,13 +8083,6 @@ recma-stringify@^1.0.0: unified "^11.0.0" vfile "^6.0.0" -recursive-readdir@^2.2.2: - version "2.2.3" - resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.3.tgz#e726f328c0d69153bcabd5c322d3195252379372" - integrity sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA== - dependencies: - minimatch "^3.0.5" - regenerate-unicode-properties@^10.2.0: version "10.2.0" resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-10.2.0.tgz#626e39df8c372338ea9b8028d1f99dc3fd9c3db0" @@ -8316,7 +8310,12 @@ resolve-pathname@^3.0.0: resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-3.0.0.tgz#99d02224d3cf263689becbb393bc560313025dcd" integrity sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng== -resolve@^1.1.6, resolve@^1.1.7, resolve@^1.14.2, resolve@^1.22.8: +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== + +resolve@^1.1.7, resolve@^1.14.2, resolve@^1.22.8: version "1.22.10" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.10.tgz#b663e83ffb09bbf2386944736baae803029b8b39" integrity sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w== @@ -8391,14 +8390,10 @@ scheduler@^0.25.0: resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.25.0.tgz#336cd9768e8cceebf52d3c80e3dcf5de23e7e015" integrity sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA== -schema-utils@2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.0.tgz#17151f76d8eae67fbbf77960c33c676ad9f4efc7" - integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A== - dependencies: - "@types/json-schema" "^7.0.4" - ajv "^6.12.2" - ajv-keywords "^3.4.1" +schema-dts@^1.1.2: + version "1.1.5" + resolved "https://registry.yarnpkg.com/schema-dts/-/schema-dts-1.1.5.tgz#9237725d305bac3469f02b292a035107595dc324" + integrity sha512-RJr9EaCmsLzBX2NDiO5Z3ux2BVosNZN5jo0gWgsyKvxKIUL5R3swNvoorulAeL9kLB0iTSX7V6aokhla2m7xbg== schema-utils@^3.0.0, schema-utils@^3.2.0: version "3.3.0" @@ -8452,7 +8447,7 @@ semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.4: +semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.4: version "7.6.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== @@ -8565,20 +8560,11 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shell-quote@^1.7.3, shell-quote@^1.8.1: +shell-quote@^1.8.1: version "1.8.2" resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.2.tgz#d2d83e057959d53ec261311e9e9b8f51dcb2934a" integrity sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA== -shelljs@^0.8.5: - version "0.8.5" - resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c" - integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow== - dependencies: - glob "^7.0.0" - interpret "^1.0.0" - rechoir "^0.6.2" - side-channel-list@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/side-channel-list/-/side-channel-list-1.0.0.tgz#10cb5984263115d3b7a0e336591e290a830af8ad" @@ -8975,11 +8961,6 @@ tailwindcss@^3.4.17: resolve "^1.22.8" sucrase "^3.35.0" -tapable@^1.0.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" - integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== - tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0, tapable@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" @@ -9006,11 +8987,6 @@ terser@^5.10.0, terser@^5.15.1, terser@^5.31.1: commander "^2.20.0" source-map-support "~0.5.20" -text-table@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== - thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726" @@ -9040,6 +9016,11 @@ tiny-warning@^1.0.0: resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754" integrity sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA== +tinypool@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-1.1.1.tgz#059f2d042bd37567fbc017d3d426bdd2a2612591" + integrity sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg== + to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" @@ -9090,6 +9071,16 @@ tslib@^2.0.3, tslib@^2.6.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== +tsx@^4.20.3: + version "4.20.3" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.20.3.tgz#f913e4911d59ad177c1bcee19d1035ef8dd6e2fb" + integrity sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ== + dependencies: + esbuild "~0.25.0" + get-tsconfig "^4.7.5" + optionalDependencies: + fsevents "~2.3.3" + type-fest@^0.21.3: version "0.21.3" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" @@ -9289,6 +9280,14 @@ update-browserslist-db@^1.1.1: escalade "^3.2.0" picocolors "^1.1.1" +update-browserslist-db@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz#348377dd245216f9e7060ff50b15a1b740b75420" + integrity sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw== + dependencies: + escalade "^3.2.0" + picocolors "^1.1.1" + update-notifier@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-6.0.2.tgz#a6990253dfe6d5a02bd04fbb6a61543f55026b60" @@ -9585,13 +9584,6 @@ websocket-extensions@>=0.1.1: resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== -which@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" - integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== - dependencies: - isexe "^2.0.0" - which@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" @@ -9692,21 +9684,11 @@ yallist@^3.0.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yaml@^1.7.2: - version "1.10.2" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" - integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== - yaml@^2.3.4: version "2.7.0" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.7.0.tgz#aef9bb617a64c937a9a748803786ad8d3ffe1e98" integrity sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA== -yocto-queue@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" - integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== - yocto-queue@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.1.1.tgz#fef65ce3ac9f8a32ceac5a634f74e17e5b232110" diff --git a/pyproject.toml b/pyproject.toml index a90cfac2f..427a4300e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ verify = ["black --check .", "pylint --output-format=colorized -j 0 src tests"] update_github_urls = "python docs/dqx/update_github_urls.py" +[tool.hatch.envs.docs] +dependencies = [ + "python-markdown" +] + [tool.isort] profile = "black" diff --git a/scripts/docgen.py b/scripts/docgen.py new file mode 100644 index 000000000..70eb3a80b --- /dev/null +++ b/scripts/docgen.py @@ -0,0 +1,668 @@ +#!/usr/bin/env python3 +""" +Script to generate comprehensive markdown documentation from Python module docstrings. + +This script analyzes Python modules and generates a single markdown file with +structured documentation including function signatures, parameters, and metadata. +""" + +import ast +import importlib +import inspect +import logging +import re +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional + + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%H:%M:%S' +) +logger = logging.getLogger(__name__) + + +@dataclass +class DocstringInfo: + """Container for docstring information.""" + + source_file_name: str + source_file_path: str + module_name: str + class_name: Optional[str] + function_name: str + method_type: Optional[str] # 'classmethod', 'staticmethod', 'instancemethod', or None + doc_body: str + parameters: List[Dict[str, Any]] + return_info: Optional[Dict[str, Any]] + docstring: str + line_start: int + line_end: int + + +class MarkdownDocGenerator: + """Generates markdown documentation from collected docstrings.""" + + def __init__(self, project_root: Path): + self.project_root = project_root + self.collected_docstrings: List[DocstringInfo] = [] + self.stats = { + 'modules_processed': 0, + 'classes_processed': 0, + 'functions_processed': 0, + 'docstrings_found': 0, + 'errors': 0 + } + + def _get_relative_path(self, file_path: str) -> str: + """Get relative path from project root.""" + try: + return str(Path(file_path).relative_to(self.project_root)) + except ValueError: + return file_path + + def _get_method_type(self, func: Any, class_obj: Any = None) -> Optional[str]: + """Determine if a method is classmethod, staticmethod, or instancemethod.""" + if class_obj is None: + return None + + # Check if it's a classmethod + if isinstance(inspect.getattr_static(class_obj, func.__name__, None), classmethod): + return "classmethod" + + # Check if it's a staticmethod + if isinstance(inspect.getattr_static(class_obj, func.__name__, None), staticmethod): + return "staticmethod" + + # Check if it's a regular method (has 'self' parameter) + try: + sig = inspect.signature(func) + params = list(sig.parameters.keys()) + if params and params[0] in ('self', 'cls'): + return "instancemethod" + except (ValueError, TypeError): + pass + + return None + + def _extract_parameters(self, func: Any) -> List[Dict[str, Any]]: + """Extract parameter information from a function.""" + try: + sig = inspect.signature(func) + parameters = [] + + for param_name, param in sig.parameters.items(): + param_info = { + "name": param_name, + "annotation": str(param.annotation) if param.annotation != inspect.Parameter.empty else None, + "default": str(param.default) if param.default != inspect.Parameter.empty else None, + "kind": param.kind.name + } + parameters.append(param_info) + + return parameters + except (ValueError, TypeError): + return [] + + def _extract_return_info(self, func: Any) -> Optional[Dict[str, Any]]: + """Extract return type information from a function.""" + try: + sig = inspect.signature(func) + if sig.return_annotation != inspect.Signature.empty: + return { + "annotation": str(sig.return_annotation), + "description": None # Could be parsed from docstring + } + except (ValueError, TypeError): + pass + return None + + def _get_source_lines(self, obj: Any) -> tuple[int, int]: + """Get start and end line numbers for an object.""" + try: + source_lines = inspect.getsourcelines(obj) + line_start = source_lines[1] + line_end = line_start + len(source_lines[0]) - 1 + return line_start, line_end + except (OSError, TypeError): + return 0, 0 + + def _escape_markdown(self, text: str) -> str: + """Escape special markdown characters in text.""" + if not text: + return "" + # Escape MDX/JSX template syntax that could cause issues + text = text.replace('{{', '{{') + text = text.replace('}}', '}}') + # Escape markdown special characters + text = re.sub(r'([*_`\[\]{}()#+\-.!|\\])', r'\\\1', text) + return text + + def _clean_type_annotation(self, annotation: str) -> str: + """Clean up type annotations to avoid MDX parsing issues.""" + if not annotation: + return "" + + # Replace problematic patterns in type annotations + # Convert to str + annotation = re.sub(r"", r'\1', annotation) + + # Convert to module.Class + annotation = re.sub(r"", r'\1', annotation) + + # Handle other <...> patterns that might contain quotes + annotation = re.sub(r"<([^<>]*)'([^<>]*)'([^<>]*)>", r'<\1"\2"\3>', annotation) + + # Escape any remaining single quotes that might cause issues + annotation = annotation.replace("'", "'") + + return annotation + + def _format_signature(self, func_name: str, parameters: List[Dict[str, Any]], + return_info: Optional[Dict[str, Any]], method_type: Optional[str] = None) -> str: + """Format function signature for markdown.""" + params_str = [] + for param in parameters: + param_str = param['name'] + if param['annotation']: + clean_annotation = self._clean_type_annotation(param['annotation']) + param_str += f": {clean_annotation}" + if param['default'] is not None: + param_str += f" = {param['default']}" + params_str.append(param_str) + + signature = f"{func_name}({', '.join(params_str)})" + if return_info and return_info['annotation']: + clean_return_annotation = self._clean_type_annotation(return_info['annotation']) + signature += f" -> {clean_return_annotation}" + + if method_type: + signature = f"@{method_type}\n{signature}" + + return signature + + def _process_function(self, func: Any, module_name: str, class_obj: Any = None, + class_name: Optional[str] = None) -> Optional[DocstringInfo]: + """Process a function and extract its docstring information.""" + try: + self.stats['functions_processed'] += 1 + + # Skip if no docstring + docstring = inspect.getdoc(func) + if not docstring: + return None + + logger.debug(f"πŸ“ Processing function: {func.__name__}") + self.stats['docstrings_found'] += 1 + + # Get source file information + source_file = inspect.getfile(func) + source_file_name = Path(source_file).name + source_file_path = self._get_relative_path(source_file) + + # Get method type + method_type = self._get_method_type(func, class_obj) + + # Extract parameters and return info + parameters = self._extract_parameters(func) + return_info = self._extract_return_info(func) + + # Get source line numbers + line_start, line_end = self._get_source_lines(func) + + # Create docstring info + docstring_info = DocstringInfo( + source_file_name=source_file_name, + source_file_path=source_file_path, + module_name=module_name, + class_name=class_name, + function_name=func.__name__, + method_type=method_type, + doc_body=docstring, + parameters=parameters, + return_info=return_info, + docstring=docstring, + line_start=line_start, + line_end=line_end + ) + + return docstring_info + + except Exception as e: + self.stats['errors'] += 1 + logger.error(f"❌ Error processing function {func.__name__}: {e}") + return None + + def _process_class(self, class_obj: Any, module_name: str) -> List[DocstringInfo]: + """Process a class and extract docstring information for it and its methods.""" + self.stats['classes_processed'] += 1 + logger.info(f"πŸ—οΈ Processing class: {class_obj.__name__}") + + docstrings = [] + + # Process class docstring + class_docstring = inspect.getdoc(class_obj) + if class_docstring: + try: + source_file = inspect.getfile(class_obj) + source_file_name = Path(source_file).name + source_file_path = self._get_relative_path(source_file) + line_start, line_end = self._get_source_lines(class_obj) + + class_docstring_info = DocstringInfo( + source_file_name=source_file_name, + source_file_path=source_file_path, + module_name=module_name, + class_name=class_obj.__name__, + function_name="__class__", + method_type=None, + doc_body=class_docstring, + parameters=[], + return_info=None, + docstring=class_docstring, + line_start=line_start, + line_end=line_end + ) + docstrings.append(class_docstring_info) + except Exception as e: + self.stats['errors'] += 1 + logger.error(f"❌ Error processing class {class_obj.__name__}: {e}") + + # Process methods + methods = [(method_name, method) for method_name, method in inspect.getmembers(class_obj, inspect.isfunction) + if not method_name.startswith('_') or method_name in ['__init__', '__new__']] + + if methods: + logger.debug(f" πŸ”§ Processing {len(methods)} methods in {class_obj.__name__}") + for method_name, method in methods: + method_info = self._process_function(method, module_name, class_obj, class_obj.__name__) + if method_info: + docstrings.append(method_info) + + # Process static methods and class methods + for method_name, method in inspect.getmembers(class_obj): + if isinstance(method, staticmethod): + if hasattr(method, '__func__'): + method_info = self._process_function(method.__func__, module_name, class_obj, class_obj.__name__) + if method_info: + docstrings.append(method_info) + elif isinstance(method, classmethod): + if hasattr(method, '__func__'): + method_info = self._process_function(method.__func__, module_name, class_obj, class_obj.__name__) + if method_info: + docstrings.append(method_info) + elif (callable(method) and not method_name.startswith('_') and + not inspect.isfunction(method) and not inspect.ismethod(method)): + # Handle other callable objects that might have docstrings + try: + method_info = self._process_function(method, module_name, class_obj, class_obj.__name__) + if method_info: + docstrings.append(method_info) + except Exception: + # Skip if we can't process this callable + pass + + return docstrings + + def _process_module(self, module: Any, module_name: str) -> List[DocstringInfo]: + """Process a module and extract all docstring information.""" + self.stats['modules_processed'] += 1 + logger.info(f"πŸ“¦ Processing module: {module_name}") + + docstrings = [] + + # Process module-level functions + module_functions = [func for func_name, func in inspect.getmembers(module, inspect.isfunction) + if func.__module__ == module_name] + + if module_functions: + logger.info(f" πŸ”§ Found {len(module_functions)} functions") + for func in module_functions: + func_info = self._process_function(func, module_name) + if func_info: + docstrings.append(func_info) + + # Process classes + module_classes = [class_obj for class_name, class_obj in inspect.getmembers(module, inspect.isclass) + if class_obj.__module__ == module_name] + + if module_classes: + logger.info(f" πŸ—οΈ Found {len(module_classes)} classes") + for i, class_obj in enumerate(module_classes, 1): + logger.debug(f" πŸ“ Processing class {i}/{len(module_classes)}: {class_obj.__name__}") + class_docstrings = self._process_class(class_obj, module_name) + docstrings.extend(class_docstrings) + + logger.info(f" βœ… Collected {len(docstrings)} docstrings from {module_name}") + return docstrings + + def collect_from_module(self, module_name: str) -> List[DocstringInfo]: + """Collect docstrings from a given module.""" + logger.info(f"πŸš€ Starting docstring collection from module: {module_name}") + + try: + # Import the module + logger.info(f"πŸ“₯ Importing module: {module_name}") + module = importlib.import_module(module_name) + + # Process the module + docstrings = self._process_module(module, module_name) + + # Also process submodules if it's a package + if hasattr(module, '__path__'): + logger.info(f"πŸ“ Discovering submodules in package: {module_name}") + try: + import pkgutil + submodules = list(pkgutil.iter_modules(module.__path__, module_name + ".")) + logger.info(f" πŸ” Found {len(submodules)} submodules") + + for i, (importer, submodule_name, ispkg) in enumerate(submodules, 1): + try: + logger.debug(f" πŸ“ Processing submodule {i}/{len(submodules)}: {submodule_name}") + submodule = importlib.import_module(submodule_name) + sub_docstrings = self._process_module(submodule, submodule_name) + docstrings.extend(sub_docstrings) + except Exception as e: + self.stats['errors'] += 1 + logger.error(f"❌ Error processing submodule {submodule_name}: {e}") + except Exception as e: + self.stats['errors'] += 1 + logger.error(f"❌ Error iterating submodules: {e}") + + logger.info(f"πŸŽ‰ Successfully collected {len(docstrings)} docstrings from {module_name}") + return docstrings + + except ImportError as e: + self.stats['errors'] += 1 + logger.error(f"❌ Error importing module {module_name}: {e}") + return [] + except Exception as e: + self.stats['errors'] += 1 + logger.error(f"❌ Error processing module {module_name}: {e}") + return [] + + def _generate_markdown_content(self, docstrings: List[DocstringInfo], module_name: str) -> str: + """Generate markdown content from collected docstrings.""" + logger.info(f"πŸ“ Generating markdown content for {len(docstrings)} docstrings") + + # Group docstrings by module and class + modules = {} + for ds in docstrings: + if ds.module_name not in modules: + modules[ds.module_name] = {'functions': [], 'classes': {}} + + if ds.class_name and ds.function_name != '__class__': + # This is a method + if ds.class_name not in modules[ds.module_name]['classes']: + modules[ds.module_name]['classes'][ds.class_name] = {'docstring': None, 'methods': []} + modules[ds.module_name]['classes'][ds.class_name]['methods'].append(ds) + elif ds.class_name and ds.function_name == '__class__': + # This is a class docstring + if ds.class_name not in modules[ds.module_name]['classes']: + modules[ds.module_name]['classes'][ds.class_name] = {'docstring': None, 'methods': []} + modules[ds.module_name]['classes'][ds.class_name]['docstring'] = ds + else: + # This is a module-level function + modules[ds.module_name]['functions'].append(ds) + + # Generate markdown + md_content = [] + + # Header + md_content.append(f"# {module_name} API Reference") + md_content.append("") + md_content.append('
') + md_content.append("") + + # Note: Table of contents is handled by Docusaurus automatically + + # Generate content for each module + for mod_name in sorted(modules.keys()): + mod_data = modules[mod_name] + + md_content.append(f"## {mod_name}") + md_content.append("") + md_content.append(f'
') + + # Module-level functions + if mod_data['functions']: + md_content.append("### Functions") + md_content.append("") + md_content.append('
') + + for func in sorted(mod_data['functions'], key=lambda x: x.function_name): + md_content.append(f"#### {func.function_name}") + md_content.append("") + md_content.append(f'
') + + # Source info + md_content.append(f'
') + md_content.append(f"**Source:** `{func.source_file_path}:{func.line_start}-{func.line_end}`") + md_content.append("
") + md_content.append("") + + # Function signature + signature = self._format_signature(func.function_name, func.parameters, func.return_info, func.method_type) + md_content.append('
') + md_content.append("```python") + md_content.append(signature) + md_content.append("```") + md_content.append("
") + md_content.append("") + + # Docstring + md_content.append('
') + md_content.append(self._escape_markdown(func.docstring)) + md_content.append("
") + + # Parameters + if func.parameters: + md_content.append("") + md_content.append('
') + md_content.append("**Parameters:**") + md_content.append("") + for param in func.parameters: + param_line = f"- `{param['name']}`" + if param['annotation']: + clean_annotation = self._clean_type_annotation(param['annotation']) + param_line += f" ({clean_annotation})" + if param['default'] is not None: + param_line += f", default: `{param['default']}`" + md_content.append(param_line) + md_content.append("
") + + # Return type + if func.return_info: + md_content.append("") + md_content.append('
') + clean_return_annotation = self._clean_type_annotation(func.return_info['annotation']) + md_content.append(f"**Returns:** `{clean_return_annotation}`") + md_content.append("
") + + md_content.append("") + md_content.append("
") + md_content.append("") + + md_content.append("
") + md_content.append("") + + # Classes + if mod_data['classes']: + md_content.append("### Classes") + md_content.append("") + md_content.append('
') + + for class_name in sorted(mod_data['classes'].keys()): + class_data = mod_data['classes'][class_name] + + md_content.append(f"#### {class_name}") + md_content.append("") + md_content.append(f'
') + + # Class docstring + if class_data['docstring']: + ds = class_data['docstring'] + md_content.append(f'
') + md_content.append(f"**Source:** `{ds.source_file_path}:{ds.line_start}-{ds.line_end}`") + md_content.append("
") + md_content.append("") + + md_content.append('
') + md_content.append(self._escape_markdown(ds.docstring)) + md_content.append("
") + md_content.append("") + + # Methods + if class_data['methods']: + md_content.append("##### Methods") + md_content.append("") + md_content.append('
') + + for method in sorted(class_data['methods'], key=lambda x: x.function_name): + md_content.append(f"###### {method.function_name}") + md_content.append("") + md_content.append(f'
') + + # Method signature + signature = self._format_signature(method.function_name, method.parameters, method.return_info, method.method_type) + md_content.append('
') + md_content.append("```python") + md_content.append(signature) + md_content.append("```") + md_content.append("
") + md_content.append("") + + # Docstring + md_content.append('
') + md_content.append(self._escape_markdown(method.docstring)) + md_content.append("
") + + # Parameters + if method.parameters: + md_content.append("") + md_content.append('
') + md_content.append("**Parameters:**") + md_content.append("") + for param in method.parameters: + param_line = f"- `{param['name']}`" + if param['annotation']: + clean_annotation = self._clean_type_annotation(param['annotation']) + param_line += f" ({clean_annotation})" + if param['default'] is not None: + param_line += f", default: `{param['default']}`" + md_content.append(param_line) + md_content.append("
") + + # Return type + if method.return_info: + md_content.append("") + md_content.append('
') + clean_return_annotation = self._clean_type_annotation(method.return_info['annotation']) + md_content.append(f"**Returns:** `{clean_return_annotation}`") + md_content.append("
") + + md_content.append("") + md_content.append("
") + md_content.append("") + + md_content.append("
") + + md_content.append("
") + md_content.append("") + + md_content.append("
") + + md_content.append("
") + md_content.append("") + + md_content.append("
") + + return "\n".join(md_content) + + +def generate_markdown_docs(module_name: str, output_file_path: Path) -> None: + """ + Generate markdown documentation from a given module and save to a file. + + Args: + module_name: Name of the module to analyze (e.g., 'databricks.labs.dqx') + output_file_path: Path where to save the generated markdown documentation + """ + start_time = time.time() + + logger.info(f"🌟 Starting DQX Markdown Documentation Generator") + logger.info(f"πŸ“ Target module: {module_name}") + logger.info(f"πŸ“ Output file: {output_file_path}") + + # Get project root (assume script is in scripts/ directory) + project_root = Path(__file__).parent.parent + logger.info(f"🏠 Project root: {project_root}") + + # Create generator + generator = MarkdownDocGenerator(project_root) + + # Collect docstrings + docstrings = generator.collect_from_module(module_name) + + # Generate markdown content + markdown_content = generator._generate_markdown_content(docstrings, module_name) + + # Save to file + logger.info(f"πŸ’Ύ Saving markdown documentation to {output_file_path}") + output_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_file_path, 'w', encoding='utf-8') as f: + f.write(markdown_content) + + # Print summary statistics + elapsed_time = time.time() - start_time + logger.info(f"πŸ“Š Generation Summary:") + logger.info(f" πŸ“¦ Modules processed: {generator.stats['modules_processed']}") + logger.info(f" πŸ—οΈ Classes processed: {generator.stats['classes_processed']}") + logger.info(f" πŸ”§ Functions processed: {generator.stats['functions_processed']}") + logger.info(f" πŸ“ Docstrings found: {generator.stats['docstrings_found']}") + logger.info(f" ❌ Errors encountered: {generator.stats['errors']}") + logger.info(f" πŸ’Ύ Output file size: {output_file_path.stat().st_size / 1024:.1f} KB") + logger.info(f" ⏱️ Execution time: {elapsed_time:.2f} seconds") + + logger.info(f"βœ… Successfully completed! Generated markdown documentation at {output_file_path}") + + +def main(): + """Main function for command-line usage.""" + import argparse + + # Print header + print("πŸš€ DQX Markdown Documentation Generator") + print("=" * 50) + + parser = argparse.ArgumentParser( + description="Generate comprehensive markdown documentation from Python modules", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python scripts/docgen.py databricks.labs.dqx docs/generated/api-reference.md + python scripts/docgen.py databricks.labs.dqx.engine /tmp/engine_docs.md --verbose + """ + ) + + parser.add_argument("module_name", help="Name of the module to analyze (e.g., 'databricks.labs.dqx')") + parser.add_argument("output_file", help="Output file path for generated markdown documentation") + parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Set log level based on verbose flag + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + logger.info("πŸ” Verbose logging enabled") + + output_path = Path(args.output_file) + generate_markdown_docs(args.module_name, output_path) + + +if __name__ == "__main__": + main() \ No newline at end of file From 4fb240a5e4043a4e76094c9272f76a7742ac21eb Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 13 Aug 2025 12:55:47 +0200 Subject: [PATCH 02/37] add docgen --- Makefile | 5 +- docs/dqx/docs/reference/api.md | 4 - docs/dqx/docs/reference/api/__main__.md | 14 + docs/dqx/docs/reference/api/base.md | 234 +++++ docs/dqx/docs/reference/api/check_funcs.md | 821 ++++++++++++++++++ .../dqx/docs/reference/api/checks_resolver.md | 26 + .../docs/reference/api/checks_serializer.md | 140 +++ docs/dqx/docs/reference/api/checks_storage.md | 308 +++++++ .../docs/reference/api/checks_validator.md | 72 ++ docs/dqx/docs/reference/api/cli.md | 123 +++ docs/dqx/docs/reference/api/config.md | 198 +++++ docs/dqx/docs/reference/api/config_loader.md | 42 + .../reference/api/contexts/application.md | 41 + .../docs/reference/api/contexts/workflows.md | 91 ++ .../docs/reference/api/contexts/workspace.md | 27 + docs/dqx/docs/reference/api/engine.md | 318 +++++++ docs/dqx/docs/reference/api/executor.md | 159 ++++ docs/dqx/docs/reference/api/index.mdx | 8 + .../docs/reference/api/installer/install.md | 188 ++++ docs/dqx/docs/reference/api/installer/logs.md | 41 + .../reference/api/installer/workflow_task.md | 85 ++ .../api/installer/workflows_installer.md | 19 + .../api/llm/extract_yaml_checks_examples.md | 59 ++ docs/dqx/docs/reference/api/llm/utils.md | 38 + docs/dqx/docs/reference/api/manager.md | 60 ++ .../dqx/docs/reference/api/profiler/common.md | 39 + .../reference/api/profiler/dlt_generator.md | 36 + .../docs/reference/api/profiler/generator.md | 107 +++ .../docs/reference/api/profiler/profiler.md | 104 +++ .../dqx/docs/reference/api/profiler/runner.md | 48 + .../docs/reference/api/profiler/workflow.md | 24 + docs/dqx/docs/reference/api/rule.md | 202 +++++ docs/dqx/docs/reference/api/runtime.md | 44 + docs/dqx/docs/reference/api/sidebar.json | 79 ++ docs/dqx/docs/reference/api/utils.md | 169 ++++ docs/dqx/docs/reference/engine.mdx | 4 + docs/dqx/docs/reference/profiler.mdx | 4 + docs/dqx/docs/reference/quality_rules.mdx | 4 + docs/dqx/docs/reference/testing.mdx | 4 + docs/dqx/sidebars.ts | 1 - pyproject.toml | 117 ++- scripts/docgen.py | 668 -------------- src/databricks/labs/dqx/base.py | 1 - src/databricks/labs/dqx/check_funcs.py | 10 +- src/databricks/labs/dqx/contexts/workflows.py | 1 - .../labs/dqx/installer/workflows_installer.py | 1 - .../labs/dqx/profiler/dlt_generator.py | 1 - src/databricks/labs/dqx/profiler/generator.py | 1 - src/databricks/labs/dqx/rule.py | 1 - tests/e2e/test_run_demos.py | 2 - tests/unit/test_build_rules.py | 1 - tests/unit/test_logs.py | 1 - uv.lock | 303 +++++++ 53 files changed, 4371 insertions(+), 727 deletions(-) delete mode 100644 docs/dqx/docs/reference/api.md create mode 100644 docs/dqx/docs/reference/api/__main__.md create mode 100644 docs/dqx/docs/reference/api/base.md create mode 100644 docs/dqx/docs/reference/api/check_funcs.md create mode 100644 docs/dqx/docs/reference/api/checks_resolver.md create mode 100644 docs/dqx/docs/reference/api/checks_serializer.md create mode 100644 docs/dqx/docs/reference/api/checks_storage.md create mode 100644 docs/dqx/docs/reference/api/checks_validator.md create mode 100644 docs/dqx/docs/reference/api/cli.md create mode 100644 docs/dqx/docs/reference/api/config.md create mode 100644 docs/dqx/docs/reference/api/config_loader.md create mode 100644 docs/dqx/docs/reference/api/contexts/application.md create mode 100644 docs/dqx/docs/reference/api/contexts/workflows.md create mode 100644 docs/dqx/docs/reference/api/contexts/workspace.md create mode 100644 docs/dqx/docs/reference/api/engine.md create mode 100644 docs/dqx/docs/reference/api/executor.md create mode 100644 docs/dqx/docs/reference/api/index.mdx create mode 100644 docs/dqx/docs/reference/api/installer/install.md create mode 100644 docs/dqx/docs/reference/api/installer/logs.md create mode 100644 docs/dqx/docs/reference/api/installer/workflow_task.md create mode 100644 docs/dqx/docs/reference/api/installer/workflows_installer.md create mode 100644 docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md create mode 100644 docs/dqx/docs/reference/api/llm/utils.md create mode 100644 docs/dqx/docs/reference/api/manager.md create mode 100644 docs/dqx/docs/reference/api/profiler/common.md create mode 100644 docs/dqx/docs/reference/api/profiler/dlt_generator.md create mode 100644 docs/dqx/docs/reference/api/profiler/generator.md create mode 100644 docs/dqx/docs/reference/api/profiler/profiler.md create mode 100644 docs/dqx/docs/reference/api/profiler/runner.md create mode 100644 docs/dqx/docs/reference/api/profiler/workflow.md create mode 100644 docs/dqx/docs/reference/api/rule.md create mode 100644 docs/dqx/docs/reference/api/runtime.md create mode 100644 docs/dqx/docs/reference/api/sidebar.json create mode 100644 docs/dqx/docs/reference/api/utils.md delete mode 100644 scripts/docgen.py create mode 100644 uv.lock diff --git a/Makefile b/Makefile index f032be839..af69e1a60 100644 --- a/Makefile +++ b/Makefile @@ -31,14 +31,17 @@ e2e: coverage: hatch run coverage; open htmlcov/index.html -docs-build: +docs-build: + hatch run docs:pydoc-markdown yarn --cwd docs/dqx build docs-serve-dev: + hatch run docs:pydoc-markdown yarn --cwd docs/dqx start docs-install: yarn --cwd docs/dqx install docs-serve: docs-build + hatch run docs:pydoc-markdown yarn --cwd docs/dqx serve \ No newline at end of file diff --git a/docs/dqx/docs/reference/api.md b/docs/dqx/docs/reference/api.md deleted file mode 100644 index 158946bc1..000000000 --- a/docs/dqx/docs/reference/api.md +++ /dev/null @@ -1,4 +0,0 @@ - - -# databricks.labs.dqx - diff --git a/docs/dqx/docs/reference/api/__main__.md b/docs/dqx/docs/reference/api/__main__.md new file mode 100644 index 000000000..553de5aed --- /dev/null +++ b/docs/dqx/docs/reference/api/__main__.md @@ -0,0 +1,14 @@ +--- +sidebar_label: __main__ +title: databricks.labs.dqx.__main__ +--- + +#### me + +```python +@dqx.command +def me(w: WorkspaceClient, greeting: str) +``` + +Shows current username + diff --git a/docs/dqx/docs/reference/api/base.md b/docs/dqx/docs/reference/api/base.md new file mode 100644 index 000000000..bf766893c --- /dev/null +++ b/docs/dqx/docs/reference/api/base.md @@ -0,0 +1,234 @@ +--- +sidebar_label: base +title: databricks.labs.dqx.base +--- + +## DQEngineBase Objects + +```python +class DQEngineBase(abc.ABC) +``` + +#### ws + +```python +@cached_property +def ws() -> WorkspaceClient +``` + +Cached property to verify and return the workspace client. + +#### spark + +```python +@cached_property +def spark() -> SparkSession +``` + +Cached property return the SparkSession. + +## DQEngineCoreBase Objects + +```python +class DQEngineCoreBase(DQEngineBase) +``` + +#### apply\_checks + +```python +@abc.abstractmethod +def apply_checks(df: DataFrame, + checks: list[DQRule], + ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame +``` + +Applies data quality checks to a given dataframe. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. +- `ref_dfs`: reference dataframes to use in the checks, if applicable + +**Returns**: + +dataframe with errors and warning result columns + +#### apply\_checks\_and\_split + +```python +@abc.abstractmethod +def apply_checks_and_split( + df: DataFrame, + checks: list[DQRule], + ref_dfs: dict[str, DataFrame] | None = None +) -> tuple[DataFrame, DataFrame] +``` + +Applies data quality checks to a given dataframe and split it into two ("good" and "bad"), + +according to the data quality checks. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. +- `ref_dfs`: reference dataframes to use in the checks, if applicable + +**Returns**: + +two dataframes - "good" which includes warning rows but no result columns, and "data" having +error and warning rows and corresponding result columns + +#### apply\_checks\_by\_metadata\_and\_split + +```python +@abc.abstractmethod +def apply_checks_by_metadata_and_split( + df: DataFrame, + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None, + ref_dfs: dict[str, DataFrame] | None = None +) -> tuple[DataFrame, DataFrame] +``` + +Wrapper around `apply_checks_and_split` for use in the metadata-driven pipelines. The main difference + +is how the checks are specified - instead of using functions directly, they are described as function name plus +arguments. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - +it will be used as an error/warning message, or `null` if it's evaluated to `false` +* `name` - name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), +and `warn` (data is going into both dataframes) +- `df`0: dictionary with custom check functions (eg. `df`1globals()`df`1 of the calling module). +If not specified, then only built-in functions are used for the checks. +- `df`3: reference dataframes to use in the checks, if applicable + +**Returns**: + +two dataframes - "good" which includes warning rows but no result columns, and "bad" having +error and warning rows and corresponding result columns + +#### apply\_checks\_by\_metadata + +```python +@abc.abstractmethod +def apply_checks_by_metadata( + df: DataFrame, + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None, + ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame +``` + +Wrapper around `apply_checks` for use in the metadata-driven pipelines. The main difference + +is how the checks are specified - instead of using functions directly, they are described as function name plus +arguments. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - +it will be used as an error/warning message, or `null` if it's evaluated to `false` +* `name` - name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), +and `warn` (data is going into both dataframes) +- `df`0: dictionary with custom check functions (eg. `df`1globals()`df`1 of calling module). +If not specified, then only built-in functions are used for the checks. +- `df`3: reference dataframes to use in the checks, if applicable + +**Returns**: + +dataframe with errors and warning result columns + +#### validate\_checks + +```python +@staticmethod +@abc.abstractmethod +def validate_checks( + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None +) -> ChecksValidationStatus +``` + +Validate the input dict to ensure they conform to expected structure and types. + +Each check can be a dictionary. The function validates +the presence of required keys, the existence and callability of functions, and the types +of arguments passed to these functions. + +**Arguments**: + +- `checks`: List of checks to apply to the dataframe. Each check should be a dictionary. +- `custom_check_functions`: Optional dictionary with custom check functions. + +**Returns**: + +`ValidationStatus`: The validation status. + +#### get\_invalid + +```python +@abc.abstractmethod +def get_invalid(df: DataFrame) -> DataFrame +``` + +Get records that violate data quality checks (records with warnings and errors). +@param df: input DataFrame. +@return: dataframe with error and warning rows and corresponding result columns. + +#### get\_valid + +```python +@abc.abstractmethod +def get_valid(df: DataFrame) -> DataFrame +``` + +Get records that don't violate data quality checks (records with warnings but no errors). +@param df: input DataFrame. +@return: dataframe with warning rows but no result columns. + +#### load\_checks\_from\_local\_file + +```python +@staticmethod +@abc.abstractmethod +def load_checks_from_local_file(filepath: str) -> list[dict] +``` + +Load checks (dq rules) from a file (json or yaml) in the local file system. + +This does not require installation of DQX in the workspace. +The returning checks can be used as input for `apply_checks_by_metadata` function. + +**Arguments**: + +- `filepath`: path to a file containing the checks. + +**Returns**: + +list of dq rules + +#### save\_checks\_in\_local\_file + +```python +@staticmethod +@abc.abstractmethod +def save_checks_in_local_file(checks: list[dict], filepath: str) +``` + +Save checks (dq rules) to a file (yaml or json) in the local file system. + +**Arguments**: + +- `checks`: list of dq rules to save +- `filepath`: path to a file containing the checks. + diff --git a/docs/dqx/docs/reference/api/check_funcs.md b/docs/dqx/docs/reference/api/check_funcs.md new file mode 100644 index 000000000..163f7ac77 --- /dev/null +++ b/docs/dqx/docs/reference/api/check_funcs.md @@ -0,0 +1,821 @@ +--- +sidebar_label: check_funcs +title: databricks.labs.dqx.check_funcs +--- + +## DQPattern Objects + +```python +class DQPattern(Enum) +``` + +Enum class to represent DQ patterns used to match data in columns. + +#### make\_condition + +```python +def make_condition(condition: Column, message: Column | str, + alias: str) -> Column +``` + +Helper function to create a condition column. + +**Arguments**: + +- `condition`: condition expression. +Pass the check if the condition evaluates to False. +Fail the check if condition evaluates to True. +- `message`: message to output - it could be either `Column` object, or string constant +- `alias`: name for the resulting column + +**Returns**: + +an instance of `Column` type, that either returns string if condition is evaluated to `true`, +or `null` if condition is evaluated to `false` + +#### matches\_pattern + +```python +def matches_pattern(column: str | Column, pattern: DQPattern) -> Column +``` + +Checks whether the values in the input column match a given pattern. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `pattern`: pattern to match against + +**Returns**: + +Column object for condition + +#### is\_not\_null\_and\_not\_empty + +```python +@register_rule("row") +def is_not_null_and_not_empty(column: str | Column, + trim_strings: bool | None = False) -> Column +``` + +Checks whether the values in the input column are not null and not empty. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `trim_strings`: boolean flag to trim spaces from strings + +**Returns**: + +Column object for condition + +#### is\_not\_empty + +```python +@register_rule("row") +def is_not_empty(column: str | Column) -> Column +``` + +Checks whether the values in the input column are not empty (but may be null). + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression + +**Returns**: + +Column object for condition + +#### is\_not\_null + +```python +@register_rule("row") +def is_not_null(column: str | Column) -> Column +``` + +Checks whether the values in the input column are not null. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression + +**Returns**: + +Column object for condition + +#### is\_not\_null\_and\_is\_in\_list + +```python +@register_rule("row") +def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column +``` + +Checks whether the values in the input column are not null and present in the list of allowed values. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `allowed`: list of allowed values (actual values or Column objects) + +**Returns**: + +Column object for condition + +#### is\_in\_list + +```python +@register_rule("row") +def is_in_list(column: str | Column, allowed: list) -> Column +``` + +Checks whether the values in the input column are present in the list of allowed values + +(null values are allowed). + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `allowed`: list of allowed values (actual values or Column objects) + +**Returns**: + +Column object for condition + +#### sql\_expression + +```python +@register_rule("row") +def sql_expression(expression: str, + msg: str | None = None, + name: str | None = None, + negate: bool = False, + columns: list[str | Column] | None = None) -> Column +``` + +Checks whether the condition provided as an SQL expression is met. + +**Arguments**: + +- `expression`: SQL expression. Fail if expression evaluates to True, pass if it evaluates to False. +- `msg`: optional message of the `Column` type, automatically generated if None +- `name`: optional name of the resulting column, automatically generated if None +- `negate`: if the condition should be negated (true) or not. For example, "col is not null" will mark null +values as "bad". Although sometimes it's easier to specify it other way around "col is null" + negate set to False +- `columns`: optional list of columns to be used for reporting. Unused in the actual logic. + +**Returns**: + +new Column + +#### is\_older\_than\_col2\_for\_n\_days + +```python +@register_rule("row") +def is_older_than_col2_for_n_days(column1: str | Column, + column2: str | Column, + days: int = 0, + negate: bool = False) -> Column +``` + +Checks whether the values in one input column are at least N days older than the values in another column. + +**Arguments**: + +- `column1`: first column to check; can be a string column name or a column expression +- `column2`: second column to check; can be a string column name or a column expression +- `days`: number of days +- `negate`: if the condition should be negated (true) or not; if negated, the check will fail when values in the +first column are at least N days older than values in the second column + +**Returns**: + +new Column + +#### is\_older\_than\_n\_days + +```python +@register_rule("row") +def is_older_than_n_days(column: str | Column, + days: int, + curr_date: Column | None = None, + negate: bool = False) -> Column +``` + +Checks whether the values in the input column are at least N days older than the current date. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `days`: number of days +- `curr_date`: (optional) set current date +- `negate`: if the condition should be negated (true) or not; if negated, the check will fail when values in the +first column are at least N days older than values in the second column + +**Returns**: + +new Column + +#### is\_not\_in\_future + +```python +@register_rule("row") +def is_not_in_future(column: str | Column, + offset: int = 0, + curr_timestamp: Column | None = None) -> Column +``` + +Checks whether the values in the input column contain a timestamp that is not in the future, + +where 'future' is defined as current_timestamp + offset (in seconds). + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `offset`: offset (in seconds) to add to the current timestamp at time of execution +- `curr_timestamp`: (optional) set current timestamp + +**Returns**: + +new Column + +#### is\_not\_in\_near\_future + +```python +@register_rule("row") +def is_not_in_near_future(column: str | Column, + offset: int = 0, + curr_timestamp: Column | None = None) -> Column +``` + +Checks whether the values in the input column contain a timestamp that is not in the near future, + +where 'near future' is defined as greater than the current timestamp +but less than the current_timestamp + offset (in seconds). + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `offset`: offset (in seconds) to add to the current timestamp at time of execution +- `curr_timestamp`: (optional) set current timestamp + +**Returns**: + +new Column + +#### is\_not\_less\_than + +```python +@register_rule("row") +def is_not_less_than( + column: str | Column, + limit: int | datetime.date | datetime.datetime | str | Column | None = None +) -> Column +``` + +Checks whether the values in the input column are not less than the provided limit. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `limit`: limit to use in the condition as number, date, timestamp, column name or sql expression + +**Returns**: + +new Column + +#### is\_not\_greater\_than + +```python +@register_rule("row") +def is_not_greater_than( + column: str | Column, + limit: int | datetime.date | datetime.datetime | str | Column | None = None +) -> Column +``` + +Checks whether the values in the input column are not greater than the provided limit. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `limit`: limit to use in the condition as number, date, timestamp, column name or sql expression + +**Returns**: + +new Column + +#### is\_in\_range + +```python +@register_rule("row") +def is_in_range( + column: str | Column, + min_limit: int | datetime.date | datetime.datetime | str | Column + | None = None, + max_limit: int | datetime.date | datetime.datetime | str | Column + | None = None +) -> Column +``` + +Checks whether the values in the input column are in the provided limits (inclusive of both boundaries). + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `min_limit`: min limit to use in the condition as number, date, timestamp, column name or sql expression +- `max_limit`: max limit to use in the condition as number, date, timestamp, column name or sql expression + +**Returns**: + +new Column + +#### is\_not\_in\_range + +```python +@register_rule("row") +def is_not_in_range( + column: str | Column, + min_limit: int | datetime.date | datetime.datetime | str | Column + | None = None, + max_limit: int | datetime.date | datetime.datetime | str | Column + | None = None +) -> Column +``` + +Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries). + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `min_limit`: min limit to use in the condition as number, date, timestamp, column name or sql expression +- `max_limit`: min limit to use in the condition as number, date, timestamp, column name or sql expression + +**Returns**: + +new Column + +#### regex\_match + +```python +@register_rule("row") +def regex_match(column: str | Column, + regex: str, + negate: bool = False) -> Column +``` + +Checks whether the values in the input column matches a given regex. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `regex`: regex to check +- `negate`: if the condition should be negated (true) or not + +**Returns**: + +Column object for condition + +#### is\_not\_null\_and\_not\_empty\_array + +```python +@register_rule("row") +def is_not_null_and_not_empty_array(column: str | Column) -> Column +``` + +Checks whether the values in the array input column are not null and not empty. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression + +**Returns**: + +Column object for condition + +#### is\_valid\_date + +```python +@register_rule("row") +def is_valid_date(column: str | Column, + date_format: str | None = None) -> Column +``` + +Checks whether the values in the input column have valid date formats. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `date_format`: date format (e.g. 'yyyy-mm-dd') + +**Returns**: + +Column object for condition + +#### is\_valid\_timestamp + +```python +@register_rule("row") +def is_valid_timestamp(column: str | Column, + timestamp_format: str | None = None) -> Column +``` + +Checks whether the values in the input column have valid timestamp formats. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `timestamp_format`: timestamp format (e.g. 'yyyy-mm-dd HH:mm:ss') + +**Returns**: + +Column object for condition + +#### is\_valid\_ipv4\_address + +```python +@register_rule("row") +def is_valid_ipv4_address(column: str | Column) -> Column +``` + +Checks whether the values in the input column have valid IPv4 address formats. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression + +**Returns**: + +Column object for condition + +#### is\_ipv4\_address\_in\_cidr + +```python +@register_rule("row") +def is_ipv4_address_in_cidr(column: str | Column, cidr_block: str) -> Column +``` + +Checks if an IP column value falls within the given CIDR block. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression +- `cidr_block`: CIDR block string (e.g., '192.168.1.0/24') + +**Raises**: + +- `ValueError`: If cidr_block is not a valid string in CIDR notation. + +**Returns**: + +Column object for condition + +#### is\_data\_fresh + +```python +@register_rule("row") +def is_data_fresh( + column: str | Column, + max_age_minutes: int, + base_timestamp: str | datetime.date | datetime.datetime | Column + | None = None +) -> Column +``` + +Checks whether the values in the timestamp column are not older than the specified number of minutes from the base timestamp column. + +This is useful for identifying stale data due to delayed pipelines and helps catch upstream issues early. + +**Arguments**: + +- `column`: column to check; can be a string column name or a column expression containing timestamp values +- `max_age_minutes`: maximum age in minutes before data is considered stale +- `base_timestamp`: (optional) set base timestamp column from which the stale check is calculated, if not provided uses current_timestamp() + +**Returns**: + +Column object for condition + +#### is\_unique + +```python +@register_rule("dataset") +def is_unique(columns: list[str | Column], + nulls_distinct: bool = True, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Build a uniqueness check condition and closure for dataset-level validation. + +This function checks whether the specified columns contain unique values within the dataset +and reports rows with duplicate combinations. When `nulls_distinct` +is True (default), rows with NULLs are treated as distinct (SQL ANSI behavior); otherwise, +NULLs are treated as equal when checking for duplicates. + +In streaming, uniqueness is validated within individual micro-batches only. + +**Arguments**: + +- `columns`: List of column names (str) or Spark Column expressions to validate for uniqueness. +- `nulls_distinct`: Whether NULLs are treated as distinct (default: True). +- `row_filter`: Optional SQL expression for filtering rows before checking uniqueness. +Auto-injected from the check filter. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for uniqueness violations. +- A closure that applies the uniqueness check and adds the necessary condition/count columns. + +#### foreign\_key + +```python +@register_rule("dataset") +def foreign_key(columns: list[str | Column], + ref_columns: list[str | Column], + ref_df_name: str | None = None, + ref_table: str | None = None, + negate: bool = False, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Build a foreign key check condition and closure for dataset-level validation. + +This function verifies that values in the specified foreign key columns exist (or don't exist, if `negate=True`) in +the corresponding reference columns of another DataFrame or table. Rows where +foreign key values do not match the reference are reported as violations. + +NULL values in the foreign key columns are ignored (SQL ANSI behavior). + +**Arguments**: + +- `columns`: List of column names (str) or Column expressions in the dataset (foreign key). +- `ref_columns`: List of column names (str) or Column expressions in the reference dataset. +- `ref_df_name`: Name of the reference DataFrame (used when passing DataFrames directly). +- `ref_table`: Name of the reference table (used when reading from catalog). +- `row_filter`: Optional SQL expression for filtering rows before checking the foreign key. +Auto-injected from the check filter. +- `negate`: If True, the condition is negated (i.e., the check fails when the foreign key values exist in the +reference DataFrame/Table). If False, the check fails when the foreign key values do not exist in the reference. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for foreign key violations. +- A closure that applies the foreign key validation by joining against the reference. + +#### sql\_query + +```python +@register_rule("dataset") +def sql_query(query: str, + merge_columns: list[str], + msg: str | None = None, + name: str | None = None, + negate: bool = False, + condition_column: str = "condition", + input_placeholder: str = "input_view", + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Checks whether the condition column generated by SQL query is met. + +**Arguments**: + +- `query`: SQL query that must return as a minimum a condition column and +all merge columns. The resulting DataFrame is automatically joined back to the input DataFrame. +using the merge_columns. Reference DataFrames when provided in the ref_dfs parameter are registered as temp view. +- `condition_column`: Column name indicating violation (boolean). Fail the check if True, pass it if False +- `merge_columns`: List of columns for join back to the input DataFrame. +They must provide a unique key for the join, otherwise a duplicate records may be produced. +- `msg`: Optional custom message or Column expression. +- `name`: Optional name for the result. +- `negate`: If True, the condition is negated (i.e., the check fails when the condition is False). +- `input_placeholder`: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the +input DataFrame on which the checks are applied. +- `row_filter`: Optional SQL expression for filtering rows before checking the foreign key. +Auto-injected from the check filter. + +**Returns**: + +Tuple (condition column, apply function). + +#### is\_aggr\_not\_greater\_than + +```python +@register_rule("dataset") +def is_aggr_not_greater_than( + column: str | Column, + limit: int | float | str | Column, + aggr_type: str = "count", + group_by: list[str | Column] | None = None, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Build an aggregation check condition and closure for dataset-level validation. + +This function verifies that an aggregation (count, sum, avg, min, max) on a column +or group of columns does not exceed a specified limit. Rows where the aggregation +result exceeds the limit are flagged. + +**Arguments**: + +- `column`: Column name (str) or Column expression to aggregate. +- `limit`: Numeric value, column name, or SQL expression for the limit. +- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). +- `group_by`: Optional list of column names or Column expressions to group by. +- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for aggregation limit violations. +- A closure that applies the aggregation check and adds the necessary condition/metric columns. + +#### is\_aggr\_not\_less\_than + +```python +@register_rule("dataset") +def is_aggr_not_less_than( + column: str | Column, + limit: int | float | str | Column, + aggr_type: str = "count", + group_by: list[str | Column] | None = None, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Build an aggregation check condition and closure for dataset-level validation. + +This function verifies that an aggregation (count, sum, avg, min, max) on a column +or group of columns is not below a specified limit. Rows where the aggregation +result is below the limit are flagged. + +**Arguments**: + +- `column`: Column name (str) or Column expression to aggregate. +- `limit`: Numeric value, column name, or SQL expression for the limit. +- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). +- `group_by`: Optional list of column names or Column expressions to group by. +- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for aggregation limit violations. +- A closure that applies the aggregation check and adds the necessary condition/metric columns. + +#### is\_aggr\_equal + +```python +@register_rule("dataset") +def is_aggr_equal(column: str | Column, + limit: int | float | str | Column, + aggr_type: str = "count", + group_by: list[str | Column] | None = None, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Build an aggregation check condition and closure for dataset-level validation. + +This function verifies that an aggregation (count, sum, avg, min, max) on a column +or group of columns is equal to a specified limit. Rows where the aggregation +result is not equal to the limit are flagged. + +**Arguments**: + +- `column`: Column name (str) or Column expression to aggregate. +- `limit`: Numeric value, column name, or SQL expression for the limit. +- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). +- `group_by`: Optional list of column names or Column expressions to group by. +- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for aggregation limit violations. +- A closure that applies the aggregation check and adds the necessary condition/metric columns. + +#### is\_aggr\_not\_equal + +```python +@register_rule("dataset") +def is_aggr_not_equal( + column: str | Column, + limit: int | float | str | Column, + aggr_type: str = "count", + group_by: list[str | Column] | None = None, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Build an aggregation check condition and closure for dataset-level validation. + +This function verifies that an aggregation (count, sum, avg, min, max) on a column +or group of columns is not equal to a specified limit. Rows where the aggregation +result is equal to the limit are flagged. + +**Arguments**: + +- `column`: Column name (str) or Column expression to aggregate. +- `limit`: Numeric value, column name, or SQL expression for the limit. +- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). +- `group_by`: Optional list of column names or Column expressions to group by. +- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for aggregation limit violations. +- A closure that applies the aggregation check and adds the necessary condition/metric columns. + +#### compare\_datasets + +```python +@register_rule("dataset") +def compare_datasets(columns: list[str | Column], + ref_columns: list[str | Column], + ref_df_name: str | None = None, + ref_table: str | None = None, + check_missing_records: bool | None = False, + exclude_columns: list[str | Column] | None = None, + null_safe_row_matching: bool | None = True, + null_safe_column_value_matching: bool | None = True, + row_filter: str | None = None) -> tuple[Column, Callable] +``` + +Dataset-level check that compares two datasets and returns a condition for changed rows, + +with details on a row and column-level differences. +Only columns that are common across both datasets will be compared. Mismatched columns are ignored. +Detailed information about the differences is provided in the condition column. +The comparison does not support Map types (any column comparison on map type is skipped automatically). + +The log containing detailed differences is written to the message field of the check result as JSON string. +Example: `{"row_missing":false,"row_extra":true,"changed":{"val":{"df":"val1"}}}` + +**Arguments**: + +- `columns`: List of columns to use for row matching with the reference DataFrame +(can be a list of string column names or column expressions). +Only simple column expressions are supported, e.g. F.col("col_name") +- `ref_columns`: List of columns in the reference DataFrame or Table to row match against the source DataFrame +(can be a list of string column names or column expressions). The `columns` parameter is matched with `ref_columns` +by position, so the order of the provided columns in both lists must be exactly aligned. +Only simple column expressions are supported, e.g. F.col("col_name") +- `ref_df_name`: Name of the reference DataFrame (used when passing DataFrames directly). +- `ref_table`: Name of the reference table (used when reading from catalog). +- `check_missing_records`: Perform FULL OUTER JOIN between the DataFrames to find also records +that could be missing from the DataFrame. Use this with caution as it may produce output with more rows +than in the original DataFrame. +- `exclude_columns`: List of columns to exclude from the value comparison but not from row matching +(can be a list of string column names or column expressions). +Only simple column expressions are supported, e.g. F.col("col_name") +The parameter does not alter the list of columns used to determine row matches, +it only controls which columns are skipped during the column value comparison. +- `null_safe_row_matching`: If True, treats nulls as equal when matching rows. +- `columns`0: If True, treats nulls as equal when matching column values. +If enabled (NULL, NULL) column values are equal and matching. +- `columns`1: Optional SQL expression to filter rows in the input DataFrame. +Auto-injected from the check filter. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for comparison violations. +- A closure that applies the comparison validation. + +#### is\_data\_fresh\_per\_time\_window + +```python +@register_rule("dataset") +def is_data_fresh_per_time_window( + column: str | Column, + window_minutes: int, + min_records_per_window: int, + lookback_windows: int | None = None, + row_filter: str | None = None, + curr_timestamp: Column | None = None) -> tuple[Column, Callable] +``` + +Build a completeness freshness check that validates records arrive at least every X minutes + +with a threshold for the expected number of rows per time window. + +If `lookback_windows` is provided, only data within that lookback period will be validated. +If omitted, the entire dataset will be checked. + +**Arguments**: + +- `column`: Column name (str) or Column expression containing timestamps to check. +- `window_minutes`: Time window in minutes to check for data arrival. +- `min_records_per_window`: Minimum number of records expected per time window. +- `lookback_windows`: Optional number of time windows to look back from `curr_timestamp`. +This filters records to include only those within the specified number of time windows from `curr_timestamp`. +If no lookback is provided, the check is applied to the entire dataset. +- `row_filter`: Optional SQL expression to filter rows before checking. +- `curr_timestamp`: Optional current timestamp column. If not provided, current_timestamp() function is used. + +**Returns**: + +A tuple of: +- A Spark Column representing the condition for missing data within a time window. +- A closure that applies the completeness check and adds the necessary condition columns. + diff --git a/docs/dqx/docs/reference/api/checks_resolver.md b/docs/dqx/docs/reference/api/checks_resolver.md new file mode 100644 index 000000000..7d02bf610 --- /dev/null +++ b/docs/dqx/docs/reference/api/checks_resolver.md @@ -0,0 +1,26 @@ +--- +sidebar_label: checks_resolver +title: databricks.labs.dqx.checks_resolver +--- + +#### resolve\_check\_function + +```python +def resolve_check_function(function_name: str, + custom_check_functions: dict[str, Any] + | None = None, + fail_on_missing: bool = True) -> Callable | None +``` + +Resolves a function by name from the predefined functions and custom checks. + +**Arguments**: + +- `function_name`: name of the function to resolve. +- `custom_check_functions`: dictionary with custom check functions (eg. ``globals()`` of the calling module). +- `fail_on_missing`: if True, raise an AttributeError if the function is not found. + +**Returns**: + +function or None if not found. + diff --git a/docs/dqx/docs/reference/api/checks_serializer.md b/docs/dqx/docs/reference/api/checks_serializer.md new file mode 100644 index 000000000..4eb56c3ef --- /dev/null +++ b/docs/dqx/docs/reference/api/checks_serializer.md @@ -0,0 +1,140 @@ +--- +sidebar_label: checks_serializer +title: databricks.labs.dqx.checks_serializer +--- + +#### serialize\_checks\_from\_dataframe + +```python +def serialize_checks_from_dataframe(df: DataFrame, + run_config_name: str = "default" + ) -> list[dict] +``` + +Converts a list of quality checks defined in a DataFrame to a list of quality checks + +defined as Python dictionaries. + +**Arguments**: + +- `df`: DataFrame with data quality check rules. Each row should define a check. Rows should +have the following columns: +* `name` - Name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` (data is going into both dataframes) +* `check` - DQX check function used in the check; A `StructType` column defining the data quality check +* `filter` - Expression for filtering data quality checks +* `run_config_name` (optional) - Run configuration name for storing checks across runs +* `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. +- `run_config_name`: Run configuration name for filtering quality rules + +**Returns**: + +List of data quality check specifications as a Python dictionary + +#### deserialize\_checks\_to\_dataframe + +```python +def deserialize_checks_to_dataframe( + spark: SparkSession, + checks: list[dict], + run_config_name: str = "default") -> DataFrame +``` + +Converts a list of quality checks defined as Python dictionaries to a DataFrame. + +**Arguments**: + +- `spark`: Spark session. +- `checks`: list of check specifications as Python dictionaries. Each check consists of the following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to + true (it will be used as an error/warning message) or `null` if it's evaluated to `false` +* `name` - Name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` + (data is going into both dataframes) +* `filter` (optional) - Expression for filtering data quality checks +* `checks`0 (optional) - User-defined key-value pairs added to metadata generated by the check. +- `checks`1: Run configuration name for storing quality checks across runs + +**Returns**: + +DataFrame with data quality check rules + +#### deserialize\_checks + +```python +def deserialize_checks( + checks: list[dict], + custom_checks: dict[str, Any] | None = None) -> list[DQRule] +``` + +Converts a list of quality checks defined as Python dictionaries to a list of `DQRule` objects. + +**Arguments**: + +- `checks`: list of dictionaries describing checks. Each check is a dictionary +consisting of following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true +- it will be used as an error/warning message, or `null` if it's evaluated to `false` +* `name` - name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), +and `warn` (data is going into both dataframes) +* `filter` (optional) - Expression for filtering data quality checks +* `checks`0 (optional) - User-defined key-value pairs added to metadata generated by the check. +- `checks`1: dictionary with custom check functions (eg. `checks`2globals()`checks`2 of the calling module). +If not specified, then only built-in functions are used for the checks. + +**Returns**: + +list of data quality check rules + +#### serialize\_checks + +```python +def serialize_checks(checks: list[DQRule]) -> list[dict] +``` + +Converts a list of quality checks defined as `DQRule` objects to a list of quality checks + +defined as Python dictionaries. + +**Arguments**: + +- `checks`: List of DQRule instances to convert. + +**Returns**: + +List of dictionaries representing the DQRule instances. + +#### serialize\_checks\_to\_bytes + +```python +def serialize_checks_to_bytes(checks: list[dict], file_path: Path) -> bytes +``` + +Serializes a list of checks to bytes in json or yaml (default) format. + +**Arguments**: + +- `checks`: List of checks to serialize. +- `file_path`: Path to the file where the checks will be serialized. + +**Returns**: + +Serialized checks as bytes. + +#### get\_file\_deserializer + +```python +def get_file_deserializer(filepath: str) -> Callable +``` + +Get the deserializer function based on file. + +**Arguments**: + +- `filepath`: Path to the file. + +**Returns**: + +Deserializer function. + diff --git a/docs/dqx/docs/reference/api/checks_storage.md b/docs/dqx/docs/reference/api/checks_storage.md new file mode 100644 index 000000000..c7e874775 --- /dev/null +++ b/docs/dqx/docs/reference/api/checks_storage.md @@ -0,0 +1,308 @@ +--- +sidebar_label: checks_storage +title: databricks.labs.dqx.checks_storage +--- + +## ChecksStorageHandler Objects + +```python +class ChecksStorageHandler(ABC, Generic[T]) +``` + +Abstract base class for handling storage of quality rules (checks). + +#### load + +```python +@abstractmethod +def load(config: T) -> list[dict] +``` + +Load quality rules from the source. + +The returned checks can be used as input for `apply_checks_by_metadata` or +`apply_checks_by_metadata_and_split` functions. + +**Arguments**: + +- `config`: configuration for loading checks, including the table location and run configuration name. + +**Returns**: + +list of dq rules or raise an error if checks file is missing or is invalid. + +#### save + +```python +@abstractmethod +def save(checks: list[dict], config: T) -> None +``` + +Save quality rules to the target. + +## TableChecksStorageHandler Objects + +```python +class TableChecksStorageHandler(ChecksStorageHandler[TableChecksStorageConfig] + ) +``` + +Handler for storing quality rules (checks) in a Delta table in the workspace. + +#### load + +```python +def load(config: TableChecksStorageConfig) -> list[dict] +``` + +Load checks (dq rules) from a Delta table in the workspace. + +**Arguments**: + +- `config`: configuration for loading checks, including the table location and run configuration name. + +**Returns**: + +list of dq rules or raise an error if checks table is missing or is invalid. + +#### save + +```python +def save(checks: list[dict], config: TableChecksStorageConfig) -> None +``` + +Save checks to a Delta table in the workspace. + +**Arguments**: + +- `checks`: list of dq rules to save +- `config`: configuration for saving checks, including the table location and run configuration name. + +**Raises**: + +- `ValueError`: if the table name is not provided + +## WorkspaceFileChecksStorageHandler Objects + +```python +class WorkspaceFileChecksStorageHandler( + ChecksStorageHandler[WorkspaceFileChecksStorageConfig]) +``` + +Handler for storing quality rules (checks) in a file (json or yaml) in the workspace. + +#### load + +```python +def load(config: WorkspaceFileChecksStorageConfig) -> list[dict] +``` + +Load checks (dq rules) from a file (json or yaml) in the workspace. + +This does not require installation of DQX in the workspace. + +**Arguments**: + +- `config`: configuration for loading checks, including the file location and storage type. + +**Returns**: + +list of dq rules or raise an error if checks file is missing or is invalid. + +#### save + +```python +def save(checks: list[dict], config: WorkspaceFileChecksStorageConfig) -> None +``` + +Save checks (dq rules) to yaml file in the workspace. + +This does not require installation of DQX in the workspace. + +**Arguments**: + +- `checks`: list of dq rules to save +- `config`: configuration for saving checks, including the file location and storage type. + +## FileChecksStorageHandler Objects + +```python +class FileChecksStorageHandler(ChecksStorageHandler[FileChecksStorageConfig]) +``` + +Handler for storing quality rules (checks) in a file (json or yaml) in the local filesystem. + +#### load + +```python +def load(config: FileChecksStorageConfig) -> list[dict] +``` + +Load checks (dq rules) from a file (json or yaml) in the local filesystem. + +**Arguments**: + +- `config`: configuration for loading checks, including the file location. + +**Raises**: + +- `ValueError`: if the file path is not provided +- `FileNotFoundError`: if the file path does not exist + +**Returns**: + +list of dq rules or raise an error if checks file is missing or is invalid. + +#### save + +```python +def save(checks: list[dict], config: FileChecksStorageConfig) -> None +``` + +Save checks (dq rules) to a file (json or yaml) in the local filesystem. + +**Arguments**: + +- `checks`: list of dq rules to save +- `config`: configuration for saving checks, including the file location. + +**Raises**: + +- `ValueError`: if the file path is not provided +- `FileNotFoundError`: if the file path does not exist + +## InstallationChecksStorageHandler Objects + +```python +class InstallationChecksStorageHandler( + ChecksStorageHandler[InstallationChecksStorageConfig]) +``` + +Handler for storing quality rules (checks) defined in the installation configuration. + +#### load + +```python +def load(config: InstallationChecksStorageConfig) -> list[dict] +``` + +Load checks (dq rules) from the installation configuration. + +**Arguments**: + +- `config`: configuration for loading checks, including the run configuration name and method. + +**Raises**: + +- `NotFound`: if the checks file or table is not found in the installation. + +**Returns**: + +list of dq rules or raise an error if checks file is missing or is invalid. + +#### save + +```python +def save(checks: list[dict], config: InstallationChecksStorageConfig) -> None +``` + +Save checks (dq rules) to yaml file or table in the installation folder. + +This will overwrite existing checks file or table. + +**Arguments**: + +- `checks`: list of dq rules to save +- `config`: configuration for saving checks, including the run configuration name, method, and table location. + +## VolumeFileChecksStorageHandler Objects + +```python +class VolumeFileChecksStorageHandler( + ChecksStorageHandler[VolumeFileChecksStorageConfig]) +``` + +Handler for storing quality rules (checks) in a file (json or yaml) in a Unity Catalog volume. + +#### load + +```python +def load(config: VolumeFileChecksStorageConfig) -> list[dict] +``` + +Load checks (dq rules) from a file (json or yaml) in a Unity Catalog volume. + +**Arguments**: + +- `config`: configuration for loading checks, including the file location and storage type. + +**Returns**: + +list of dq rules or raise an error if checks file is missing or is invalid. + +#### save + +```python +def save(checks: list[dict], config: VolumeFileChecksStorageConfig) -> None +``` + +Save checks (dq rules) to yaml file in a Unity Catalog volume. + +This does not require installation of DQX in a Unity Catalog volume. + +**Arguments**: + +- `checks`: list of dq rules to save +- `config`: configuration for saving checks, including the file location and storage type. + +## BaseChecksStorageHandlerFactory Objects + +```python +class BaseChecksStorageHandlerFactory(ABC) +``` + +Abstract base class for factories that create storage handlers for checks. + +#### create + +```python +@abstractmethod +def create(config: BaseChecksStorageConfig) -> ChecksStorageHandler +``` + +Abstract method to create a handler based on the type of the provided configuration object. + +**Arguments**: + +- `config`: Configuration object for loading or saving checks. + +**Returns**: + +An instance of the corresponding BaseChecksStorageHandler. + +## ChecksStorageHandlerFactory Objects + +```python +class ChecksStorageHandlerFactory(BaseChecksStorageHandlerFactory) +``` + +#### create + +```python +def create(config: BaseChecksStorageConfig) -> ChecksStorageHandler +``` + +Factory method to create a handler based on the type of the provided configuration object. + +**Arguments**: + +- `config`: Configuration object for loading or saving checks. + +**Raises**: + +- `ValueError`: If the configuration type is unsupported. + +**Returns**: + +An instance of the corresponding BaseChecksStorageHandler. + diff --git a/docs/dqx/docs/reference/api/checks_validator.md b/docs/dqx/docs/reference/api/checks_validator.md new file mode 100644 index 000000000..0b064a4cf --- /dev/null +++ b/docs/dqx/docs/reference/api/checks_validator.md @@ -0,0 +1,72 @@ +--- +sidebar_label: checks_validator +title: databricks.labs.dqx.checks_validator +--- + +## ChecksValidationStatus Objects + +```python +@dataclass(frozen=True) +class ChecksValidationStatus() +``` + +Class to represent the validation status. + +#### add\_error + +```python +def add_error(error: str) +``` + +Add an error to the validation status. + +#### add\_errors + +```python +def add_errors(errors: list[str]) +``` + +Add an error to the validation status. + +#### has\_errors + +```python +@property +def has_errors() -> bool +``` + +Check if there are any errors in the validation status. + +#### errors + +```python +@property +def errors() -> list[str] +``` + +Get the list of errors in the validation status. + +#### to\_string + +```python +def to_string() -> str +``` + +Convert the validation status to a string. + +#### \_\_str\_\_ + +```python +def __str__() -> str +``` + +String representation of the ValidationStatus class. + +## ChecksValidator Objects + +```python +class ChecksValidator() +``` + +Class to validate quality rules (checks). + diff --git a/docs/dqx/docs/reference/api/cli.md b/docs/dqx/docs/reference/api/cli.md new file mode 100644 index 000000000..276eb9130 --- /dev/null +++ b/docs/dqx/docs/reference/api/cli.md @@ -0,0 +1,123 @@ +--- +sidebar_label: cli +title: databricks.labs.dqx.cli +--- + +#### open\_remote\_config + +```python +@dqx.command +def open_remote_config(w: WorkspaceClient, + *, + ctx: WorkspaceContext | None = None) +``` + +Opens remote configuration in the browser. + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `ctx`: The WorkspaceContext instance to use for accessing the workspace. + +#### open\_dashboards + +```python +@dqx.command +def open_dashboards(w: WorkspaceClient, + *, + ctx: WorkspaceContext | None = None) +``` + +Opens remote dashboard directory in the browser. + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `ctx`: The WorkspaceContext instance to use for accessing the workspace. + +#### installations + +```python +@dqx.command +def installations(w: WorkspaceClient, + *, + product_name: str = "dqx") -> list[dict] +``` + +Show installations by different users on the same workspace. + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `product_name`: The name of the product to search for in the installation folder. + +#### validate\_checks + +```python +@dqx.command +def validate_checks(w: WorkspaceClient, + *, + run_config: str = "default", + validate_custom_check_functions: bool = True, + ctx: WorkspaceContext | None = None) -> list[dict] +``` + +Validate checks stored in the installation directory as a file. + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `run_config`: The name of the run configuration to use. +- `validate_custom_check_functions`: Whether to validate custom check functions (default is True). +- `ctx`: The WorkspaceContext instance to use for accessing the workspace. + +#### profile + +```python +@dqx.command +def profile(w: WorkspaceClient, + *, + run_config: str = "default", + ctx: WorkspaceContext | None = None) -> None +``` + +Profile input data and generate quality rule (checks) candidates. + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `run_config`: The name of the run configuration to use. +- `ctx`: The WorkspaceContext instance to use for accessing the workspace. + +#### workflows + +```python +@dqx.command +def workflows(w: WorkspaceClient, *, ctx: WorkspaceContext | None = None) +``` + +Show deployed workflows and their state + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `ctx`: The WorkspaceContext instance to use for accessing the workspace. + +#### logs + +```python +@dqx.command +def logs(w: WorkspaceClient, + *, + workflow: str | None = None, + ctx: WorkspaceContext | None = None) +``` + +Show logs of the latest job run. + +**Arguments**: + +- `w`: The WorkspaceClient instance to use for accessing the workspace. +- `workflow`: The name of the workflow to show logs for. +- `ctx`: The WorkspaceContext instance to use for accessing the workspace + diff --git a/docs/dqx/docs/reference/api/config.md b/docs/dqx/docs/reference/api/config.md new file mode 100644 index 000000000..da4df925a --- /dev/null +++ b/docs/dqx/docs/reference/api/config.md @@ -0,0 +1,198 @@ +--- +sidebar_label: config +title: databricks.labs.dqx.config +--- + +## InputConfig Objects + +```python +@dataclass +class InputConfig() +``` + +Configuration class for input data sources (e.g. tables or files). + +## OutputConfig Objects + +```python +@dataclass +class OutputConfig() +``` + +Configuration class for output data sinks (e.g. tables or files). + +## ProfilerConfig Objects + +```python +@dataclass +class ProfilerConfig() +``` + +Configuration class for profiler. + +#### summary\_stats\_file + +file containing profile summary statistics + +#### sample\_fraction + +fraction of data to sample (30%) + +#### sample\_seed + +seed for sampling + +#### limit + +limit the number of records to profile + +## RunConfig Objects + +```python +@dataclass +class RunConfig() +``` + +Configuration class for the data quality checks + +#### name + +name of the run configuration + +#### quarantine\_config + +quarantined data table + +#### checks\_location + +relative workspace file path or table containing quality rules / checks + +#### warehouse\_id + +warehouse id to use in the dashboard + +## WorkspaceConfig Objects + +```python +@dataclass +class WorkspaceConfig() +``` + +Configuration class for the workspace + +#### get\_run\_config + +```python +def get_run_config(run_config_name: str | None = "default") -> RunConfig +``` + +Get the run configuration for a given run name, or the default configuration if no run name is provided. + +**Arguments**: + +- `run_config_name`: The name of the run configuration to get. + +**Raises**: + +- `ValueError`: If no run configurations are available or if the specified run configuration name is +not found. + +**Returns**: + +The run configuration. + +## BaseChecksStorageConfig Objects + +```python +@dataclass +class BaseChecksStorageConfig(abc.ABC) +``` + +Marker base class for storage configuration. + +## FileChecksStorageConfig Objects + +```python +@dataclass +class FileChecksStorageConfig(BaseChecksStorageConfig) +``` + +Configuration class for storing checks in a file. + +**Arguments**: + +- `location`: The file path where the checks are stored. + +## WorkspaceFileChecksStorageConfig Objects + +```python +@dataclass +class WorkspaceFileChecksStorageConfig(BaseChecksStorageConfig) +``` + +Configuration class for storing checks in a workspace file. + +**Arguments**: + +- `location`: The workspace file path where the checks are stored. + +## TableChecksStorageConfig Objects + +```python +@dataclass +class TableChecksStorageConfig(BaseChecksStorageConfig) +``` + +Configuration class for storing checks in a table. + +**Arguments**: + +- `location`: The table name where the checks are stored. +- `run_config_name`: The name of the run configuration to use for checks (default is 'default'). +- `mode`: The mode for writing checks to a table (e.g., 'append' or 'overwrite'). +The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. + +#### run\_config\_name + +to filter checks by run config + +## VolumeFileChecksStorageConfig Objects + +```python +@dataclass +class VolumeFileChecksStorageConfig(BaseChecksStorageConfig) +``` + +Configuration class for storing checks in a Unity Catalog volume file. + +**Arguments**: + +- `location`: The Unity Catalog volume file path where the checks are stored. + +## InstallationChecksStorageConfig Objects + +```python +@dataclass +class InstallationChecksStorageConfig(WorkspaceFileChecksStorageConfig, + TableChecksStorageConfig, + VolumeFileChecksStorageConfig) +``` + +Configuration class for storing checks in an installation. + +**Arguments**: + +- `location`: The installation path where the checks are stored (e.g., table name, file path). +Not used when using installation method, as it is retrieved from the installation config. +- `run_config_name`: The name of the run configuration to use for checks (default is 'default'). +- `product_name`: The product name for retrieving checks from the installation (default is 'dqx'). +- `assume_user`: Whether to assume the user is the owner of the checks (default is True). + +#### location + +retrieved from the installation config + +#### run\_config\_name + +to retrieve run config + diff --git a/docs/dqx/docs/reference/api/config_loader.md b/docs/dqx/docs/reference/api/config_loader.md new file mode 100644 index 000000000..d40a97d5e --- /dev/null +++ b/docs/dqx/docs/reference/api/config_loader.md @@ -0,0 +1,42 @@ +--- +sidebar_label: config_loader +title: databricks.labs.dqx.config_loader +--- + +## RunConfigLoader Objects + +```python +class RunConfigLoader() +``` + +Class to handle loading of run configurations from the installation. + +#### load\_run\_config + +```python +def load_run_config(run_config_name: str | None, + assume_user: bool = True, + product_name: str = "dqx") -> RunConfig +``` + +Load run configuration from the installation. + +**Arguments**: + +- `run_config_name`: name of the run configuration to use +- `assume_user`: if True, assume user installation +- `product_name`: name of the product + +#### get\_installation + +```python +def get_installation(assume_user: bool, product_name: str) -> Installation +``` + +Get the installation for the given product name. + +**Arguments**: + +- `assume_user`: if True, assume user installation +- `product_name`: name of the product + diff --git a/docs/dqx/docs/reference/api/contexts/application.md b/docs/dqx/docs/reference/api/contexts/application.md new file mode 100644 index 000000000..34894cf5f --- /dev/null +++ b/docs/dqx/docs/reference/api/contexts/application.md @@ -0,0 +1,41 @@ +--- +sidebar_label: application +title: databricks.labs.dqx.contexts.application +--- + +## GlobalContext Objects + +```python +class GlobalContext(abc.ABC) +``` + +Returns the parent run ID. + +**Returns**: + +The parent run ID as an integer. + +#### replace + +```python +def replace(**kwargs) +``` + +Replace cached properties. + +**Arguments**: + +- `kwargs`: Key-value pairs of properties to replace. + +**Returns**: + +The updated GlobalContext instance. + +## CliContext Objects + +```python +class CliContext(GlobalContext, abc.ABC) +``` + +Abstract base class for global context, providing common properties and methods for workspace management. + diff --git a/docs/dqx/docs/reference/api/contexts/workflows.md b/docs/dqx/docs/reference/api/contexts/workflows.md new file mode 100644 index 000000000..ac5e76e9a --- /dev/null +++ b/docs/dqx/docs/reference/api/contexts/workflows.md @@ -0,0 +1,91 @@ +--- +sidebar_label: workflows +title: databricks.labs.dqx.contexts.workflows +--- + +## RuntimeContext Objects + +```python +class RuntimeContext(GlobalContext) +``` + +#### config + +```python +@cached_property +def config() -> WorkspaceConfig +``` + +Loads and returns the workspace configuration. + +#### run\_config + +```python +@cached_property +def run_config() -> RunConfig +``` + +Loads and returns the run configuration. + +#### connect\_config + +```python +@cached_property +def connect_config() -> core.Config +``` + +Returns the connection configuration. + +**Raises**: + +- `AssertionError`: If the connect configuration is not provided. + +**Returns**: + +The core.Config instance. + +#### workspace\_client + +```python +@cached_property +def workspace_client() -> WorkspaceClient +``` + +Returns the WorkspaceClient instance. + +#### installation + +```python +@cached_property +def installation() -> Installation +``` + +Returns the installation instance for the runtime. + +#### workspace\_id + +```python +@cached_property +def workspace_id() -> int +``` + +Returns the workspace ID. + +#### parent\_run\_id + +```python +@cached_property +def parent_run_id() -> int +``` + +Returns the parent run ID. + +#### profiler + +```python +@cached_property +def profiler() -> ProfilerRunner +``` + +Returns the ProfilerRunner instance. + diff --git a/docs/dqx/docs/reference/api/contexts/workspace.md b/docs/dqx/docs/reference/api/contexts/workspace.md new file mode 100644 index 000000000..e0cc24aee --- /dev/null +++ b/docs/dqx/docs/reference/api/contexts/workspace.md @@ -0,0 +1,27 @@ +--- +sidebar_label: workspace +title: databricks.labs.dqx.contexts.workspace +--- + +## WorkspaceContext Objects + +```python +class WorkspaceContext(CliContext) +``` + +WorkspaceContext class that extends CliContext to provide workspace-specific functionality. + +**Arguments**: + +- `ws`: The WorkspaceClient instance to use for accessing the workspace. +- `named_parameters`: Optional dictionary of named parameters. + +#### workspace\_client + +```python +@cached_property +def workspace_client() -> WorkspaceClient +``` + +Returns the WorkspaceClient instance. + diff --git a/docs/dqx/docs/reference/api/engine.md b/docs/dqx/docs/reference/api/engine.md new file mode 100644 index 000000000..15eb1d961 --- /dev/null +++ b/docs/dqx/docs/reference/api/engine.md @@ -0,0 +1,318 @@ +--- +sidebar_label: engine +title: databricks.labs.dqx.engine +--- + +## DQEngineCore Objects + +```python +class DQEngineCore(DQEngineCoreBase) +``` + +Data Quality Engine Core class to apply data quality checks to a given dataframe. + +**Arguments**: + +- `workspace_client` _WorkspaceClient_ - WorkspaceClient instance to use for accessing the workspace. +- `extra_params` _ExtraParams_ - Extra parameters for the DQEngine. + +## DQEngine Objects + +```python +class DQEngine(DQEngineBase) +``` + +Data Quality Engine class to apply data quality checks to a given dataframe. + +#### apply\_checks + +```python +def apply_checks(df: DataFrame, + checks: list[DQRule], + ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame +``` + +Applies data quality checks to a given dataframe. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. +- `ref_dfs`: reference dataframes to use in the checks, if applicable + +**Returns**: + +dataframe with errors and warning result columns + +#### apply\_checks\_and\_split + +```python +def apply_checks_and_split( + df: DataFrame, + checks: list[DQRule], + ref_dfs: dict[str, DataFrame] | None = None +) -> tuple[DataFrame, DataFrame] +``` + +Applies data quality checks to a given dataframe and split it into two ("good" and "bad"), + +according to the data quality checks. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. +- `ref_dfs`: reference dataframes to use in the checks, if applicable + +**Returns**: + +two dataframes - "good" which includes warning rows but no result columns, and "data" having +error and warning rows and corresponding result columns + +#### apply\_checks\_by\_metadata\_and\_split + +```python +def apply_checks_by_metadata_and_split( + df: DataFrame, + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None, + ref_dfs: dict[str, DataFrame] | None = None +) -> tuple[DataFrame, DataFrame] +``` + +Wrapper around `apply_checks_and_split` for use in the metadata-driven pipelines. The main difference + +is how the checks are specified - instead of using functions directly, they are described as function name plus +arguments. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - +it will be used as an error/warning message, or `null` if it's evaluated to `false` +* `name` - name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), +and `warn` (data is going into both dataframes) +* `df`0 (optional) - Expression for filtering data quality checks +* `df`1 (optional) - User-defined key-value pairs added to metadata generated by the check. +- `df`2: dictionary with custom check functions (eg. `df`3globals()`df`3 of the calling module). +If not specified, then only built-in functions are used for the checks. +- `df`5: reference dataframes to use in the checks, if applicable + +**Returns**: + +two dataframes - "good" which includes warning rows but no result columns, and "bad" having +error and warning rows and corresponding result columns + +#### apply\_checks\_by\_metadata + +```python +def apply_checks_by_metadata( + df: DataFrame, + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None, + ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame +``` + +Wrapper around `apply_checks` for use in the metadata-driven pipelines. The main difference + +is how the checks are specified - instead of using functions directly, they are described as function name plus +arguments. + +**Arguments**: + +- `df`: dataframe to check +- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - +it will be used as an error/warning message, or `null` if it's evaluated to `false` +* `name` - name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), +and `warn` (data is going into both dataframes) +* `df`0 (optional) - Expression for filtering data quality checks +* `df`1 (optional) - User-defined key-value pairs added to metadata generated by the check. +- `df`2: dictionary with custom check functions (eg. `df`3globals()`df`3 of calling module). +- `df`5: reference dataframes to use in the checks, if applicable +If not specified, then only built-in functions are used for the checks. + +**Returns**: + +dataframe with errors and warning result columns + +#### apply\_checks\_and\_save\_in\_table + +```python +def apply_checks_and_save_in_table( + checks: list[DQRule], + input_config: InputConfig, + output_config: OutputConfig, + quarantine_config: OutputConfig | None = None, + ref_dfs: dict[str, DataFrame] | None = None) -> None +``` + +Apply data quality checks to a table or view and write the result to table(s). + +If quarantine_config is provided, the data will be split into good and bad records, +with good records written to the output table and bad records to the quarantine table. +If quarantine_config is not provided, all records (with error/warning columns) +will be written to the output table. + +**Arguments**: + +- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. +- `input_config`: Input data configuration (e.g. table name or file location, read options) +- `output_config`: Output data configuration (e.g. table name, output mode, write options) +- `quarantine_config`: Optional quarantine data configuration (e.g. table name, output mode, write options) +- `ref_dfs`: Reference dataframes to use in the checks, if applicable + +#### apply\_checks\_by\_metadata\_and\_save\_in\_table + +```python +def apply_checks_by_metadata_and_save_in_table( + checks: list[dict], + input_config: InputConfig, + output_config: OutputConfig, + quarantine_config: OutputConfig | None = None, + custom_check_functions: dict[str, Any] | None = None, + ref_dfs: dict[str, DataFrame] | None = None) -> None +``` + +Apply data quality checks to a table or view and write the result to table(s). + +If quarantine_config is provided, the data will be split into good and bad records, +with good records written to the output table and bad records to the quarantine table. +If quarantine_config is not provided, all records (with error/warning columns) +will be written to the output table. + +**Arguments**: + +- `checks`: List of dictionaries describing checks. Each check is a dictionary consisting of following fields: +* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - +it will be used as an error/warning message, or `null` if it's evaluated to `false` +* `name` - Name that will be given to a resulting column. Autogenerated if not provided +* `criticality` (optional) -Possible values are `error` (data going only into "bad" dataframe), +and `warn` (data is going into both dataframes) +- `input_config`: Input data configuration (e.g. table name or file location, read options) +- `output_config`: Output data configuration (e.g. table name, output mode, write options) +- `check`0: Optional quarantine data configuration (e.g. table name, output mode, write options) +- `check`1: Dictionary with custom check functions (eg. `check`2globals()`check`2 of calling module). +- `check`4: Reference dataframes to use in the checks, if applicable + +#### validate\_checks + +```python +@staticmethod +def validate_checks( + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None, + validate_custom_check_functions: bool = True +) -> ChecksValidationStatus +``` + +Validate the input dict to ensure they conform to expected structure and types. + +Each check can be a dictionary. The function validates +the presence of required keys, the existence and callability of functions, and the types +of arguments passed to these functions. + +**Arguments**: + +- `checks`: List of checks to apply to the dataframe. Each check should be a dictionary. +- `custom_check_functions`: Optional dictionary with custom check functions. +- `validate_custom_check_functions`: If True, validate custom check functions. + +**Returns**: + +`ValidationStatus`: The validation status. + +#### get\_invalid + +```python +def get_invalid(df: DataFrame) -> DataFrame +``` + +Get records that violate data quality checks (records with warnings and errors). +@param df: input DataFrame. +@return: dataframe with error and warning rows and corresponding result columns. + +#### get\_valid + +```python +def get_valid(df: DataFrame) -> DataFrame +``` + +Get records that don't violate data quality checks (records with warnings but no errors). +@param df: input DataFrame. +@return: dataframe with warning rows but no result columns. + +#### save\_results\_in\_table + +```python +def save_results_in_table(output_df: DataFrame | None = None, + quarantine_df: DataFrame | None = None, + output_config: OutputConfig | None = None, + quarantine_config: OutputConfig | None = None, + run_config_name: str | None = "default", + product_name: str = "dqx", + assume_user: bool = True) +``` + +Save quarantine and output data to the `quarantine_table` and `output_table`. + +**Arguments**: + +- `quarantine_df`: Optional Dataframe containing the quarantine data +- `output_df`: Optional Dataframe containing the output data. If not provided, use run config +- `output_config`: Optional configuration for saving the output data. If not provided, use run config +- `quarantine_config`: Optional configuration for saving the quarantine data. If not provided, use run config +- `run_config_name`: Optional name of the run (config) to use +- `product_name`: name of the product/installation directory +- `assume_user`: if True, assume user installation + +#### load\_checks + +```python +def load_checks(config: BaseChecksStorageConfig) -> list[dict] +``` + +Load checks (dq rules) from the specified source type (file or table). + +**Arguments**: + +- `config`: storage configuration +Allowed configs are: +- `FileChecksStorageConfig`: for loading checks from a file in the local filesystem +- `WorkspaceFileChecksStorageConfig`: for loading checks from a workspace file +- `TableChecksStorageConfig`: for loading checks from a table +- `InstallationChecksStorageConfig`: for loading checks from the installation directory +- `VolumeFileChecksStorageConfig`: for loading checks from a Unity Catalog volume file +- ... + +**Raises**: + +- `ValueError`: if the source type is unknown + +#### save\_checks + +```python +def save_checks(checks: list[dict], config: BaseChecksStorageConfig) -> None +``` + +Save checks (dq rules) to the specified storage type (file or table). + +**Arguments**: + +- `checks`: list of dq rules to save +- `config`: storage configuration +Allowed configs are: +- `FileChecksStorageConfig`: for saving checks in a file in the local filesystem +- `WorkspaceFileChecksStorageConfig`: for saving checks in a workspace file +- `TableChecksStorageConfig`: for saving checks in a table +- `InstallationChecksStorageConfig`: for saving checks in the installation directory +- `VolumeFileChecksStorageConfig`: for saving checks in a Unity Catalog volume file +- ... + +**Raises**: + +- `ValueError`: if the storage type is unknown + diff --git a/docs/dqx/docs/reference/api/executor.md b/docs/dqx/docs/reference/api/executor.md new file mode 100644 index 000000000..a1b6157f2 --- /dev/null +++ b/docs/dqx/docs/reference/api/executor.md @@ -0,0 +1,159 @@ +--- +sidebar_label: executor +title: databricks.labs.dqx.executor +--- + +## DQCheckResult Objects + +```python +@dataclass(frozen=True) +class DQCheckResult() +``` + +Represents the result of applying a data quality rule to a DataFrame. + +This class holds: +- The Spark Column containing the result of the data quality check + (typically a struct with metadata and the outcome of the rule). +- The DataFrame that was evaluated or generated by the check. + +## DQRuleExecutor Objects + +```python +class DQRuleExecutor(abc.ABC) +``` + +Abstract base class for executing a data quality rule on a DataFrame. + +The executor is responsible for: +- Applying the rule's logic to the provided DataFrame (and optional reference DataFrames). +- Returning the raw result of the check, without applying filters or attaching metadata. + +Executors are specialized for different types of rules: +- DQRowRuleExecutor: Handles row-level rules (rules that produce a condition per row). +- DQDatasetRuleExecutor: Handles dataset-level rules (rules that may aggregate or join DataFrames). + +Subclasses must implement the `apply` method, which performs the rule check and returns +a DQCheckResult containing: + - The raw condition produced by the rule. + - The DataFrame used for reporting (original or transformed). + +#### apply + +```python +@abc.abstractmethod +def apply(df: DataFrame, + spark: SparkSession, + ref_dfs: dict[str, DataFrame] | None = None) -> DQCheckResult +``` + +Apply a rule and return results + +## DQRowRuleExecutor Objects + +```python +class DQRowRuleExecutor(DQRuleExecutor) +``` + +Executor for row-level data quality rules. + +This executor applies a DQRowRule to the provided DataFrame. Row-level rules generate +a condition (Spark Column) that evaluates to a message string if the rule is violated, +or null otherwise, for each row of the DataFrame. + +Responsibilities: +- Obtain the condition column. +- Return a DQCheckResult containing: + - The condition column. + - The input DataFrame. + +#### apply + +```python +def apply(df: DataFrame, + spark: SparkSession, + ref_dfs: dict[str, DataFrame] | None = None) -> DQCheckResult +``` + +Apply the row-level data quality rule to the provided DataFrame. + +The rule produces a condition (Spark Column) that indicates whether each row satisfies +or violates the check. The condition contains a message when the check fails, +or null when it passes. + +**Arguments**: + +- `df`: The input DataFrame to which the rule is applied. +- `spark`: The SparkSession used for executing the rule (unused for row rules). +- `ref_dfs`: Optional dictionary of reference DataFrames (unused for row rules). + +**Returns**: + +DQCheckResult containing: +- condition: Spark Column representing the check condition. +- check_df: The input (main) DataFrame (used for downstream processing). + +## DQDatasetRuleExecutor Objects + +```python +class DQDatasetRuleExecutor(DQRuleExecutor) +``` + +Executor for dataset-level data quality rules. + +This executor applies a DQDatasetRule to the provided DataFrame (and optional reference DataFrames). +Dataset-level rules can produce conditions that involve multiple rows, aggregations, +or comparisons across datasets. + +Responsibilities: +- Obtain condition column and check function closure containing computation logic. +- Return a DQCheckResult containing: + - The condition column. + - The resulting DataFrame produced by the rule. + +#### apply + +```python +def apply(df: DataFrame, + spark: SparkSession, + ref_dfs: dict[str, DataFrame] | None = None) -> DQCheckResult +``` + +Apply the dataset-level data quality rule to the provided DataFrame. + +This method executes the rules logic by executing check function closure. +By convention the first argument to the closure is the DataFrame +and the second argument is optional reference DataFrames. + +The rule produces: +- A condition (Spark Column) that represents the overall dataset check status. +- A DataFrame with the results of the dataset-level evaluation. + +**Arguments**: + +- `df`: The input DataFrame to which the rule is applied. +- `spark`: The SparkSession used for executing the rule. +- `ref_dfs`: Optional dictionary of reference DataFrames for dataset-level checks. + +**Returns**: + +DQCheckResult containing: +- condition: Spark Column representing the check condition. +- check_df: DataFrame produced by the check, containing evaluation results. + +## DQRuleExecutorFactory Objects + +```python +class DQRuleExecutorFactory() +``` + +Factory for creating the appropriate DQRuleExecutor instance for a given DQRule. + +This class encapsulates the logic for selecting the correct executor type +(row-level or dataset-level) based on the rule instance provided. + +Responsibilities: +- Determine the type of rule (DQRowRule or DQDatasetRule). +- Return the corresponding executor (DQRowRuleExecutor or DQDatasetRuleExecutor). +- Raise an error if the rule type is unsupported. + diff --git a/docs/dqx/docs/reference/api/index.mdx b/docs/dqx/docs/reference/api/index.mdx new file mode 100644 index 000000000..11319441f --- /dev/null +++ b/docs/dqx/docs/reference/api/index.mdx @@ -0,0 +1,8 @@ +--- +title: API Reference +sidebar_position: 8 +--- + +# API Reference + +This section contains the API reference for the Databricks Labs DQX project. \ No newline at end of file diff --git a/docs/dqx/docs/reference/api/installer/install.md b/docs/dqx/docs/reference/api/installer/install.md new file mode 100644 index 000000000..36d982bd4 --- /dev/null +++ b/docs/dqx/docs/reference/api/installer/install.md @@ -0,0 +1,188 @@ +--- +sidebar_label: install +title: databricks.labs.dqx.installer.install +--- + +## WorkspaceInstaller Objects + +```python +class WorkspaceInstaller(WorkspaceContext) +``` + +Installer for DQX workspace. + +**Arguments**: + +- `ws`: The WorkspaceClient instance. +- `environ`: Optional dictionary of environment variables. + +#### upgrades + +```python +@cached_property +def upgrades() +``` + +Returns the Upgrades instance for the product. + +**Returns**: + +An Upgrades instance. + +#### installation + +```python +@cached_property +def installation() +``` + +Returns the current installation for the product. + +**Raises**: + +- `NotFound`: If the installation is not found. + +**Returns**: + +An Installation instance. + +#### run + +```python +def run(default_config: WorkspaceConfig | None = None) -> WorkspaceConfig +``` + +Runs the installation process. + +**Arguments**: + +- `default_config`: Optional default configuration. + +**Raises**: + +- `ManyError`: If multiple errors occur during installation. +- `TimeoutError`: If a timeout occurs during installation. + +**Returns**: + +The final WorkspaceConfig used for the installation. + +#### extract\_major\_minor + +```python +@staticmethod +def extract_major_minor(version_string: str) +``` + +Extracts the major and minor version from a version string. + +**Arguments**: + +- `version_string`: The version string to extract from. + +**Returns**: + +The major.minor version as a string, or None if not found. + +#### configure + +```python +def configure( + default_config: WorkspaceConfig | None = None) -> WorkspaceConfig +``` + +Configures the workspace. + +Notes: +1. Connection errors are not handled within this configure method. + +**Arguments**: + +- `default_config`: Optional default configuration. + +**Raises**: + +- `NotFound`: If the previous installation is not found. +- `RuntimeWarning`: If the existing installation is corrupted. + +**Returns**: + +The final WorkspaceConfig used for the installation. + +#### replace\_config + +```python +def replace_config(**changes: Any) -> WorkspaceConfig | None +``` + +Persist the list of workspaces where UCX is successfully installed in the config + +## WorkspaceInstallation Objects + +```python +class WorkspaceInstallation() +``` + +#### current + +```python +@classmethod +def current(cls, ws: WorkspaceClient) +``` + +Creates a current WorkspaceInstallation instance based on the current workspace client. + +**Arguments**: + +- `ws`: The WorkspaceClient instance. + +**Returns**: + +A WorkspaceInstallation instance. + +#### config + +```python +@property +def config() +``` + +Returns the configuration of the workspace installation. + +**Returns**: + +The WorkspaceConfig instance. + +#### folder + +```python +@property +def folder() +``` + +Returns the installation folder path. + +**Returns**: + +The installation folder path as a string. + +#### run + +```python +def run() -> bool +``` + +Runs the workflow installation. + +**Returns**: + +True if the installation finished successfully, False otherwise. + +#### uninstall + +```python +def uninstall() +``` + +Uninstalls DQX from the workspace, including project folder, dashboards, and jobs. + diff --git a/docs/dqx/docs/reference/api/installer/logs.md b/docs/dqx/docs/reference/api/installer/logs.md new file mode 100644 index 000000000..85464d538 --- /dev/null +++ b/docs/dqx/docs/reference/api/installer/logs.md @@ -0,0 +1,41 @@ +--- +sidebar_label: logs +title: databricks.labs.dqx.installer.logs +--- + +## PartialLogRecord Objects + +```python +@dataclass +class PartialLogRecord() +``` + +The information found within a log file record. + +#### peak\_multi\_line\_message + +```python +def peak_multi_line_message( + log: TextIO, pattern: re.Pattern) -> tuple[str, re.Match | None, str] +``` + +A single log record message may span multiple log lines. In this case, the regex on +subsequent lines do not match. + +**Arguments**: + +- `log` _TextIO_ - The log file IO. +- `pattern` _re.Pattern_ - The regex pattern for a log line. + +#### parse\_logs + +```python +def parse_logs(log: TextIO) -> Iterator[PartialLogRecord] +``` + +Parse the logs to retrieve values for PartialLogRecord fields. + +**Arguments**: + +- `log` _TextIO_ - The log file IO. + diff --git a/docs/dqx/docs/reference/api/installer/workflow_task.md b/docs/dqx/docs/reference/api/installer/workflow_task.md new file mode 100644 index 000000000..a7b835d97 --- /dev/null +++ b/docs/dqx/docs/reference/api/installer/workflow_task.md @@ -0,0 +1,85 @@ +--- +sidebar_label: workflow_task +title: databricks.labs.dqx.installer.workflow_task +--- + +## Task Objects + +```python +@dataclass +class Task() +``` + +#### dependencies + +```python +def dependencies() +``` + +List of dependencies + +## Workflow Objects + +```python +class Workflow() +``` + +#### name + +```python +@property +def name() +``` + +Name of the workflow + +#### spark\_conf + +```python +@property +def spark_conf() -> dict[str, str] | None +``` + +Spark configuration for the workflow + +#### override\_clusters + +```python +@property +def override_clusters() -> dict[str, str] | None +``` + +Override clusters for the workflow + +#### tasks + +```python +def tasks() -> Iterable[Task] +``` + +List of tasks + +#### workflow\_task + +```python +def workflow_task( + fn=None, + *, + depends_on=None, + job_cluster=Task.job_cluster) -> Callable[[Callable], Callable] +``` + +Decorator to register a task in a workflow. + +#### remove\_extra\_indentation + +```python +def remove_extra_indentation(doc: str) -> str +``` + +Remove extra indentation from docstring. + +**Arguments**: + +- `doc`: Docstring + diff --git a/docs/dqx/docs/reference/api/installer/workflows_installer.md b/docs/dqx/docs/reference/api/installer/workflows_installer.md new file mode 100644 index 000000000..68891f960 --- /dev/null +++ b/docs/dqx/docs/reference/api/installer/workflows_installer.md @@ -0,0 +1,19 @@ +--- +sidebar_label: workflows_installer +title: databricks.labs.dqx.installer.workflows_installer +--- + +#### TEST\_NIGHTLY\_CI\_RESOURCES\_PURGE\_TIMEOUT + +Buffer for debugging nightly integration test runs + +## MaxedStreamHandler Objects + +```python +class MaxedStreamHandler(logging.StreamHandler) +``` + +#### MAX\_STREAM\_SIZE + +1 Mb minus some buffer + diff --git a/docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md b/docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md new file mode 100644 index 000000000..c7ce4ac8d --- /dev/null +++ b/docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md @@ -0,0 +1,59 @@ +--- +sidebar_label: extract_yaml_checks_examples +title: databricks.labs.dqx.llm.extract_yaml_checks_examples +--- + +#### extract\_yaml\_checks\_from\_content + +```python +def extract_yaml_checks_from_content(content: str, + source_name: str = "content" + ) -> list[dict[str, Any]] +``` + +Extract all YAML examples from MDX content string. + +**Arguments**: + +- `content`: The MDX content string to extract YAML from +- `source_name`: Name of the source for logging purposes (default: "content") + +**Returns**: + +List of parsed YAML objects from all valid blocks + +#### extract\_yaml\_checks\_from\_mdx + +```python +def extract_yaml_checks_from_mdx(mdx_file_path: str) -> list[dict[str, Any]] +``` + +Extract all YAML examples from a given MDX file. + +**Arguments**: + +- `mdx_file_path`: Path to the MDX file to extract YAML from + +**Raises**: + +- `FileNotFoundError`: If the MDX file does not exist + +**Returns**: + +List of parsed YAML objects from all valid blocks + +#### extract\_yaml\_checks\_examples + +```python +def extract_yaml_checks_examples(output_file_path: Path | None = None) -> bool +``` + +Extract all YAML examples from both quality_rules.mdx and quality_checks.mdx. + +Creates a combined YAML file with all examples from the documentation files +in the LLM resources directory for use in language model processing. + +**Returns**: + +True if extraction was successful, False otherwise + diff --git a/docs/dqx/docs/reference/api/llm/utils.md b/docs/dqx/docs/reference/api/llm/utils.md new file mode 100644 index 000000000..edcf4177a --- /dev/null +++ b/docs/dqx/docs/reference/api/llm/utils.md @@ -0,0 +1,38 @@ +--- +sidebar_label: utils +title: databricks.labs.dqx.llm.utils +--- + +#### get\_check\_function\_definition + +```python +def get_check_function_definition( + custom_check_functions: dict[str, Any] | None = None +) -> list[dict[str, str]] +``` + +A utility function to get the definition of all check functions. + +This function is primarily used to generate a prompt for the LLM to generate check functions. + +**Arguments**: + +- `custom_check_functions`: A dictionary of custom check functions. +If provided, the function will use the custom check functions to resolve the check function. + If not provided, the function will use only the built-in check functions. + +Returns: + list[dict]: A list of dictionaries, each containing the definition of a check function. + +#### load\_yaml\_checks\_examples + +```python +def load_yaml_checks_examples() -> str +``` + +Load yaml_checks_examples.yml file from the llm/resources folder. + +**Returns**: + +checks examples as yaml string. + diff --git a/docs/dqx/docs/reference/api/manager.md b/docs/dqx/docs/reference/api/manager.md new file mode 100644 index 000000000..c2191200f --- /dev/null +++ b/docs/dqx/docs/reference/api/manager.md @@ -0,0 +1,60 @@ +--- +sidebar_label: manager +title: databricks.labs.dqx.manager +--- + +## DQRuleManager Objects + +```python +@dataclass(frozen=True) +class DQRuleManager() +``` + +Orchestrates the application of a data quality rule to a DataFrame and builds the final check result. + +The manager is responsible for: +- Executing the rule using the appropriate row or dataset executor. +- Applying any filter condition specified in the rule to the check result. +- Combining user-defined and engine-provided metadata into the result. +- Constructing the final structured output (including check name, function, columns, metadata, etc.) +as a DQCheckResult. + +The manager does not implement the logic of individual checks. Instead, it delegates +rule application to the appropriate DQRuleExecutor based on the rule type (row-level or dataset-level). + +**Attributes**: + +- `check` - The DQRule instance that defines the check to apply. +- `df` - The DataFrame on which to apply the check. +- `engine_user_metadata` - Metadata provided by the engine (overridden by check.user_metadata if present). +- `run_time` - The timestamp when the check is executed. +- `ref_dfs` - Optional reference DataFrames for dataset-level checks. + +#### user\_metadata + +```python +@cached_property +def user_metadata() -> dict[str, str] +``` + +Returns user metadata as a dictionary. + +#### filter\_condition + +```python +@cached_property +def filter_condition() +``` + +Returns the filter condition for the check. + +#### process + +```python +def process() -> DQCheckResult +``` + +Process the data quality rule and return results as DQCheckResult containing: +- Column with the check result +- optional DataFrame with the results of the check + diff --git a/docs/dqx/docs/reference/api/profiler/common.md b/docs/dqx/docs/reference/api/profiler/common.md new file mode 100644 index 000000000..8e4d74159 --- /dev/null +++ b/docs/dqx/docs/reference/api/profiler/common.md @@ -0,0 +1,39 @@ +--- +sidebar_label: common +title: databricks.labs.dqx.profiler.common +--- + +#### val\_to\_str + +```python +def val_to_str(value: Any, include_sql_quotes: bool = True) +``` + +Converts a value to a string. + +**Arguments**: + +- `value`: The value to convert. Can be a datetime, date, int, float, or other type. +- `include_sql_quotes`: Whether to include quotes around the value. Default is True. + +**Returns**: + +The string representation of the value + +#### val\_maybe\_to\_str + +```python +def val_maybe_to_str(value: Any, include_sql_quotes: bool = True) +``` + +Converts a value to a string if it is a datetime or date. + +**Arguments**: + +- `value`: The value to convert. Can be a datetime, date, or other type. +- `include_sql_quotes`: Whether to include quotes around the value. Default is True. + +**Returns**: + +The string representation of the value. + diff --git a/docs/dqx/docs/reference/api/profiler/dlt_generator.md b/docs/dqx/docs/reference/api/profiler/dlt_generator.md new file mode 100644 index 000000000..ed9bd0c30 --- /dev/null +++ b/docs/dqx/docs/reference/api/profiler/dlt_generator.md @@ -0,0 +1,36 @@ +--- +sidebar_label: dlt_generator +title: databricks.labs.dqx.profiler.dlt_generator +--- + +## DQDltGenerator Objects + +```python +class DQDltGenerator(DQEngineBase) +``` + +#### generate\_dlt\_rules + +```python +def generate_dlt_rules(rules: list[DQProfile], + action: str | None = None, + language: str = "SQL") -> list[str] | str | dict +``` + +Generates Lakeflow Pipelines (formerly Delta Live Table (DLT)) rules in the specified language. + +**Arguments**: + +- `rules`: A list of data quality profiles to generate rules for. +- `action`: The action to take on rule violation (e.g., "drop", "fail"). +- `language`: The language to generate the rules in ("SQL", "Python" or "Python_Dict"). + +**Raises**: + +- `ValueError`: If the specified language is not supported. + +**Returns**: + +A list of strings representing the Lakeflow Pipelines rules in SQL, a string representing +the Lakeflow Pipelines rules in Python, or dictionary with expressions. + diff --git a/docs/dqx/docs/reference/api/profiler/generator.md b/docs/dqx/docs/reference/api/profiler/generator.md new file mode 100644 index 000000000..40d06dafa --- /dev/null +++ b/docs/dqx/docs/reference/api/profiler/generator.md @@ -0,0 +1,107 @@ +--- +sidebar_label: generator +title: databricks.labs.dqx.profiler.generator +--- + +## DQGenerator Objects + +```python +class DQGenerator(DQEngineBase) +``` + +#### generate\_dq\_rules + +```python +def generate_dq_rules(profiles: list[DQProfile] | None = None, + level: str = "error") -> list[dict] +``` + +Generates a list of data quality rules based on the provided dq profiles. + +**Arguments**: + +- `profiles`: A list of data quality profiles to generate rules for. +- `level`: The criticality level of the rules (default is "error"). + +**Returns**: + +A list of dictionaries representing the data quality rules. + +#### dq\_generate\_is\_in + +```python +@staticmethod +def dq_generate_is_in(column: str, level: str = "error", **params: dict) +``` + +Generates a data quality rule to check if a column's value is in a specified list. + +**Arguments**: + +- `column`: The name of the column to check. +- `level`: The criticality level of the rule (default is "error"). +- `params`: Additional parameters, including the list of values to check against. + +**Returns**: + +A dictionary representing the data quality rule. + +#### dq\_generate\_min\_max + +```python +@staticmethod +def dq_generate_min_max(column: str, level: str = "error", **params: dict) +``` + +Generates a data quality rule to check if a column's value is within a specified range. + +**Arguments**: + +- `column`: The name of the column to check. +- `level`: The criticality level of the rule (default is "error"). +- `params`: Additional parameters, including the minimum and maximum values. + +**Returns**: + +A dictionary representing the data quality rule, or None if no limits are provided. + +#### dq\_generate\_is\_not\_null + +```python +@staticmethod +def dq_generate_is_not_null(column: str, level: str = "error", **params: dict) +``` + +Generates a data quality rule to check if a column's value is not null. + +**Arguments**: + +- `column`: The name of the column to check. +- `level`: The criticality level of the rule (default is "error"). +- `params`: Additional parameters. + +**Returns**: + +A dictionary representing the data quality rule. + +#### dq\_generate\_is\_not\_null\_or\_empty + +```python +@staticmethod +def dq_generate_is_not_null_or_empty(column: str, + level: str = "error", + **params: dict) +``` + +Generates a data quality rule to check if a column's value is not null or empty. + +**Arguments**: + +- `column`: The name of the column to check. +- `level`: The criticality level of the rule (default is "error"). +- `params`: Additional parameters, including whether to trim strings. + +**Returns**: + +A dictionary representing the data quality rule. + diff --git a/docs/dqx/docs/reference/api/profiler/profiler.md b/docs/dqx/docs/reference/api/profiler/profiler.md new file mode 100644 index 000000000..6fa584e5d --- /dev/null +++ b/docs/dqx/docs/reference/api/profiler/profiler.md @@ -0,0 +1,104 @@ +--- +sidebar_label: profiler +title: databricks.labs.dqx.profiler.profiler +--- + +## DQProfiler Objects + +```python +class DQProfiler(DQEngineBase) +``` + +Data Quality Profiler class to profile input data. + +#### get\_columns\_or\_fields + +```python +@staticmethod +def get_columns_or_fields(columns: list[T.StructField]) -> list[T.StructField] +``` + +Extracts all fields from a list of StructField objects, including nested fields from StructType columns. + +**Arguments**: + +- `columns`: A list of StructField objects to process. + +**Returns**: + +A list of StructField objects, including nested fields with prefixed names. + +#### profile + +```python +def profile( + df: DataFrame, + columns: list[str] | None = None, + options: dict[str, Any] | None = None +) -> tuple[dict[str, Any], list[DQProfile]] +``` + +Profiles a DataFrame to generate summary statistics and data quality rules. + +**Arguments**: + +- `df`: The DataFrame to profile. +- `columns`: An optional list of column names to include in the profile. If None, all columns are included. +- `options`: An optional dictionary of options for profiling. + +**Returns**: + +A tuple containing a dictionary of summary statistics and a list of data quality profiles. + +#### profile\_table + +```python +def profile_table( + table: str, + columns: list[str] | None = None, + options: dict[str, Any] | None = None +) -> tuple[dict[str, Any], list[DQProfile]] +``` + +Profiles a table to generate summary statistics and data quality rules. + +**Arguments**: + +- `table`: The fully-qualified table name (`catalog.schema.table`) to be profiled +- `columns`: An optional list of column names to include in the profile. If None, all columns are included. +- `options`: An optional dictionary of options for profiling. + +**Returns**: + +A tuple containing a dictionary of summary statistics and a list of data quality profiles. + +#### profile\_tables + +```python +def profile_tables( + tables: list[str] | None = None, + patterns: list[str] | None = None, + exclude_matched: bool = False, + columns: dict[str, list[str]] | None = None, + options: list[dict[str, Any]] | None = None +) -> dict[str, tuple[dict[str, Any], list[DQProfile]]] +``` + +Profiles Delta tables in Unity Catalog to generate summary statistics and data quality rules. + +**Arguments**: + +- `tables`: An optional list of table names to include. +- `patterns`: An optional list of table names or filesystem-style wildcards (e.g. 'schema.*') to include. +If None, all tables are included. By default, tables matching the pattern are included. +- `exclude_matched`: Specifies whether to include tables matched by the pattern. If True, matched tables +are excluded. If False, matched tables are included. +- `columns`: A dictionary with column names to include in the profile. Keys should be fully-qualified table +names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. +- `options`: A dictionary with options for profiling each table. Keys should be fully-qualified table names +(e.g. `catalog.schema.table`) and values should be options for profiling. + +**Returns**: + +A dictionary mapping table names to tuples containing summary statistics and data quality profiles. + diff --git a/docs/dqx/docs/reference/api/profiler/runner.md b/docs/dqx/docs/reference/api/profiler/runner.md new file mode 100644 index 000000000..caab474b8 --- /dev/null +++ b/docs/dqx/docs/reference/api/profiler/runner.md @@ -0,0 +1,48 @@ +--- +sidebar_label: runner +title: databricks.labs.dqx.profiler.runner +--- + +## ProfilerRunner Objects + +```python +class ProfilerRunner() +``` + +Runs the DQX profiler on the input data and saves the generated checks and profile summary stats. + +#### run + +```python +def run(input_config: InputConfig, + profiler_config: ProfilerConfig) -> tuple[list[dict], dict[str, Any]] +``` + +Run the DQX profiler on the input data and return the generated checks and profile summary stats. + +**Arguments**: + +- `input_config`: Input data configuration (e.g. table name or file location, read options). +- `profiler_config`: Profiler configuration. + +**Returns**: + +A tuple containing the generated checks and profile summary statistics. + +#### save + +```python +def save(checks: list[dict], summary_stats: dict[str, Any], + checks_location: str | None, + profile_summary_stats_file: str | None) -> None +``` + +Save the generated checks and profile summary statistics to the specified files. + +**Arguments**: + +- `checks`: The generated checks. +- `summary_stats`: The profile summary statistics. +- `checks_location`: The file to save the checks to. +- `profile_summary_stats_file`: The file to save the profile summary statistics to. + diff --git a/docs/dqx/docs/reference/api/profiler/workflow.md b/docs/dqx/docs/reference/api/profiler/workflow.md new file mode 100644 index 000000000..ea2f2a558 --- /dev/null +++ b/docs/dqx/docs/reference/api/profiler/workflow.md @@ -0,0 +1,24 @@ +--- +sidebar_label: workflow +title: databricks.labs.dqx.profiler.workflow +--- + +## ProfilerWorkflow Objects + +```python +class ProfilerWorkflow(Workflow) +``` + +#### profile + +```python +@workflow_task +def profile(ctx: RuntimeContext) +``` + +Profile the input data and save the generated checks and profile summary stats. + +**Arguments**: + +- `ctx`: Runtime context. + diff --git a/docs/dqx/docs/reference/api/rule.md b/docs/dqx/docs/reference/api/rule.md new file mode 100644 index 000000000..9aab514dc --- /dev/null +++ b/docs/dqx/docs/reference/api/rule.md @@ -0,0 +1,202 @@ +--- +sidebar_label: rule +title: databricks.labs.dqx.rule +--- + +## Criticality Objects + +```python +class Criticality(Enum) +``` + +Enum class to represent criticality of the check. + +## DefaultColumnNames Objects + +```python +class DefaultColumnNames(Enum) +``` + +Enum class to represent columns in the dataframe that will be used for error and warning reporting. + +## ColumnArguments Objects + +```python +class ColumnArguments(Enum) +``` + +Enum class that is used as input parsing for custom column naming. + +## ExtraParams Objects + +```python +@dataclass(frozen=True) +class ExtraParams() +``` + +Class to represent extra parameters for DQEngine. + +## SingleColumnMixin Objects + +```python +class SingleColumnMixin() +``` + +Mixin to handle column-related functionalities. + +## MultipleColumnsMixin Objects + +```python +class MultipleColumnsMixin() +``` + +Mixin to handle columns-related functionalities. + +## DQRule Objects + +```python +@dataclass(frozen=True) +class DQRule(abc.ABC, DQRuleTypeMixin, SingleColumnMixin, + MultipleColumnsMixin) +``` + +Represents a data quality rule that applies a quality check function to column(s) or +column expression(s). This class includes the following attributes: +* `check_func` - The function used to perform the quality check. +* `name` (optional) - A custom name for the check; autogenerated if not provided. +* `criticality` (optional) - Defines the severity level of the check: + - `error`: Critical issues. + - `warn`: Potential issues. +* `column` (optional) - A single column to which the check function is applied. +* `columns` (optional) - A list of columns to which the check function is applied. +* `filter` (optional) - A filter expression to apply the check only to rows meeting specific conditions. +* `check_func_args` (optional) - Positional arguments for the check function (excluding `column`). +* `name`0 (optional) - Keyword arguments for the check function (excluding `column`). +* `name`2 (optional) - User-defined key-value pairs added to metadata generated by the check. + +#### columns + +some checks require list of columns instead of column + +#### get\_check\_condition + +```python +@abc.abstractmethod +def get_check_condition() -> Column +``` + +Compute the check condition for the rule. + +**Returns**: + +The Spark Column representing the check condition. + +#### columns\_as\_string\_expr + +```python +@ft.cached_property +def columns_as_string_expr() -> Column +``` + +Spark Column expression representing the column(s) as a string (not normalized). + +**Returns**: + +A Spark Column object representing the column(s) as a string (not normalized). + +#### prepare\_check\_func\_args\_and\_kwargs + +```python +def prepare_check_func_args_and_kwargs() -> tuple[list, dict] +``` + +Prepares positional arguments and keyword arguments for the check function. +Includes only arguments supported by the check function and skips empty values. + +#### to\_dict + +```python +def to_dict() -> dict +``` + +Converts a DQRule instance into a structured dictionary. + +## DQRowRule Objects + +```python +@dataclass(frozen=True) +class DQRowRule(DQRule) +``` + +Represents a row-level data quality rule that applies a quality check function to a column or column expression. +Works with check functions that take a single column or no column as input. + +#### get\_check\_condition + +```python +def get_check_condition() -> Column +``` + +Compute the check condition for this rule. + +**Returns**: + +The Spark Column representing the check condition. + +## DQDatasetRule Objects + +```python +@dataclass(frozen=True) +class DQDatasetRule(DQRule) +``` + +Represents a dataset-level data quality rule that applies a quality check function to a column or +column expression or list of columns depending on the check function. +Either column or columns can be provided but not both. The rules are applied to the entire dataset or group of rows +rather than individual rows. Failed checks are appended to the result columns in the same way as row-level rules. + +#### get\_check\_condition + +```python +def get_check_condition() -> Column +``` + +Compute the check condition for this rule. + +**Returns**: + +The Spark Column representing the check condition. + +## DQForEachColRule Objects + +```python +@dataclass(frozen=True) +class DQForEachColRule(DQRuleTypeMixin) +``` + +Represents a data quality rule that applies to a quality check function +repeatedly on each specified column of the provided list of columns. +This class includes the following attributes: +* `columns` - A list of column names or expressions to which the check function should be applied. +* `check_func` - The function used to perform the quality check. +* `name` (optional) - A custom name for the check; autogenerated if not provided. +* `criticality` - The severity level of the check: + - `'warn'` for potential issues. + - `'error'` for critical issues. +* `filter` (optional) - A filter expression to apply the check only to rows meeting specific conditions. +* `check_func_args` (optional) - Positional arguments for the check function (excluding column names). +* `check_func_kwargs` (optional) - Keyword arguments for the check function (excluding column names). +* `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + +#### get\_rules + +```python +def get_rules() -> list[DQRule] +``` + +Build a list of rules for a set of columns. + +**Returns**: + +list of dq rules + diff --git a/docs/dqx/docs/reference/api/runtime.md b/docs/dqx/docs/reference/api/runtime.md new file mode 100644 index 000000000..f4bd44d20 --- /dev/null +++ b/docs/dqx/docs/reference/api/runtime.md @@ -0,0 +1,44 @@ +--- +sidebar_label: runtime +title: databricks.labs.dqx.runtime +--- + +## Workflows Objects + +```python +class Workflows() +``` + +#### all + +```python +@classmethod +def all(cls, config: WorkspaceConfig) +``` + +Return all workflows. + +#### tasks + +```python +def tasks() -> list[Task] +``` + +Return all tasks. + +#### trigger + +```python +def trigger(*argv) +``` + +Trigger a workflow. + +#### main + +```python +def main(*argv) +``` + +Main entry point. + diff --git a/docs/dqx/docs/reference/api/sidebar.json b/docs/dqx/docs/reference/api/sidebar.json new file mode 100644 index 000000000..bf6cc09a0 --- /dev/null +++ b/docs/dqx/docs/reference/api/sidebar.json @@ -0,0 +1,79 @@ +{ + "items": [ + { + "items": [ + { + "items": [ + { + "items": [ + { + "items": [ + "reference/api/databricks/labs/dqx/contexts/application", + "reference/api/databricks/labs/dqx/contexts/workflows", + "reference/api/databricks/labs/dqx/contexts/workspace" + ], + "label": "databricks.labs.dqx.contexts", + "type": "category" + }, + { + "items": [ + "reference/api/databricks/labs/dqx/installer/install", + "reference/api/databricks/labs/dqx/installer/logs", + "reference/api/databricks/labs/dqx/installer/workflow_task", + "reference/api/databricks/labs/dqx/installer/workflows_installer" + ], + "label": "databricks.labs.dqx.installer", + "type": "category" + }, + { + "items": [ + "reference/api/databricks/labs/dqx/llm/extract_yaml_checks_examples", + "reference/api/databricks/labs/dqx/llm/utils" + ], + "label": "databricks.labs.dqx.llm", + "type": "category" + }, + { + "items": [ + "reference/api/databricks/labs/dqx/profiler/common", + "reference/api/databricks/labs/dqx/profiler/dlt_generator", + "reference/api/databricks/labs/dqx/profiler/generator", + "reference/api/databricks/labs/dqx/profiler/profiler", + "reference/api/databricks/labs/dqx/profiler/runner", + "reference/api/databricks/labs/dqx/profiler/workflow" + ], + "label": "databricks.labs.dqx.profiler", + "type": "category" + }, + "reference/api/databricks/labs/dqx/__main__", + "reference/api/databricks/labs/dqx/base", + "reference/api/databricks/labs/dqx/check_funcs", + "reference/api/databricks/labs/dqx/checks_resolver", + "reference/api/databricks/labs/dqx/checks_serializer", + "reference/api/databricks/labs/dqx/checks_storage", + "reference/api/databricks/labs/dqx/checks_validator", + "reference/api/databricks/labs/dqx/cli", + "reference/api/databricks/labs/dqx/config", + "reference/api/databricks/labs/dqx/config_loader", + "reference/api/databricks/labs/dqx/engine", + "reference/api/databricks/labs/dqx/executor", + "reference/api/databricks/labs/dqx/manager", + "reference/api/databricks/labs/dqx/rule", + "reference/api/databricks/labs/dqx/runtime", + "reference/api/databricks/labs/dqx/utils" + ], + "label": "databricks.labs.dqx", + "type": "category" + } + ], + "label": "databricks.labs", + "type": "category" + } + ], + "label": "databricks", + "type": "category" + } + ], + "label": "Reference", + "type": "category" +} \ No newline at end of file diff --git a/docs/dqx/docs/reference/api/utils.md b/docs/dqx/docs/reference/api/utils.md new file mode 100644 index 000000000..89f6846b0 --- /dev/null +++ b/docs/dqx/docs/reference/api/utils.md @@ -0,0 +1,169 @@ +--- +sidebar_label: utils +title: databricks.labs.dqx.utils +--- + +#### get\_column\_name\_or\_alias + +```python +def get_column_name_or_alias( + column: str | Column | ConnectColumn, + normalize: bool = False, + allow_simple_expressions_only: bool = False) -> str +``` + +Extracts the column alias or name from a PySpark Column or ConnectColumn expression. + +PySpark does not provide direct access to the alias of an unbound column, so this function +parses the alias from the column's string representation. + +- Supports columns with one or multiple aliases. +- Ensures the extracted expression is truncated to 255 characters. +- Provides an optional normalization step for consistent naming. + +**Arguments**: + +- `column`: Column, ConnectColumn or string representing a column. +- `normalize`: If True, normalizes the column name (removes special characters, converts to lowercase). +- `allow_simple_expressions_only`: If True, raises an error if the column expression is not a simple expression. +Complex PySpark expressions (e.g., conditionals, arithmetic, or nested transformations), cannot be fully +reconstructed correctly when converting to string (e.g. F.col("a") + F.lit(1)). +However, in certain situations this is acceptable, e.g. when using the output for reporting purposes. + +**Raises**: + +- `ValueError`: If the column expression is invalid. +- `TypeError`: If the column type is unsupported. + +**Returns**: + +The extracted column alias or name. + +#### get\_columns\_as\_strings + +```python +def get_columns_as_strings( + columns: list[str | Column], + allow_simple_expressions_only: bool = True) -> list[str] +``` + +Extracts column names from a list of PySpark Column or ConnectColumn expressions. + +This function processes each column, ensuring that only valid column names are returned. + +**Arguments**: + +- `columns`: List of columns, ConnectColumns or strings representing columns. +- `allow_simple_expressions_only`: If True, raises an error if the column expression is not a simple expression. + +**Returns**: + +List of column names as strings. + +#### is\_simple\_column\_expression + +```python +def is_simple_column_expression(col_name: str) -> bool +``` + +Returns True if the column name does not contain any disallowed characters: + +space, comma, semicolon, curly braces, parentheses, newline, tab, or equals sign. + +**Arguments**: + +- `col_name`: Column name to validate. + +**Returns**: + +True if the column name is valid, False otherwise. + +#### normalize\_bound\_args + +```python +def normalize_bound_args(val: Any) -> Any +``` + +Normalize a value or collection of values for consistent processing. + +Handles primitives, dates, and column-like objects. Lists, tuples, and sets are +recursively normalized with type preserved. + +**Arguments**: + +- `val`: Value or collection of values to normalize. + +**Raises**: + +- `ValueError`: If a column resolves to an invalid name. +- `TypeError`: If a column type is unsupported. + +**Returns**: + +Normalized value or collection. + +#### normalize\_col\_str + +```python +def normalize_col_str(col_str: str) -> str +``` + +Normalizes string to be compatible with metastore column names by applying the following transformations: + +* remove special characters +* convert to lowercase +* limit the length to 255 characters to be compatible with metastore column names + +**Arguments**: + +- `col_str`: Column or string representing a column. + +**Returns**: + +Normalized column name. + +#### read\_input\_data + +```python +def read_input_data(spark: SparkSession, + input_config: InputConfig) -> DataFrame +``` + +Reads input data from the specified location and format. + +**Arguments**: + +- `spark`: SparkSession +- `input_config`: InputConfig with source location/table name, format, and options + +**Returns**: + +DataFrame with values read from the input data + +#### save\_dataframe\_as\_table + +```python +def save_dataframe_as_table(df: DataFrame, output_config: OutputConfig) +``` + +Helper method to save a DataFrame to a Delta table. + +**Arguments**: + +- `df`: The DataFrame to save +- `output_config`: Output table name, write mode, and options + +#### safe\_json\_load + +```python +def safe_json_load(value: str) +``` + +Safely load a JSON string, returning the original value if it fails to parse. + +This allows to specify string value without a need to escape the quotes. + +**Arguments**: + +- `value`: The value to parse as JSON. + diff --git a/docs/dqx/docs/reference/engine.mdx b/docs/dqx/docs/reference/engine.mdx index 4cc85051b..19376abdb 100644 --- a/docs/dqx/docs/reference/engine.mdx +++ b/docs/dqx/docs/reference/engine.mdx @@ -1,3 +1,7 @@ +--- +sidebar_position: 1 +--- + import Admonition from '@theme/Admonition'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/dqx/docs/reference/profiler.mdx b/docs/dqx/docs/reference/profiler.mdx index 822983f16..6ec62e2ec 100644 --- a/docs/dqx/docs/reference/profiler.mdx +++ b/docs/dqx/docs/reference/profiler.mdx @@ -1,3 +1,7 @@ +--- +sidebar_position: 2 +--- + import Admonition from '@theme/Admonition'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/dqx/docs/reference/quality_rules.mdx b/docs/dqx/docs/reference/quality_rules.mdx index 494a8b424..fca32c19e 100644 --- a/docs/dqx/docs/reference/quality_rules.mdx +++ b/docs/dqx/docs/reference/quality_rules.mdx @@ -1,3 +1,7 @@ +--- +sidebar_position: 3 +--- + import Admonition from '@theme/Admonition'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/dqx/docs/reference/testing.mdx b/docs/dqx/docs/reference/testing.mdx index 7e196f090..f469420ef 100644 --- a/docs/dqx/docs/reference/testing.mdx +++ b/docs/dqx/docs/reference/testing.mdx @@ -1,3 +1,7 @@ +--- +sidebar_position: 4 +--- + import Admonition from '@theme/Admonition'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/docs/dqx/sidebars.ts b/docs/dqx/sidebars.ts index 37eb1ec69..5956b79c4 100644 --- a/docs/dqx/sidebars.ts +++ b/docs/dqx/sidebars.ts @@ -15,7 +15,6 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; const sidebars: SidebarsConfig = { // By default, Docusaurus generates a sidebar from the docs folder structure tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], - }; export default sidebars; diff --git a/pyproject.toml b/pyproject.toml index 114fb2269..8ba23c082 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,10 @@ classifiers = [ "Topic :: Software Development :: Libraries", "Topic :: Utilities", ] -dependencies = ["databricks-labs-blueprint>=0.9.1,<0.10", - "databricks-sdk~=0.57", - "databricks-labs-lsql>=0.5,<0.15", +dependencies = [ + "databricks-labs-blueprint>=0.9.1,<0.10", + "databricks-sdk~=0.57", + "databricks-labs-lsql>=0.5,<0.15", ] [project.optional-dependencies] @@ -75,31 +76,39 @@ dependencies = [ "pyspark~=3.5.0", ] -python="3.10" +python = "3.10" # store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better path = ".venv" [tool.hatch.envs.default.scripts] -test = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20" -coverage = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20" +test = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20" +coverage = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20" integration = "pytest tests/integration/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20" -e2e = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20" -fmt = ["black .", - "ruff check . --fix", - "mypy .", - "pylint --output-format=colorized -j 0 src tests"] -verify = ["black --check .", - "ruff check .", - "mypy .", - "pylint --output-format=colorized -j 0 src tests"] +e2e = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20" +fmt = [ + "black .", + "ruff check . --fix", + "mypy .", + "pylint --output-format=colorized -j 0 src tests", +] +verify = [ + "black --check .", + "ruff check .", + "mypy .", + "pylint --output-format=colorized -j 0 src tests", +] update_github_urls = "python docs/dqx/update_github_urls.py" extract_yaml_checks_examples = "python src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py" [tool.hatch.envs.docs] -dependencies = [ - "python-markdown" -] +dependencies = ["pydoc-markdown>=4.8.2"] + +[tool.pydoc-markdown] +loaders = [{ type = "python", search_path = ["./src"], packages = ["databricks.labs.dqx"] }] +processors = [{ type = "smart" }, { type = "crossref" }, {type = "filter", skip_empty_modules = true}] +renderer = { type = "docusaurus", docs_base_path = "docs/dqx/docs", relative_output_path = "reference/api", relative_sidebar_path = "sidebar.json" } +hooks = {post-render = ["cp -r docs/dqx/docs/reference/api/databricks/labs/dqx/* docs/dqx/docs/reference/api", "rm -rf docs/dqx/docs/reference/api/databricks"]} [tool.isort] profile = "black" @@ -133,11 +142,7 @@ parallel = true [tool.coverage.report] omit = ["*/working-copy/*", 'src/databricks/labs/dqx/__main__.py'] -exclude_lines = [ - "no cov", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", -] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] [tool.pylint.main] # PyLint configuration is adapted from Google Python Style Guide with modifications. @@ -315,15 +320,20 @@ function-rgx = "[A-Za-z_][A-Za-z0-9_]{2,}$" # Good variable names which should always be accepted, separated by a comma. good-names = [ - "f", # use for file handles - "i", "j", "k", # use for loops - "df", # use for pyspark.sql.DataFrame - "ex", "e", # use for exceptions - "fn", "cb", # use for callbacks - "_", # use for ignores - "a", # use for databricks.sdk.AccountClient - "w", "ws", # use for databricks.sdk.WorkspaceClient - "me" # use for current user + "f", # use for file handles + "i", + "j", + "k", # use for loops + "df", # use for pyspark.sql.DataFrame + "ex", + "e", # use for exceptions + "fn", + "cb", # use for callbacks + "_", # use for ignores + "a", # use for databricks.sdk.AccountClient + "w", + "ws", # use for databricks.sdk.WorkspaceClient + "me", # use for current user ] # Good variable names regexes, separated by a comma. If names match any regex, @@ -525,7 +535,13 @@ logging-modules = ["logging"] [tool.pylint."messages control"] # Only show warnings with the listed confidence levels. Leave empty to show all. # Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] +confidence = [ + "HIGH", + "CONTROL_FLOW", + "INFERENCE", + "INFERENCE_FAILURE", + "UNDEFINED", +] # Disable the message, report, category or checker with the given id(s). You can # either give multiple identifiers separated by comma (,) or put this option @@ -557,7 +573,7 @@ disable = [ "unnecessary-default-type-args", "mock-no-usage", "broad-exception-caught", - "rewrite-as-for-loop" + "rewrite-as-for-loop", ] # Enable the message, report, category or checker with the given id(s). You can @@ -569,7 +585,16 @@ enable = ["useless-suppression", "use-symbolic-message-instead"] [tool.pylint.method_args] # List of qualified names (i.e., library.method) which require a timeout # parameter e.g. 'requests.api.get,requests.api.post' -timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"] +timeout-methods = [ + "requests.api.delete", + "requests.api.get", + "requests.api.head", + "requests.api.options", + "requests.api.patch", + "requests.api.post", + "requests.api.put", + "requests.api.request", +] [tool.pylint.miscellaneous] # List of note tags to take in consideration, separated by a comma. @@ -695,12 +720,22 @@ ignore-none = true ignore-on-opaque-inference = true # List of symbolic message names to ignore for Mixin members. -ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] +ignored-checks-for-mixins = [ + "no-member", + "not-async-context-manager", + "not-context-manager", + "attribute-defined-outside-init", +] # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes = ["SQLObject", "optparse.Values", "thread._local", "_thread._local"] +ignored-classes = [ + "SQLObject", + "optparse.Values", + "thread._local", + "_thread._local", +] # Show a hint with possible names when a member name was not found. The aspect of # finding the hint is based on edit distance. @@ -747,4 +782,10 @@ ignored-argument-names = "_.*|^ignored_|^unused_" # List of qualified module names which can have objects that can redefine # builtins. -redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] +redefining-builtins-modules = [ + "six.moves", + "past.builtins", + "future.builtins", + "builtins", + "io", +] diff --git a/scripts/docgen.py b/scripts/docgen.py deleted file mode 100644 index 70eb3a80b..000000000 --- a/scripts/docgen.py +++ /dev/null @@ -1,668 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to generate comprehensive markdown documentation from Python module docstrings. - -This script analyzes Python modules and generates a single markdown file with -structured documentation including function signatures, parameters, and metadata. -""" - -import ast -import importlib -import inspect -import logging -import re -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, List, Optional - - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%H:%M:%S' -) -logger = logging.getLogger(__name__) - - -@dataclass -class DocstringInfo: - """Container for docstring information.""" - - source_file_name: str - source_file_path: str - module_name: str - class_name: Optional[str] - function_name: str - method_type: Optional[str] # 'classmethod', 'staticmethod', 'instancemethod', or None - doc_body: str - parameters: List[Dict[str, Any]] - return_info: Optional[Dict[str, Any]] - docstring: str - line_start: int - line_end: int - - -class MarkdownDocGenerator: - """Generates markdown documentation from collected docstrings.""" - - def __init__(self, project_root: Path): - self.project_root = project_root - self.collected_docstrings: List[DocstringInfo] = [] - self.stats = { - 'modules_processed': 0, - 'classes_processed': 0, - 'functions_processed': 0, - 'docstrings_found': 0, - 'errors': 0 - } - - def _get_relative_path(self, file_path: str) -> str: - """Get relative path from project root.""" - try: - return str(Path(file_path).relative_to(self.project_root)) - except ValueError: - return file_path - - def _get_method_type(self, func: Any, class_obj: Any = None) -> Optional[str]: - """Determine if a method is classmethod, staticmethod, or instancemethod.""" - if class_obj is None: - return None - - # Check if it's a classmethod - if isinstance(inspect.getattr_static(class_obj, func.__name__, None), classmethod): - return "classmethod" - - # Check if it's a staticmethod - if isinstance(inspect.getattr_static(class_obj, func.__name__, None), staticmethod): - return "staticmethod" - - # Check if it's a regular method (has 'self' parameter) - try: - sig = inspect.signature(func) - params = list(sig.parameters.keys()) - if params and params[0] in ('self', 'cls'): - return "instancemethod" - except (ValueError, TypeError): - pass - - return None - - def _extract_parameters(self, func: Any) -> List[Dict[str, Any]]: - """Extract parameter information from a function.""" - try: - sig = inspect.signature(func) - parameters = [] - - for param_name, param in sig.parameters.items(): - param_info = { - "name": param_name, - "annotation": str(param.annotation) if param.annotation != inspect.Parameter.empty else None, - "default": str(param.default) if param.default != inspect.Parameter.empty else None, - "kind": param.kind.name - } - parameters.append(param_info) - - return parameters - except (ValueError, TypeError): - return [] - - def _extract_return_info(self, func: Any) -> Optional[Dict[str, Any]]: - """Extract return type information from a function.""" - try: - sig = inspect.signature(func) - if sig.return_annotation != inspect.Signature.empty: - return { - "annotation": str(sig.return_annotation), - "description": None # Could be parsed from docstring - } - except (ValueError, TypeError): - pass - return None - - def _get_source_lines(self, obj: Any) -> tuple[int, int]: - """Get start and end line numbers for an object.""" - try: - source_lines = inspect.getsourcelines(obj) - line_start = source_lines[1] - line_end = line_start + len(source_lines[0]) - 1 - return line_start, line_end - except (OSError, TypeError): - return 0, 0 - - def _escape_markdown(self, text: str) -> str: - """Escape special markdown characters in text.""" - if not text: - return "" - # Escape MDX/JSX template syntax that could cause issues - text = text.replace('{{', '{{') - text = text.replace('}}', '}}') - # Escape markdown special characters - text = re.sub(r'([*_`\[\]{}()#+\-.!|\\])', r'\\\1', text) - return text - - def _clean_type_annotation(self, annotation: str) -> str: - """Clean up type annotations to avoid MDX parsing issues.""" - if not annotation: - return "" - - # Replace problematic patterns in type annotations - # Convert to str - annotation = re.sub(r"", r'\1', annotation) - - # Convert to module.Class - annotation = re.sub(r"", r'\1', annotation) - - # Handle other <...> patterns that might contain quotes - annotation = re.sub(r"<([^<>]*)'([^<>]*)'([^<>]*)>", r'<\1"\2"\3>', annotation) - - # Escape any remaining single quotes that might cause issues - annotation = annotation.replace("'", "'") - - return annotation - - def _format_signature(self, func_name: str, parameters: List[Dict[str, Any]], - return_info: Optional[Dict[str, Any]], method_type: Optional[str] = None) -> str: - """Format function signature for markdown.""" - params_str = [] - for param in parameters: - param_str = param['name'] - if param['annotation']: - clean_annotation = self._clean_type_annotation(param['annotation']) - param_str += f": {clean_annotation}" - if param['default'] is not None: - param_str += f" = {param['default']}" - params_str.append(param_str) - - signature = f"{func_name}({', '.join(params_str)})" - if return_info and return_info['annotation']: - clean_return_annotation = self._clean_type_annotation(return_info['annotation']) - signature += f" -> {clean_return_annotation}" - - if method_type: - signature = f"@{method_type}\n{signature}" - - return signature - - def _process_function(self, func: Any, module_name: str, class_obj: Any = None, - class_name: Optional[str] = None) -> Optional[DocstringInfo]: - """Process a function and extract its docstring information.""" - try: - self.stats['functions_processed'] += 1 - - # Skip if no docstring - docstring = inspect.getdoc(func) - if not docstring: - return None - - logger.debug(f"πŸ“ Processing function: {func.__name__}") - self.stats['docstrings_found'] += 1 - - # Get source file information - source_file = inspect.getfile(func) - source_file_name = Path(source_file).name - source_file_path = self._get_relative_path(source_file) - - # Get method type - method_type = self._get_method_type(func, class_obj) - - # Extract parameters and return info - parameters = self._extract_parameters(func) - return_info = self._extract_return_info(func) - - # Get source line numbers - line_start, line_end = self._get_source_lines(func) - - # Create docstring info - docstring_info = DocstringInfo( - source_file_name=source_file_name, - source_file_path=source_file_path, - module_name=module_name, - class_name=class_name, - function_name=func.__name__, - method_type=method_type, - doc_body=docstring, - parameters=parameters, - return_info=return_info, - docstring=docstring, - line_start=line_start, - line_end=line_end - ) - - return docstring_info - - except Exception as e: - self.stats['errors'] += 1 - logger.error(f"❌ Error processing function {func.__name__}: {e}") - return None - - def _process_class(self, class_obj: Any, module_name: str) -> List[DocstringInfo]: - """Process a class and extract docstring information for it and its methods.""" - self.stats['classes_processed'] += 1 - logger.info(f"πŸ—οΈ Processing class: {class_obj.__name__}") - - docstrings = [] - - # Process class docstring - class_docstring = inspect.getdoc(class_obj) - if class_docstring: - try: - source_file = inspect.getfile(class_obj) - source_file_name = Path(source_file).name - source_file_path = self._get_relative_path(source_file) - line_start, line_end = self._get_source_lines(class_obj) - - class_docstring_info = DocstringInfo( - source_file_name=source_file_name, - source_file_path=source_file_path, - module_name=module_name, - class_name=class_obj.__name__, - function_name="__class__", - method_type=None, - doc_body=class_docstring, - parameters=[], - return_info=None, - docstring=class_docstring, - line_start=line_start, - line_end=line_end - ) - docstrings.append(class_docstring_info) - except Exception as e: - self.stats['errors'] += 1 - logger.error(f"❌ Error processing class {class_obj.__name__}: {e}") - - # Process methods - methods = [(method_name, method) for method_name, method in inspect.getmembers(class_obj, inspect.isfunction) - if not method_name.startswith('_') or method_name in ['__init__', '__new__']] - - if methods: - logger.debug(f" πŸ”§ Processing {len(methods)} methods in {class_obj.__name__}") - for method_name, method in methods: - method_info = self._process_function(method, module_name, class_obj, class_obj.__name__) - if method_info: - docstrings.append(method_info) - - # Process static methods and class methods - for method_name, method in inspect.getmembers(class_obj): - if isinstance(method, staticmethod): - if hasattr(method, '__func__'): - method_info = self._process_function(method.__func__, module_name, class_obj, class_obj.__name__) - if method_info: - docstrings.append(method_info) - elif isinstance(method, classmethod): - if hasattr(method, '__func__'): - method_info = self._process_function(method.__func__, module_name, class_obj, class_obj.__name__) - if method_info: - docstrings.append(method_info) - elif (callable(method) and not method_name.startswith('_') and - not inspect.isfunction(method) and not inspect.ismethod(method)): - # Handle other callable objects that might have docstrings - try: - method_info = self._process_function(method, module_name, class_obj, class_obj.__name__) - if method_info: - docstrings.append(method_info) - except Exception: - # Skip if we can't process this callable - pass - - return docstrings - - def _process_module(self, module: Any, module_name: str) -> List[DocstringInfo]: - """Process a module and extract all docstring information.""" - self.stats['modules_processed'] += 1 - logger.info(f"πŸ“¦ Processing module: {module_name}") - - docstrings = [] - - # Process module-level functions - module_functions = [func for func_name, func in inspect.getmembers(module, inspect.isfunction) - if func.__module__ == module_name] - - if module_functions: - logger.info(f" πŸ”§ Found {len(module_functions)} functions") - for func in module_functions: - func_info = self._process_function(func, module_name) - if func_info: - docstrings.append(func_info) - - # Process classes - module_classes = [class_obj for class_name, class_obj in inspect.getmembers(module, inspect.isclass) - if class_obj.__module__ == module_name] - - if module_classes: - logger.info(f" πŸ—οΈ Found {len(module_classes)} classes") - for i, class_obj in enumerate(module_classes, 1): - logger.debug(f" πŸ“ Processing class {i}/{len(module_classes)}: {class_obj.__name__}") - class_docstrings = self._process_class(class_obj, module_name) - docstrings.extend(class_docstrings) - - logger.info(f" βœ… Collected {len(docstrings)} docstrings from {module_name}") - return docstrings - - def collect_from_module(self, module_name: str) -> List[DocstringInfo]: - """Collect docstrings from a given module.""" - logger.info(f"πŸš€ Starting docstring collection from module: {module_name}") - - try: - # Import the module - logger.info(f"πŸ“₯ Importing module: {module_name}") - module = importlib.import_module(module_name) - - # Process the module - docstrings = self._process_module(module, module_name) - - # Also process submodules if it's a package - if hasattr(module, '__path__'): - logger.info(f"πŸ“ Discovering submodules in package: {module_name}") - try: - import pkgutil - submodules = list(pkgutil.iter_modules(module.__path__, module_name + ".")) - logger.info(f" πŸ” Found {len(submodules)} submodules") - - for i, (importer, submodule_name, ispkg) in enumerate(submodules, 1): - try: - logger.debug(f" πŸ“ Processing submodule {i}/{len(submodules)}: {submodule_name}") - submodule = importlib.import_module(submodule_name) - sub_docstrings = self._process_module(submodule, submodule_name) - docstrings.extend(sub_docstrings) - except Exception as e: - self.stats['errors'] += 1 - logger.error(f"❌ Error processing submodule {submodule_name}: {e}") - except Exception as e: - self.stats['errors'] += 1 - logger.error(f"❌ Error iterating submodules: {e}") - - logger.info(f"πŸŽ‰ Successfully collected {len(docstrings)} docstrings from {module_name}") - return docstrings - - except ImportError as e: - self.stats['errors'] += 1 - logger.error(f"❌ Error importing module {module_name}: {e}") - return [] - except Exception as e: - self.stats['errors'] += 1 - logger.error(f"❌ Error processing module {module_name}: {e}") - return [] - - def _generate_markdown_content(self, docstrings: List[DocstringInfo], module_name: str) -> str: - """Generate markdown content from collected docstrings.""" - logger.info(f"πŸ“ Generating markdown content for {len(docstrings)} docstrings") - - # Group docstrings by module and class - modules = {} - for ds in docstrings: - if ds.module_name not in modules: - modules[ds.module_name] = {'functions': [], 'classes': {}} - - if ds.class_name and ds.function_name != '__class__': - # This is a method - if ds.class_name not in modules[ds.module_name]['classes']: - modules[ds.module_name]['classes'][ds.class_name] = {'docstring': None, 'methods': []} - modules[ds.module_name]['classes'][ds.class_name]['methods'].append(ds) - elif ds.class_name and ds.function_name == '__class__': - # This is a class docstring - if ds.class_name not in modules[ds.module_name]['classes']: - modules[ds.module_name]['classes'][ds.class_name] = {'docstring': None, 'methods': []} - modules[ds.module_name]['classes'][ds.class_name]['docstring'] = ds - else: - # This is a module-level function - modules[ds.module_name]['functions'].append(ds) - - # Generate markdown - md_content = [] - - # Header - md_content.append(f"# {module_name} API Reference") - md_content.append("") - md_content.append('
') - md_content.append("") - - # Note: Table of contents is handled by Docusaurus automatically - - # Generate content for each module - for mod_name in sorted(modules.keys()): - mod_data = modules[mod_name] - - md_content.append(f"## {mod_name}") - md_content.append("") - md_content.append(f'
') - - # Module-level functions - if mod_data['functions']: - md_content.append("### Functions") - md_content.append("") - md_content.append('
') - - for func in sorted(mod_data['functions'], key=lambda x: x.function_name): - md_content.append(f"#### {func.function_name}") - md_content.append("") - md_content.append(f'
') - - # Source info - md_content.append(f'
') - md_content.append(f"**Source:** `{func.source_file_path}:{func.line_start}-{func.line_end}`") - md_content.append("
") - md_content.append("") - - # Function signature - signature = self._format_signature(func.function_name, func.parameters, func.return_info, func.method_type) - md_content.append('
') - md_content.append("```python") - md_content.append(signature) - md_content.append("```") - md_content.append("
") - md_content.append("") - - # Docstring - md_content.append('
') - md_content.append(self._escape_markdown(func.docstring)) - md_content.append("
") - - # Parameters - if func.parameters: - md_content.append("") - md_content.append('
') - md_content.append("**Parameters:**") - md_content.append("") - for param in func.parameters: - param_line = f"- `{param['name']}`" - if param['annotation']: - clean_annotation = self._clean_type_annotation(param['annotation']) - param_line += f" ({clean_annotation})" - if param['default'] is not None: - param_line += f", default: `{param['default']}`" - md_content.append(param_line) - md_content.append("
") - - # Return type - if func.return_info: - md_content.append("") - md_content.append('
') - clean_return_annotation = self._clean_type_annotation(func.return_info['annotation']) - md_content.append(f"**Returns:** `{clean_return_annotation}`") - md_content.append("
") - - md_content.append("") - md_content.append("
") - md_content.append("") - - md_content.append("
") - md_content.append("") - - # Classes - if mod_data['classes']: - md_content.append("### Classes") - md_content.append("") - md_content.append('
') - - for class_name in sorted(mod_data['classes'].keys()): - class_data = mod_data['classes'][class_name] - - md_content.append(f"#### {class_name}") - md_content.append("") - md_content.append(f'
') - - # Class docstring - if class_data['docstring']: - ds = class_data['docstring'] - md_content.append(f'
') - md_content.append(f"**Source:** `{ds.source_file_path}:{ds.line_start}-{ds.line_end}`") - md_content.append("
") - md_content.append("") - - md_content.append('
') - md_content.append(self._escape_markdown(ds.docstring)) - md_content.append("
") - md_content.append("") - - # Methods - if class_data['methods']: - md_content.append("##### Methods") - md_content.append("") - md_content.append('
') - - for method in sorted(class_data['methods'], key=lambda x: x.function_name): - md_content.append(f"###### {method.function_name}") - md_content.append("") - md_content.append(f'
') - - # Method signature - signature = self._format_signature(method.function_name, method.parameters, method.return_info, method.method_type) - md_content.append('
') - md_content.append("```python") - md_content.append(signature) - md_content.append("```") - md_content.append("
") - md_content.append("") - - # Docstring - md_content.append('
') - md_content.append(self._escape_markdown(method.docstring)) - md_content.append("
") - - # Parameters - if method.parameters: - md_content.append("") - md_content.append('
') - md_content.append("**Parameters:**") - md_content.append("") - for param in method.parameters: - param_line = f"- `{param['name']}`" - if param['annotation']: - clean_annotation = self._clean_type_annotation(param['annotation']) - param_line += f" ({clean_annotation})" - if param['default'] is not None: - param_line += f", default: `{param['default']}`" - md_content.append(param_line) - md_content.append("
") - - # Return type - if method.return_info: - md_content.append("") - md_content.append('
') - clean_return_annotation = self._clean_type_annotation(method.return_info['annotation']) - md_content.append(f"**Returns:** `{clean_return_annotation}`") - md_content.append("
") - - md_content.append("") - md_content.append("
") - md_content.append("") - - md_content.append("
") - - md_content.append("
") - md_content.append("") - - md_content.append("
") - - md_content.append("
") - md_content.append("") - - md_content.append("
") - - return "\n".join(md_content) - - -def generate_markdown_docs(module_name: str, output_file_path: Path) -> None: - """ - Generate markdown documentation from a given module and save to a file. - - Args: - module_name: Name of the module to analyze (e.g., 'databricks.labs.dqx') - output_file_path: Path where to save the generated markdown documentation - """ - start_time = time.time() - - logger.info(f"🌟 Starting DQX Markdown Documentation Generator") - logger.info(f"πŸ“ Target module: {module_name}") - logger.info(f"πŸ“ Output file: {output_file_path}") - - # Get project root (assume script is in scripts/ directory) - project_root = Path(__file__).parent.parent - logger.info(f"🏠 Project root: {project_root}") - - # Create generator - generator = MarkdownDocGenerator(project_root) - - # Collect docstrings - docstrings = generator.collect_from_module(module_name) - - # Generate markdown content - markdown_content = generator._generate_markdown_content(docstrings, module_name) - - # Save to file - logger.info(f"πŸ’Ύ Saving markdown documentation to {output_file_path}") - output_file_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_file_path, 'w', encoding='utf-8') as f: - f.write(markdown_content) - - # Print summary statistics - elapsed_time = time.time() - start_time - logger.info(f"πŸ“Š Generation Summary:") - logger.info(f" πŸ“¦ Modules processed: {generator.stats['modules_processed']}") - logger.info(f" πŸ—οΈ Classes processed: {generator.stats['classes_processed']}") - logger.info(f" πŸ”§ Functions processed: {generator.stats['functions_processed']}") - logger.info(f" πŸ“ Docstrings found: {generator.stats['docstrings_found']}") - logger.info(f" ❌ Errors encountered: {generator.stats['errors']}") - logger.info(f" πŸ’Ύ Output file size: {output_file_path.stat().st_size / 1024:.1f} KB") - logger.info(f" ⏱️ Execution time: {elapsed_time:.2f} seconds") - - logger.info(f"βœ… Successfully completed! Generated markdown documentation at {output_file_path}") - - -def main(): - """Main function for command-line usage.""" - import argparse - - # Print header - print("πŸš€ DQX Markdown Documentation Generator") - print("=" * 50) - - parser = argparse.ArgumentParser( - description="Generate comprehensive markdown documentation from Python modules", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - python scripts/docgen.py databricks.labs.dqx docs/generated/api-reference.md - python scripts/docgen.py databricks.labs.dqx.engine /tmp/engine_docs.md --verbose - """ - ) - - parser.add_argument("module_name", help="Name of the module to analyze (e.g., 'databricks.labs.dqx')") - parser.add_argument("output_file", help="Output file path for generated markdown documentation") - parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging") - - args = parser.parse_args() - - # Set log level based on verbose flag - if args.verbose: - logging.getLogger().setLevel(logging.DEBUG) - logger.info("πŸ” Verbose logging enabled") - - output_path = Path(args.output_file) - generate_markdown_docs(args.module_name, output_path) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index c2109f935..29eb4b747 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -45,7 +45,6 @@ def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient: class DQEngineCoreBase(DQEngineBase): - @abc.abstractmethod def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index b41a9c6ee..a1e46a15c 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -868,7 +868,7 @@ def sql_query( :param msg: Optional custom message or Column expression. :param name: Optional name for the result. :param negate: If True, the condition is negated (i.e., the check fails when the condition is False). - :param input_placeholder: Name to be used in the sql query as {{ input_placeholder }} to refer to the + :param input_placeholder: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the input DataFrame on which the checks are applied. :param row_filter: Optional SQL expression for filtering rows before checking the foreign key. Auto-injected from the check filter. @@ -890,7 +890,7 @@ def sql_query( def _replace_template(sql: str, replacements: dict[str, str]) -> str: """ - Replace {{ template }} placeholders in sql with actual names, allowing for whitespace between braces. + Replace `{{ template }}` placeholders in sql with actual names, allowing for whitespace between braces. """ for key, val in replacements.items(): pattern = r"\{\{\s*" + re.escape(key) + r"\s*\}\}" @@ -1117,7 +1117,7 @@ def compare_datasets( The comparison does not support Map types (any column comparison on map type is skipped automatically). The log containing detailed differences is written to the message field of the check result as JSON string. - Example: {\"row_missing\":false,\"row_extra\":true,\"changed\":{\"val\":{\"df\":\"val1\"}}} + Example: `{"row_missing":false,"row_extra":true,"changed":{"val":{"df":"val1"}}}` :param columns: List of columns to use for row matching with the reference DataFrame (can be a list of string column names or column expressions). @@ -1336,8 +1336,8 @@ def _match_rows( ) -> DataFrame: """ Perform a null-safe join between two DataFrames based on primary key columns. - Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols - Use eq null safe join to ensure that: 1 == 1 matches; NULL <=> NULL matches; 1 <=> NULL does not match + Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols. + Use eq null safe join to ensure that: `1 == 1 matches; NULL <=> NULL matches; 1 <=> NULL` does not match :param df: The input DataFrame to join. :param ref_df: The reference DataFrame to join against. diff --git a/src/databricks/labs/dqx/contexts/workflows.py b/src/databricks/labs/dqx/contexts/workflows.py index 5328fed0a..255846e88 100644 --- a/src/databricks/labs/dqx/contexts/workflows.py +++ b/src/databricks/labs/dqx/contexts/workflows.py @@ -13,7 +13,6 @@ class RuntimeContext(GlobalContext): - @cached_property def _config_path(self) -> Path: config = self.named_parameters.get("config") diff --git a/src/databricks/labs/dqx/installer/workflows_installer.py b/src/databricks/labs/dqx/installer/workflows_installer.py index 160907b3a..de65acaf9 100644 --- a/src/databricks/labs/dqx/installer/workflows_installer.py +++ b/src/databricks/labs/dqx/installer/workflows_installer.py @@ -573,7 +573,6 @@ def _job_clusters(self, job_clusters: set[str], spark_conf: dict[str, str] | Non class MaxedStreamHandler(logging.StreamHandler): - MAX_STREAM_SIZE = 2**20 - 2**6 # 1 Mb minus some buffer _installed_handlers: dict[str, tuple[logging.Logger, MaxedStreamHandler]] = {} _sent_bytes = 0 diff --git a/src/databricks/labs/dqx/profiler/dlt_generator.py b/src/databricks/labs/dqx/profiler/dlt_generator.py index 0b8502e31..85984428c 100644 --- a/src/databricks/labs/dqx/profiler/dlt_generator.py +++ b/src/databricks/labs/dqx/profiler/dlt_generator.py @@ -11,7 +11,6 @@ class DQDltGenerator(DQEngineBase): - def generate_dlt_rules( self, rules: list[DQProfile], action: str | None = None, language: str = "SQL" ) -> list[str] | str | dict: diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index a431e22f0..22b445135 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -9,7 +9,6 @@ class DQGenerator(DQEngineBase): - def generate_dq_rules(self, profiles: list[DQProfile] | None = None, level: str = "error") -> list[dict]: """ Generates a list of data quality rules based on the provided dq profiles. diff --git a/src/databricks/labs/dqx/rule.py b/src/databricks/labs/dqx/rule.py index f02df8feb..07ff57e31 100644 --- a/src/databricks/labs/dqx/rule.py +++ b/src/databricks/labs/dqx/rule.py @@ -107,7 +107,6 @@ def _build_columns_args(self, columns: list[str | Column] | None, valid_params: class DQRuleTypeMixin: - _expected_rule_type: str # to be defined in subclasses _alternative_rules: list[str] # e.g., "DQRowRule" or "DQDatasetRule" diff --git a/tests/e2e/test_run_demos.py b/tests/e2e/test_run_demos.py index e6a86539a..9f41829c0 100644 --- a/tests/e2e/test_run_demos.py +++ b/tests/e2e/test_run_demos.py @@ -195,7 +195,6 @@ def validate_demo_run_status(run: Run, client: WorkspaceClient) -> None: def test_run_dqx_streaming_demo_native(make_notebook, make_schema, make_job, tmp_path): - ws = WorkspaceClient() path = Path(__file__).parent.parent.parent / "demos" / "dqx_streaming_demo_native.py" with open(path, "rb") as f: @@ -226,7 +225,6 @@ def test_run_dqx_streaming_demo_native(make_notebook, make_schema, make_job, tmp def test_run_dqx_streaming_demo_diy(make_notebook, make_job, tmp_path): - ws = WorkspaceClient() path = Path(__file__).parent.parent.parent / "demos" / "dqx_streaming_demo_diy.py" with open(path, "rb") as f: diff --git a/tests/unit/test_build_rules.py b/tests/unit/test_build_rules.py index b830dfd40..40ea187a6 100644 --- a/tests/unit/test_build_rules.py +++ b/tests/unit/test_build_rules.py @@ -927,7 +927,6 @@ def test_validate_column_and_columns_provided_as_args(): def test_register_rule(): - @register_rule("single_column") def mock_check_func(): pass diff --git a/tests/unit/test_logs.py b/tests/unit/test_logs.py index 597615bfc..1f53af8de 100644 --- a/tests/unit/test_logs.py +++ b/tests/unit/test_logs.py @@ -243,7 +243,6 @@ def test_task_logger_exit(task_logger): patch("os.path.exists", return_value=True), patch("os.unlink"), ): - error = Exception("Test error") task_logger.__exit__(None, error, None) diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000..08b20a3c1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,303 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, + { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, + { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, + { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, + { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, + { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, + { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, + { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, + { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, + { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, + { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, + { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, + { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, + { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, + { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, + { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, + { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, + { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, + { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, + { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, + { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, + { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, + { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "databricks-labs-blueprint" +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "databricks-sdk" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/d0/9d818b50dc4fa86a9b5fdc0d74b96eacbd06410d7bd10b9e5f75dc416e35/databricks_labs_blueprint-0.9.3.tar.gz", hash = "sha256:a40628c0d58b6a9c8cf776b3ffa31237a8eec2f4d7a21142464cd2c285a2cd61", size = 76667, upload-time = "2024-11-14T13:33:23.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/f7/4e77bdcd83fb5e53d79526f4532dd05af53e5dcbb2c2854ae536baecf133/databricks_labs_blueprint-0.9.3-py3-none-any.whl", hash = "sha256:0e640953deef5e41bc324d1035ce8c5d549023178ce50708700ab34c438451f3", size = 61549, upload-time = "2024-11-14T13:33:21.818Z" }, +] + +[package.optional-dependencies] +yaml = [ + { name = "pyyaml" }, +] + +[[package]] +name = "databricks-labs-dqx" +source = { editable = "." } +dependencies = [ + { name = "databricks-labs-blueprint" }, + { name = "databricks-labs-lsql" }, + { name = "databricks-sdk" }, +] + +[package.optional-dependencies] +cli = [ + { name = "pyspark" }, +] + +[package.metadata] +requires-dist = [ + { name = "databricks-labs-blueprint", specifier = ">=0.9.1,<0.10" }, + { name = "databricks-labs-lsql", specifier = ">=0.5,<0.15" }, + { name = "databricks-sdk", specifier = "~=0.57" }, + { name = "pyspark", marker = "extra == 'cli'", specifier = "~=3.5.0" }, +] +provides-extras = ["cli"] + +[[package]] +name = "databricks-labs-lsql" +version = "0.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "databricks-labs-blueprint", extra = ["yaml"] }, + { name = "databricks-sdk" }, + { name = "sqlglot" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/05/0b12ab6efc3dd737ae06a509f513241a1d893f12149e8ebdefcdd23d2099/databricks_labs_lsql-0.14.2.tar.gz", hash = "sha256:4c4dddc12b27bab0587830419ef744565128a97fceec10f02b451b1ee007bb3e", size = 52776, upload-time = "2025-02-27T16:34:05.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/67/bc65eed315c45bfeb2df1fb3204894c45f16bef34c4cbbebbb0c528f9d71/databricks_labs_lsql-0.14.2-py3-none-any.whl", hash = "sha256:2f97c2bf633aa1569db77ffa234f9e456438687975d78bba568cdbb3e6fe86cb", size = 47965, upload-time = "2025-02-27T16:34:03.569Z" }, +] + +[[package]] +name = "databricks-sdk" +version = "0.62.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/16/1ad8be298b3135a3be0875afb97ce07a4fecd32489f8351bc4419e5e083b/databricks_sdk-0.62.0.tar.gz", hash = "sha256:12f8da735f74cba5265dcb0620c30941628f63787e6ea7e57cc5ed22d9e1b987", size = 725602, upload-time = "2025-08-06T11:31:06.415Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/cd/5fe949ec0b160130066d0ba82b4f499dc494a7405723836ebc37511f6bfc/databricks_sdk-0.62.0-py3-none-any.whl", hash = "sha256:79d4abe60306239985a5b718583923e13316c5417204017b0dadf74bccfcc6e1", size = 681789, upload-time = "2025-08-06T11:31:04.492Z" }, +] + +[[package]] +name = "google-auth" +version = "2.40.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/9b/e92ef23b84fa10a64ce4831390b7a4c2e53c0132568d99d4ae61d04c8855/google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77", size = 281029, upload-time = "2025-06-04T18:04:57.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "py4j" +version = "0.10.9.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b", size = 200481, upload-time = "2022-08-12T22:49:07.05Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322, upload-time = "2024-09-10T22:41:42.55Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pyspark" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py4j" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/62/36e50d38e5fe158e97cddec983b44f9417b1e205b02320e3c463b5f802fa/pyspark-3.5.6.tar.gz", hash = "sha256:f8b1c4360e41ab398c64904fae08740503bcb6bd389457d659fa6d9f2952cc48", size = 317359167, upload-time = "2025-05-27T08:24:20.82Z" } + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +] + +[[package]] +name = "requests" +version = "2.32.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, +] + +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, +] + +[[package]] +name = "sqlglot" +version = "27.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/aa/0c93a9974c0dc0726aa1a3ebc956f88bc5f6694d811b23d9a31085bb812b/sqlglot-27.6.0.tar.gz", hash = "sha256:413bc6c9978710bd97110e9a4363c517dfb92954d0a9fd2922e87c5707395deb", size = 5409152, upload-time = "2025-08-01T16:03:12.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/58/8b267b6b3c27ccb16367b49db2ac8c6032939f4fbffdc66dd0831833a7b9/sqlglot-27.6.0-py3-none-any.whl", hash = "sha256:828e6eea4e608f221c14f9e5dbe2f9f73142f11ba289d84f58c56ca44ee49d23", size = 496769, upload-time = "2025-08-01T16:03:09.914Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] From cb5fb49de97b3d03d258a52a7495b0918b7bb925 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 13 Aug 2025 12:57:13 +0200 Subject: [PATCH 03/37] remove src files --- .gitignore | 5 +- docs/dqx/docs/reference/api/__main__.md | 14 - docs/dqx/docs/reference/api/base.md | 234 ----- docs/dqx/docs/reference/api/check_funcs.md | 821 ------------------ .../dqx/docs/reference/api/checks_resolver.md | 26 - .../docs/reference/api/checks_serializer.md | 140 --- docs/dqx/docs/reference/api/checks_storage.md | 308 ------- .../docs/reference/api/checks_validator.md | 72 -- docs/dqx/docs/reference/api/cli.md | 123 --- docs/dqx/docs/reference/api/config.md | 198 ----- docs/dqx/docs/reference/api/config_loader.md | 42 - .../reference/api/contexts/application.md | 41 - .../docs/reference/api/contexts/workflows.md | 91 -- .../docs/reference/api/contexts/workspace.md | 27 - docs/dqx/docs/reference/api/engine.md | 318 ------- docs/dqx/docs/reference/api/executor.md | 159 ---- .../docs/reference/api/installer/install.md | 188 ---- docs/dqx/docs/reference/api/installer/logs.md | 41 - .../reference/api/installer/workflow_task.md | 85 -- .../api/installer/workflows_installer.md | 19 - .../api/llm/extract_yaml_checks_examples.md | 59 -- docs/dqx/docs/reference/api/llm/utils.md | 38 - docs/dqx/docs/reference/api/manager.md | 60 -- .../dqx/docs/reference/api/profiler/common.md | 39 - .../reference/api/profiler/dlt_generator.md | 36 - .../docs/reference/api/profiler/generator.md | 107 --- .../docs/reference/api/profiler/profiler.md | 104 --- .../dqx/docs/reference/api/profiler/runner.md | 48 - .../docs/reference/api/profiler/workflow.md | 24 - docs/dqx/docs/reference/api/rule.md | 202 ----- docs/dqx/docs/reference/api/runtime.md | 44 - docs/dqx/docs/reference/api/sidebar.json | 79 -- docs/dqx/docs/reference/api/utils.md | 169 ---- 33 files changed, 3 insertions(+), 3958 deletions(-) delete mode 100644 docs/dqx/docs/reference/api/__main__.md delete mode 100644 docs/dqx/docs/reference/api/base.md delete mode 100644 docs/dqx/docs/reference/api/check_funcs.md delete mode 100644 docs/dqx/docs/reference/api/checks_resolver.md delete mode 100644 docs/dqx/docs/reference/api/checks_serializer.md delete mode 100644 docs/dqx/docs/reference/api/checks_storage.md delete mode 100644 docs/dqx/docs/reference/api/checks_validator.md delete mode 100644 docs/dqx/docs/reference/api/cli.md delete mode 100644 docs/dqx/docs/reference/api/config.md delete mode 100644 docs/dqx/docs/reference/api/config_loader.md delete mode 100644 docs/dqx/docs/reference/api/contexts/application.md delete mode 100644 docs/dqx/docs/reference/api/contexts/workflows.md delete mode 100644 docs/dqx/docs/reference/api/contexts/workspace.md delete mode 100644 docs/dqx/docs/reference/api/engine.md delete mode 100644 docs/dqx/docs/reference/api/executor.md delete mode 100644 docs/dqx/docs/reference/api/installer/install.md delete mode 100644 docs/dqx/docs/reference/api/installer/logs.md delete mode 100644 docs/dqx/docs/reference/api/installer/workflow_task.md delete mode 100644 docs/dqx/docs/reference/api/installer/workflows_installer.md delete mode 100644 docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md delete mode 100644 docs/dqx/docs/reference/api/llm/utils.md delete mode 100644 docs/dqx/docs/reference/api/manager.md delete mode 100644 docs/dqx/docs/reference/api/profiler/common.md delete mode 100644 docs/dqx/docs/reference/api/profiler/dlt_generator.md delete mode 100644 docs/dqx/docs/reference/api/profiler/generator.md delete mode 100644 docs/dqx/docs/reference/api/profiler/profiler.md delete mode 100644 docs/dqx/docs/reference/api/profiler/runner.md delete mode 100644 docs/dqx/docs/reference/api/profiler/workflow.md delete mode 100644 docs/dqx/docs/reference/api/rule.md delete mode 100644 docs/dqx/docs/reference/api/runtime.md delete mode 100644 docs/dqx/docs/reference/api/sidebar.json delete mode 100644 docs/dqx/docs/reference/api/utils.md diff --git a/.gitignore b/.gitignore index 8e4363a76..dc2b4d2d4 100644 --- a/.gitignore +++ b/.gitignore @@ -166,5 +166,6 @@ dev/cleanup.py # docgen -docs/dqx/generated/** -!docs/dqx/generated/.gitkeep +docs/dqx/docs/reference/api +!docs/dqx/docs/reference/api/index.mdx + diff --git a/docs/dqx/docs/reference/api/__main__.md b/docs/dqx/docs/reference/api/__main__.md deleted file mode 100644 index 553de5aed..000000000 --- a/docs/dqx/docs/reference/api/__main__.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -sidebar_label: __main__ -title: databricks.labs.dqx.__main__ ---- - -#### me - -```python -@dqx.command -def me(w: WorkspaceClient, greeting: str) -``` - -Shows current username - diff --git a/docs/dqx/docs/reference/api/base.md b/docs/dqx/docs/reference/api/base.md deleted file mode 100644 index bf766893c..000000000 --- a/docs/dqx/docs/reference/api/base.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -sidebar_label: base -title: databricks.labs.dqx.base ---- - -## DQEngineBase Objects - -```python -class DQEngineBase(abc.ABC) -``` - -#### ws - -```python -@cached_property -def ws() -> WorkspaceClient -``` - -Cached property to verify and return the workspace client. - -#### spark - -```python -@cached_property -def spark() -> SparkSession -``` - -Cached property return the SparkSession. - -## DQEngineCoreBase Objects - -```python -class DQEngineCoreBase(DQEngineBase) -``` - -#### apply\_checks - -```python -@abc.abstractmethod -def apply_checks(df: DataFrame, - checks: list[DQRule], - ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame -``` - -Applies data quality checks to a given dataframe. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. -- `ref_dfs`: reference dataframes to use in the checks, if applicable - -**Returns**: - -dataframe with errors and warning result columns - -#### apply\_checks\_and\_split - -```python -@abc.abstractmethod -def apply_checks_and_split( - df: DataFrame, - checks: list[DQRule], - ref_dfs: dict[str, DataFrame] | None = None -) -> tuple[DataFrame, DataFrame] -``` - -Applies data quality checks to a given dataframe and split it into two ("good" and "bad"), - -according to the data quality checks. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. -- `ref_dfs`: reference dataframes to use in the checks, if applicable - -**Returns**: - -two dataframes - "good" which includes warning rows but no result columns, and "data" having -error and warning rows and corresponding result columns - -#### apply\_checks\_by\_metadata\_and\_split - -```python -@abc.abstractmethod -def apply_checks_by_metadata_and_split( - df: DataFrame, - checks: list[dict], - custom_check_functions: dict[str, Any] | None = None, - ref_dfs: dict[str, DataFrame] | None = None -) -> tuple[DataFrame, DataFrame] -``` - -Wrapper around `apply_checks_and_split` for use in the metadata-driven pipelines. The main difference - -is how the checks are specified - instead of using functions directly, they are described as function name plus -arguments. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - -it will be used as an error/warning message, or `null` if it's evaluated to `false` -* `name` - name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), -and `warn` (data is going into both dataframes) -- `df`0: dictionary with custom check functions (eg. `df`1globals()`df`1 of the calling module). -If not specified, then only built-in functions are used for the checks. -- `df`3: reference dataframes to use in the checks, if applicable - -**Returns**: - -two dataframes - "good" which includes warning rows but no result columns, and "bad" having -error and warning rows and corresponding result columns - -#### apply\_checks\_by\_metadata - -```python -@abc.abstractmethod -def apply_checks_by_metadata( - df: DataFrame, - checks: list[dict], - custom_check_functions: dict[str, Any] | None = None, - ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame -``` - -Wrapper around `apply_checks` for use in the metadata-driven pipelines. The main difference - -is how the checks are specified - instead of using functions directly, they are described as function name plus -arguments. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - -it will be used as an error/warning message, or `null` if it's evaluated to `false` -* `name` - name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), -and `warn` (data is going into both dataframes) -- `df`0: dictionary with custom check functions (eg. `df`1globals()`df`1 of calling module). -If not specified, then only built-in functions are used for the checks. -- `df`3: reference dataframes to use in the checks, if applicable - -**Returns**: - -dataframe with errors and warning result columns - -#### validate\_checks - -```python -@staticmethod -@abc.abstractmethod -def validate_checks( - checks: list[dict], - custom_check_functions: dict[str, Any] | None = None -) -> ChecksValidationStatus -``` - -Validate the input dict to ensure they conform to expected structure and types. - -Each check can be a dictionary. The function validates -the presence of required keys, the existence and callability of functions, and the types -of arguments passed to these functions. - -**Arguments**: - -- `checks`: List of checks to apply to the dataframe. Each check should be a dictionary. -- `custom_check_functions`: Optional dictionary with custom check functions. - -**Returns**: - -`ValidationStatus`: The validation status. - -#### get\_invalid - -```python -@abc.abstractmethod -def get_invalid(df: DataFrame) -> DataFrame -``` - -Get records that violate data quality checks (records with warnings and errors). -@param df: input DataFrame. -@return: dataframe with error and warning rows and corresponding result columns. - -#### get\_valid - -```python -@abc.abstractmethod -def get_valid(df: DataFrame) -> DataFrame -``` - -Get records that don't violate data quality checks (records with warnings but no errors). -@param df: input DataFrame. -@return: dataframe with warning rows but no result columns. - -#### load\_checks\_from\_local\_file - -```python -@staticmethod -@abc.abstractmethod -def load_checks_from_local_file(filepath: str) -> list[dict] -``` - -Load checks (dq rules) from a file (json or yaml) in the local file system. - -This does not require installation of DQX in the workspace. -The returning checks can be used as input for `apply_checks_by_metadata` function. - -**Arguments**: - -- `filepath`: path to a file containing the checks. - -**Returns**: - -list of dq rules - -#### save\_checks\_in\_local\_file - -```python -@staticmethod -@abc.abstractmethod -def save_checks_in_local_file(checks: list[dict], filepath: str) -``` - -Save checks (dq rules) to a file (yaml or json) in the local file system. - -**Arguments**: - -- `checks`: list of dq rules to save -- `filepath`: path to a file containing the checks. - diff --git a/docs/dqx/docs/reference/api/check_funcs.md b/docs/dqx/docs/reference/api/check_funcs.md deleted file mode 100644 index 163f7ac77..000000000 --- a/docs/dqx/docs/reference/api/check_funcs.md +++ /dev/null @@ -1,821 +0,0 @@ ---- -sidebar_label: check_funcs -title: databricks.labs.dqx.check_funcs ---- - -## DQPattern Objects - -```python -class DQPattern(Enum) -``` - -Enum class to represent DQ patterns used to match data in columns. - -#### make\_condition - -```python -def make_condition(condition: Column, message: Column | str, - alias: str) -> Column -``` - -Helper function to create a condition column. - -**Arguments**: - -- `condition`: condition expression. -Pass the check if the condition evaluates to False. -Fail the check if condition evaluates to True. -- `message`: message to output - it could be either `Column` object, or string constant -- `alias`: name for the resulting column - -**Returns**: - -an instance of `Column` type, that either returns string if condition is evaluated to `true`, -or `null` if condition is evaluated to `false` - -#### matches\_pattern - -```python -def matches_pattern(column: str | Column, pattern: DQPattern) -> Column -``` - -Checks whether the values in the input column match a given pattern. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `pattern`: pattern to match against - -**Returns**: - -Column object for condition - -#### is\_not\_null\_and\_not\_empty - -```python -@register_rule("row") -def is_not_null_and_not_empty(column: str | Column, - trim_strings: bool | None = False) -> Column -``` - -Checks whether the values in the input column are not null and not empty. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `trim_strings`: boolean flag to trim spaces from strings - -**Returns**: - -Column object for condition - -#### is\_not\_empty - -```python -@register_rule("row") -def is_not_empty(column: str | Column) -> Column -``` - -Checks whether the values in the input column are not empty (but may be null). - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression - -**Returns**: - -Column object for condition - -#### is\_not\_null - -```python -@register_rule("row") -def is_not_null(column: str | Column) -> Column -``` - -Checks whether the values in the input column are not null. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression - -**Returns**: - -Column object for condition - -#### is\_not\_null\_and\_is\_in\_list - -```python -@register_rule("row") -def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column -``` - -Checks whether the values in the input column are not null and present in the list of allowed values. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `allowed`: list of allowed values (actual values or Column objects) - -**Returns**: - -Column object for condition - -#### is\_in\_list - -```python -@register_rule("row") -def is_in_list(column: str | Column, allowed: list) -> Column -``` - -Checks whether the values in the input column are present in the list of allowed values - -(null values are allowed). - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `allowed`: list of allowed values (actual values or Column objects) - -**Returns**: - -Column object for condition - -#### sql\_expression - -```python -@register_rule("row") -def sql_expression(expression: str, - msg: str | None = None, - name: str | None = None, - negate: bool = False, - columns: list[str | Column] | None = None) -> Column -``` - -Checks whether the condition provided as an SQL expression is met. - -**Arguments**: - -- `expression`: SQL expression. Fail if expression evaluates to True, pass if it evaluates to False. -- `msg`: optional message of the `Column` type, automatically generated if None -- `name`: optional name of the resulting column, automatically generated if None -- `negate`: if the condition should be negated (true) or not. For example, "col is not null" will mark null -values as "bad". Although sometimes it's easier to specify it other way around "col is null" + negate set to False -- `columns`: optional list of columns to be used for reporting. Unused in the actual logic. - -**Returns**: - -new Column - -#### is\_older\_than\_col2\_for\_n\_days - -```python -@register_rule("row") -def is_older_than_col2_for_n_days(column1: str | Column, - column2: str | Column, - days: int = 0, - negate: bool = False) -> Column -``` - -Checks whether the values in one input column are at least N days older than the values in another column. - -**Arguments**: - -- `column1`: first column to check; can be a string column name or a column expression -- `column2`: second column to check; can be a string column name or a column expression -- `days`: number of days -- `negate`: if the condition should be negated (true) or not; if negated, the check will fail when values in the -first column are at least N days older than values in the second column - -**Returns**: - -new Column - -#### is\_older\_than\_n\_days - -```python -@register_rule("row") -def is_older_than_n_days(column: str | Column, - days: int, - curr_date: Column | None = None, - negate: bool = False) -> Column -``` - -Checks whether the values in the input column are at least N days older than the current date. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `days`: number of days -- `curr_date`: (optional) set current date -- `negate`: if the condition should be negated (true) or not; if negated, the check will fail when values in the -first column are at least N days older than values in the second column - -**Returns**: - -new Column - -#### is\_not\_in\_future - -```python -@register_rule("row") -def is_not_in_future(column: str | Column, - offset: int = 0, - curr_timestamp: Column | None = None) -> Column -``` - -Checks whether the values in the input column contain a timestamp that is not in the future, - -where 'future' is defined as current_timestamp + offset (in seconds). - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `offset`: offset (in seconds) to add to the current timestamp at time of execution -- `curr_timestamp`: (optional) set current timestamp - -**Returns**: - -new Column - -#### is\_not\_in\_near\_future - -```python -@register_rule("row") -def is_not_in_near_future(column: str | Column, - offset: int = 0, - curr_timestamp: Column | None = None) -> Column -``` - -Checks whether the values in the input column contain a timestamp that is not in the near future, - -where 'near future' is defined as greater than the current timestamp -but less than the current_timestamp + offset (in seconds). - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `offset`: offset (in seconds) to add to the current timestamp at time of execution -- `curr_timestamp`: (optional) set current timestamp - -**Returns**: - -new Column - -#### is\_not\_less\_than - -```python -@register_rule("row") -def is_not_less_than( - column: str | Column, - limit: int | datetime.date | datetime.datetime | str | Column | None = None -) -> Column -``` - -Checks whether the values in the input column are not less than the provided limit. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `limit`: limit to use in the condition as number, date, timestamp, column name or sql expression - -**Returns**: - -new Column - -#### is\_not\_greater\_than - -```python -@register_rule("row") -def is_not_greater_than( - column: str | Column, - limit: int | datetime.date | datetime.datetime | str | Column | None = None -) -> Column -``` - -Checks whether the values in the input column are not greater than the provided limit. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `limit`: limit to use in the condition as number, date, timestamp, column name or sql expression - -**Returns**: - -new Column - -#### is\_in\_range - -```python -@register_rule("row") -def is_in_range( - column: str | Column, - min_limit: int | datetime.date | datetime.datetime | str | Column - | None = None, - max_limit: int | datetime.date | datetime.datetime | str | Column - | None = None -) -> Column -``` - -Checks whether the values in the input column are in the provided limits (inclusive of both boundaries). - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `min_limit`: min limit to use in the condition as number, date, timestamp, column name or sql expression -- `max_limit`: max limit to use in the condition as number, date, timestamp, column name or sql expression - -**Returns**: - -new Column - -#### is\_not\_in\_range - -```python -@register_rule("row") -def is_not_in_range( - column: str | Column, - min_limit: int | datetime.date | datetime.datetime | str | Column - | None = None, - max_limit: int | datetime.date | datetime.datetime | str | Column - | None = None -) -> Column -``` - -Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries). - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `min_limit`: min limit to use in the condition as number, date, timestamp, column name or sql expression -- `max_limit`: min limit to use in the condition as number, date, timestamp, column name or sql expression - -**Returns**: - -new Column - -#### regex\_match - -```python -@register_rule("row") -def regex_match(column: str | Column, - regex: str, - negate: bool = False) -> Column -``` - -Checks whether the values in the input column matches a given regex. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `regex`: regex to check -- `negate`: if the condition should be negated (true) or not - -**Returns**: - -Column object for condition - -#### is\_not\_null\_and\_not\_empty\_array - -```python -@register_rule("row") -def is_not_null_and_not_empty_array(column: str | Column) -> Column -``` - -Checks whether the values in the array input column are not null and not empty. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression - -**Returns**: - -Column object for condition - -#### is\_valid\_date - -```python -@register_rule("row") -def is_valid_date(column: str | Column, - date_format: str | None = None) -> Column -``` - -Checks whether the values in the input column have valid date formats. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `date_format`: date format (e.g. 'yyyy-mm-dd') - -**Returns**: - -Column object for condition - -#### is\_valid\_timestamp - -```python -@register_rule("row") -def is_valid_timestamp(column: str | Column, - timestamp_format: str | None = None) -> Column -``` - -Checks whether the values in the input column have valid timestamp formats. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `timestamp_format`: timestamp format (e.g. 'yyyy-mm-dd HH:mm:ss') - -**Returns**: - -Column object for condition - -#### is\_valid\_ipv4\_address - -```python -@register_rule("row") -def is_valid_ipv4_address(column: str | Column) -> Column -``` - -Checks whether the values in the input column have valid IPv4 address formats. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression - -**Returns**: - -Column object for condition - -#### is\_ipv4\_address\_in\_cidr - -```python -@register_rule("row") -def is_ipv4_address_in_cidr(column: str | Column, cidr_block: str) -> Column -``` - -Checks if an IP column value falls within the given CIDR block. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression -- `cidr_block`: CIDR block string (e.g., '192.168.1.0/24') - -**Raises**: - -- `ValueError`: If cidr_block is not a valid string in CIDR notation. - -**Returns**: - -Column object for condition - -#### is\_data\_fresh - -```python -@register_rule("row") -def is_data_fresh( - column: str | Column, - max_age_minutes: int, - base_timestamp: str | datetime.date | datetime.datetime | Column - | None = None -) -> Column -``` - -Checks whether the values in the timestamp column are not older than the specified number of minutes from the base timestamp column. - -This is useful for identifying stale data due to delayed pipelines and helps catch upstream issues early. - -**Arguments**: - -- `column`: column to check; can be a string column name or a column expression containing timestamp values -- `max_age_minutes`: maximum age in minutes before data is considered stale -- `base_timestamp`: (optional) set base timestamp column from which the stale check is calculated, if not provided uses current_timestamp() - -**Returns**: - -Column object for condition - -#### is\_unique - -```python -@register_rule("dataset") -def is_unique(columns: list[str | Column], - nulls_distinct: bool = True, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Build a uniqueness check condition and closure for dataset-level validation. - -This function checks whether the specified columns contain unique values within the dataset -and reports rows with duplicate combinations. When `nulls_distinct` -is True (default), rows with NULLs are treated as distinct (SQL ANSI behavior); otherwise, -NULLs are treated as equal when checking for duplicates. - -In streaming, uniqueness is validated within individual micro-batches only. - -**Arguments**: - -- `columns`: List of column names (str) or Spark Column expressions to validate for uniqueness. -- `nulls_distinct`: Whether NULLs are treated as distinct (default: True). -- `row_filter`: Optional SQL expression for filtering rows before checking uniqueness. -Auto-injected from the check filter. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for uniqueness violations. -- A closure that applies the uniqueness check and adds the necessary condition/count columns. - -#### foreign\_key - -```python -@register_rule("dataset") -def foreign_key(columns: list[str | Column], - ref_columns: list[str | Column], - ref_df_name: str | None = None, - ref_table: str | None = None, - negate: bool = False, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Build a foreign key check condition and closure for dataset-level validation. - -This function verifies that values in the specified foreign key columns exist (or don't exist, if `negate=True`) in -the corresponding reference columns of another DataFrame or table. Rows where -foreign key values do not match the reference are reported as violations. - -NULL values in the foreign key columns are ignored (SQL ANSI behavior). - -**Arguments**: - -- `columns`: List of column names (str) or Column expressions in the dataset (foreign key). -- `ref_columns`: List of column names (str) or Column expressions in the reference dataset. -- `ref_df_name`: Name of the reference DataFrame (used when passing DataFrames directly). -- `ref_table`: Name of the reference table (used when reading from catalog). -- `row_filter`: Optional SQL expression for filtering rows before checking the foreign key. -Auto-injected from the check filter. -- `negate`: If True, the condition is negated (i.e., the check fails when the foreign key values exist in the -reference DataFrame/Table). If False, the check fails when the foreign key values do not exist in the reference. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for foreign key violations. -- A closure that applies the foreign key validation by joining against the reference. - -#### sql\_query - -```python -@register_rule("dataset") -def sql_query(query: str, - merge_columns: list[str], - msg: str | None = None, - name: str | None = None, - negate: bool = False, - condition_column: str = "condition", - input_placeholder: str = "input_view", - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Checks whether the condition column generated by SQL query is met. - -**Arguments**: - -- `query`: SQL query that must return as a minimum a condition column and -all merge columns. The resulting DataFrame is automatically joined back to the input DataFrame. -using the merge_columns. Reference DataFrames when provided in the ref_dfs parameter are registered as temp view. -- `condition_column`: Column name indicating violation (boolean). Fail the check if True, pass it if False -- `merge_columns`: List of columns for join back to the input DataFrame. -They must provide a unique key for the join, otherwise a duplicate records may be produced. -- `msg`: Optional custom message or Column expression. -- `name`: Optional name for the result. -- `negate`: If True, the condition is negated (i.e., the check fails when the condition is False). -- `input_placeholder`: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the -input DataFrame on which the checks are applied. -- `row_filter`: Optional SQL expression for filtering rows before checking the foreign key. -Auto-injected from the check filter. - -**Returns**: - -Tuple (condition column, apply function). - -#### is\_aggr\_not\_greater\_than - -```python -@register_rule("dataset") -def is_aggr_not_greater_than( - column: str | Column, - limit: int | float | str | Column, - aggr_type: str = "count", - group_by: list[str | Column] | None = None, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Build an aggregation check condition and closure for dataset-level validation. - -This function verifies that an aggregation (count, sum, avg, min, max) on a column -or group of columns does not exceed a specified limit. Rows where the aggregation -result exceeds the limit are flagged. - -**Arguments**: - -- `column`: Column name (str) or Column expression to aggregate. -- `limit`: Numeric value, column name, or SQL expression for the limit. -- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). -- `group_by`: Optional list of column names or Column expressions to group by. -- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for aggregation limit violations. -- A closure that applies the aggregation check and adds the necessary condition/metric columns. - -#### is\_aggr\_not\_less\_than - -```python -@register_rule("dataset") -def is_aggr_not_less_than( - column: str | Column, - limit: int | float | str | Column, - aggr_type: str = "count", - group_by: list[str | Column] | None = None, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Build an aggregation check condition and closure for dataset-level validation. - -This function verifies that an aggregation (count, sum, avg, min, max) on a column -or group of columns is not below a specified limit. Rows where the aggregation -result is below the limit are flagged. - -**Arguments**: - -- `column`: Column name (str) or Column expression to aggregate. -- `limit`: Numeric value, column name, or SQL expression for the limit. -- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). -- `group_by`: Optional list of column names or Column expressions to group by. -- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for aggregation limit violations. -- A closure that applies the aggregation check and adds the necessary condition/metric columns. - -#### is\_aggr\_equal - -```python -@register_rule("dataset") -def is_aggr_equal(column: str | Column, - limit: int | float | str | Column, - aggr_type: str = "count", - group_by: list[str | Column] | None = None, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Build an aggregation check condition and closure for dataset-level validation. - -This function verifies that an aggregation (count, sum, avg, min, max) on a column -or group of columns is equal to a specified limit. Rows where the aggregation -result is not equal to the limit are flagged. - -**Arguments**: - -- `column`: Column name (str) or Column expression to aggregate. -- `limit`: Numeric value, column name, or SQL expression for the limit. -- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). -- `group_by`: Optional list of column names or Column expressions to group by. -- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for aggregation limit violations. -- A closure that applies the aggregation check and adds the necessary condition/metric columns. - -#### is\_aggr\_not\_equal - -```python -@register_rule("dataset") -def is_aggr_not_equal( - column: str | Column, - limit: int | float | str | Column, - aggr_type: str = "count", - group_by: list[str | Column] | None = None, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Build an aggregation check condition and closure for dataset-level validation. - -This function verifies that an aggregation (count, sum, avg, min, max) on a column -or group of columns is not equal to a specified limit. Rows where the aggregation -result is equal to the limit are flagged. - -**Arguments**: - -- `column`: Column name (str) or Column expression to aggregate. -- `limit`: Numeric value, column name, or SQL expression for the limit. -- `aggr_type`: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). -- `group_by`: Optional list of column names or Column expressions to group by. -- `row_filter`: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for aggregation limit violations. -- A closure that applies the aggregation check and adds the necessary condition/metric columns. - -#### compare\_datasets - -```python -@register_rule("dataset") -def compare_datasets(columns: list[str | Column], - ref_columns: list[str | Column], - ref_df_name: str | None = None, - ref_table: str | None = None, - check_missing_records: bool | None = False, - exclude_columns: list[str | Column] | None = None, - null_safe_row_matching: bool | None = True, - null_safe_column_value_matching: bool | None = True, - row_filter: str | None = None) -> tuple[Column, Callable] -``` - -Dataset-level check that compares two datasets and returns a condition for changed rows, - -with details on a row and column-level differences. -Only columns that are common across both datasets will be compared. Mismatched columns are ignored. -Detailed information about the differences is provided in the condition column. -The comparison does not support Map types (any column comparison on map type is skipped automatically). - -The log containing detailed differences is written to the message field of the check result as JSON string. -Example: `{"row_missing":false,"row_extra":true,"changed":{"val":{"df":"val1"}}}` - -**Arguments**: - -- `columns`: List of columns to use for row matching with the reference DataFrame -(can be a list of string column names or column expressions). -Only simple column expressions are supported, e.g. F.col("col_name") -- `ref_columns`: List of columns in the reference DataFrame or Table to row match against the source DataFrame -(can be a list of string column names or column expressions). The `columns` parameter is matched with `ref_columns` -by position, so the order of the provided columns in both lists must be exactly aligned. -Only simple column expressions are supported, e.g. F.col("col_name") -- `ref_df_name`: Name of the reference DataFrame (used when passing DataFrames directly). -- `ref_table`: Name of the reference table (used when reading from catalog). -- `check_missing_records`: Perform FULL OUTER JOIN between the DataFrames to find also records -that could be missing from the DataFrame. Use this with caution as it may produce output with more rows -than in the original DataFrame. -- `exclude_columns`: List of columns to exclude from the value comparison but not from row matching -(can be a list of string column names or column expressions). -Only simple column expressions are supported, e.g. F.col("col_name") -The parameter does not alter the list of columns used to determine row matches, -it only controls which columns are skipped during the column value comparison. -- `null_safe_row_matching`: If True, treats nulls as equal when matching rows. -- `columns`0: If True, treats nulls as equal when matching column values. -If enabled (NULL, NULL) column values are equal and matching. -- `columns`1: Optional SQL expression to filter rows in the input DataFrame. -Auto-injected from the check filter. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for comparison violations. -- A closure that applies the comparison validation. - -#### is\_data\_fresh\_per\_time\_window - -```python -@register_rule("dataset") -def is_data_fresh_per_time_window( - column: str | Column, - window_minutes: int, - min_records_per_window: int, - lookback_windows: int | None = None, - row_filter: str | None = None, - curr_timestamp: Column | None = None) -> tuple[Column, Callable] -``` - -Build a completeness freshness check that validates records arrive at least every X minutes - -with a threshold for the expected number of rows per time window. - -If `lookback_windows` is provided, only data within that lookback period will be validated. -If omitted, the entire dataset will be checked. - -**Arguments**: - -- `column`: Column name (str) or Column expression containing timestamps to check. -- `window_minutes`: Time window in minutes to check for data arrival. -- `min_records_per_window`: Minimum number of records expected per time window. -- `lookback_windows`: Optional number of time windows to look back from `curr_timestamp`. -This filters records to include only those within the specified number of time windows from `curr_timestamp`. -If no lookback is provided, the check is applied to the entire dataset. -- `row_filter`: Optional SQL expression to filter rows before checking. -- `curr_timestamp`: Optional current timestamp column. If not provided, current_timestamp() function is used. - -**Returns**: - -A tuple of: -- A Spark Column representing the condition for missing data within a time window. -- A closure that applies the completeness check and adds the necessary condition columns. - diff --git a/docs/dqx/docs/reference/api/checks_resolver.md b/docs/dqx/docs/reference/api/checks_resolver.md deleted file mode 100644 index 7d02bf610..000000000 --- a/docs/dqx/docs/reference/api/checks_resolver.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -sidebar_label: checks_resolver -title: databricks.labs.dqx.checks_resolver ---- - -#### resolve\_check\_function - -```python -def resolve_check_function(function_name: str, - custom_check_functions: dict[str, Any] - | None = None, - fail_on_missing: bool = True) -> Callable | None -``` - -Resolves a function by name from the predefined functions and custom checks. - -**Arguments**: - -- `function_name`: name of the function to resolve. -- `custom_check_functions`: dictionary with custom check functions (eg. ``globals()`` of the calling module). -- `fail_on_missing`: if True, raise an AttributeError if the function is not found. - -**Returns**: - -function or None if not found. - diff --git a/docs/dqx/docs/reference/api/checks_serializer.md b/docs/dqx/docs/reference/api/checks_serializer.md deleted file mode 100644 index 4eb56c3ef..000000000 --- a/docs/dqx/docs/reference/api/checks_serializer.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -sidebar_label: checks_serializer -title: databricks.labs.dqx.checks_serializer ---- - -#### serialize\_checks\_from\_dataframe - -```python -def serialize_checks_from_dataframe(df: DataFrame, - run_config_name: str = "default" - ) -> list[dict] -``` - -Converts a list of quality checks defined in a DataFrame to a list of quality checks - -defined as Python dictionaries. - -**Arguments**: - -- `df`: DataFrame with data quality check rules. Each row should define a check. Rows should -have the following columns: -* `name` - Name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` (data is going into both dataframes) -* `check` - DQX check function used in the check; A `StructType` column defining the data quality check -* `filter` - Expression for filtering data quality checks -* `run_config_name` (optional) - Run configuration name for storing checks across runs -* `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. -- `run_config_name`: Run configuration name for filtering quality rules - -**Returns**: - -List of data quality check specifications as a Python dictionary - -#### deserialize\_checks\_to\_dataframe - -```python -def deserialize_checks_to_dataframe( - spark: SparkSession, - checks: list[dict], - run_config_name: str = "default") -> DataFrame -``` - -Converts a list of quality checks defined as Python dictionaries to a DataFrame. - -**Arguments**: - -- `spark`: Spark session. -- `checks`: list of check specifications as Python dictionaries. Each check consists of the following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to - true (it will be used as an error/warning message) or `null` if it's evaluated to `false` -* `name` - Name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` - (data is going into both dataframes) -* `filter` (optional) - Expression for filtering data quality checks -* `checks`0 (optional) - User-defined key-value pairs added to metadata generated by the check. -- `checks`1: Run configuration name for storing quality checks across runs - -**Returns**: - -DataFrame with data quality check rules - -#### deserialize\_checks - -```python -def deserialize_checks( - checks: list[dict], - custom_checks: dict[str, Any] | None = None) -> list[DQRule] -``` - -Converts a list of quality checks defined as Python dictionaries to a list of `DQRule` objects. - -**Arguments**: - -- `checks`: list of dictionaries describing checks. Each check is a dictionary -consisting of following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true -- it will be used as an error/warning message, or `null` if it's evaluated to `false` -* `name` - name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), -and `warn` (data is going into both dataframes) -* `filter` (optional) - Expression for filtering data quality checks -* `checks`0 (optional) - User-defined key-value pairs added to metadata generated by the check. -- `checks`1: dictionary with custom check functions (eg. `checks`2globals()`checks`2 of the calling module). -If not specified, then only built-in functions are used for the checks. - -**Returns**: - -list of data quality check rules - -#### serialize\_checks - -```python -def serialize_checks(checks: list[DQRule]) -> list[dict] -``` - -Converts a list of quality checks defined as `DQRule` objects to a list of quality checks - -defined as Python dictionaries. - -**Arguments**: - -- `checks`: List of DQRule instances to convert. - -**Returns**: - -List of dictionaries representing the DQRule instances. - -#### serialize\_checks\_to\_bytes - -```python -def serialize_checks_to_bytes(checks: list[dict], file_path: Path) -> bytes -``` - -Serializes a list of checks to bytes in json or yaml (default) format. - -**Arguments**: - -- `checks`: List of checks to serialize. -- `file_path`: Path to the file where the checks will be serialized. - -**Returns**: - -Serialized checks as bytes. - -#### get\_file\_deserializer - -```python -def get_file_deserializer(filepath: str) -> Callable -``` - -Get the deserializer function based on file. - -**Arguments**: - -- `filepath`: Path to the file. - -**Returns**: - -Deserializer function. - diff --git a/docs/dqx/docs/reference/api/checks_storage.md b/docs/dqx/docs/reference/api/checks_storage.md deleted file mode 100644 index c7e874775..000000000 --- a/docs/dqx/docs/reference/api/checks_storage.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -sidebar_label: checks_storage -title: databricks.labs.dqx.checks_storage ---- - -## ChecksStorageHandler Objects - -```python -class ChecksStorageHandler(ABC, Generic[T]) -``` - -Abstract base class for handling storage of quality rules (checks). - -#### load - -```python -@abstractmethod -def load(config: T) -> list[dict] -``` - -Load quality rules from the source. - -The returned checks can be used as input for `apply_checks_by_metadata` or -`apply_checks_by_metadata_and_split` functions. - -**Arguments**: - -- `config`: configuration for loading checks, including the table location and run configuration name. - -**Returns**: - -list of dq rules or raise an error if checks file is missing or is invalid. - -#### save - -```python -@abstractmethod -def save(checks: list[dict], config: T) -> None -``` - -Save quality rules to the target. - -## TableChecksStorageHandler Objects - -```python -class TableChecksStorageHandler(ChecksStorageHandler[TableChecksStorageConfig] - ) -``` - -Handler for storing quality rules (checks) in a Delta table in the workspace. - -#### load - -```python -def load(config: TableChecksStorageConfig) -> list[dict] -``` - -Load checks (dq rules) from a Delta table in the workspace. - -**Arguments**: - -- `config`: configuration for loading checks, including the table location and run configuration name. - -**Returns**: - -list of dq rules or raise an error if checks table is missing or is invalid. - -#### save - -```python -def save(checks: list[dict], config: TableChecksStorageConfig) -> None -``` - -Save checks to a Delta table in the workspace. - -**Arguments**: - -- `checks`: list of dq rules to save -- `config`: configuration for saving checks, including the table location and run configuration name. - -**Raises**: - -- `ValueError`: if the table name is not provided - -## WorkspaceFileChecksStorageHandler Objects - -```python -class WorkspaceFileChecksStorageHandler( - ChecksStorageHandler[WorkspaceFileChecksStorageConfig]) -``` - -Handler for storing quality rules (checks) in a file (json or yaml) in the workspace. - -#### load - -```python -def load(config: WorkspaceFileChecksStorageConfig) -> list[dict] -``` - -Load checks (dq rules) from a file (json or yaml) in the workspace. - -This does not require installation of DQX in the workspace. - -**Arguments**: - -- `config`: configuration for loading checks, including the file location and storage type. - -**Returns**: - -list of dq rules or raise an error if checks file is missing or is invalid. - -#### save - -```python -def save(checks: list[dict], config: WorkspaceFileChecksStorageConfig) -> None -``` - -Save checks (dq rules) to yaml file in the workspace. - -This does not require installation of DQX in the workspace. - -**Arguments**: - -- `checks`: list of dq rules to save -- `config`: configuration for saving checks, including the file location and storage type. - -## FileChecksStorageHandler Objects - -```python -class FileChecksStorageHandler(ChecksStorageHandler[FileChecksStorageConfig]) -``` - -Handler for storing quality rules (checks) in a file (json or yaml) in the local filesystem. - -#### load - -```python -def load(config: FileChecksStorageConfig) -> list[dict] -``` - -Load checks (dq rules) from a file (json or yaml) in the local filesystem. - -**Arguments**: - -- `config`: configuration for loading checks, including the file location. - -**Raises**: - -- `ValueError`: if the file path is not provided -- `FileNotFoundError`: if the file path does not exist - -**Returns**: - -list of dq rules or raise an error if checks file is missing or is invalid. - -#### save - -```python -def save(checks: list[dict], config: FileChecksStorageConfig) -> None -``` - -Save checks (dq rules) to a file (json or yaml) in the local filesystem. - -**Arguments**: - -- `checks`: list of dq rules to save -- `config`: configuration for saving checks, including the file location. - -**Raises**: - -- `ValueError`: if the file path is not provided -- `FileNotFoundError`: if the file path does not exist - -## InstallationChecksStorageHandler Objects - -```python -class InstallationChecksStorageHandler( - ChecksStorageHandler[InstallationChecksStorageConfig]) -``` - -Handler for storing quality rules (checks) defined in the installation configuration. - -#### load - -```python -def load(config: InstallationChecksStorageConfig) -> list[dict] -``` - -Load checks (dq rules) from the installation configuration. - -**Arguments**: - -- `config`: configuration for loading checks, including the run configuration name and method. - -**Raises**: - -- `NotFound`: if the checks file or table is not found in the installation. - -**Returns**: - -list of dq rules or raise an error if checks file is missing or is invalid. - -#### save - -```python -def save(checks: list[dict], config: InstallationChecksStorageConfig) -> None -``` - -Save checks (dq rules) to yaml file or table in the installation folder. - -This will overwrite existing checks file or table. - -**Arguments**: - -- `checks`: list of dq rules to save -- `config`: configuration for saving checks, including the run configuration name, method, and table location. - -## VolumeFileChecksStorageHandler Objects - -```python -class VolumeFileChecksStorageHandler( - ChecksStorageHandler[VolumeFileChecksStorageConfig]) -``` - -Handler for storing quality rules (checks) in a file (json or yaml) in a Unity Catalog volume. - -#### load - -```python -def load(config: VolumeFileChecksStorageConfig) -> list[dict] -``` - -Load checks (dq rules) from a file (json or yaml) in a Unity Catalog volume. - -**Arguments**: - -- `config`: configuration for loading checks, including the file location and storage type. - -**Returns**: - -list of dq rules or raise an error if checks file is missing or is invalid. - -#### save - -```python -def save(checks: list[dict], config: VolumeFileChecksStorageConfig) -> None -``` - -Save checks (dq rules) to yaml file in a Unity Catalog volume. - -This does not require installation of DQX in a Unity Catalog volume. - -**Arguments**: - -- `checks`: list of dq rules to save -- `config`: configuration for saving checks, including the file location and storage type. - -## BaseChecksStorageHandlerFactory Objects - -```python -class BaseChecksStorageHandlerFactory(ABC) -``` - -Abstract base class for factories that create storage handlers for checks. - -#### create - -```python -@abstractmethod -def create(config: BaseChecksStorageConfig) -> ChecksStorageHandler -``` - -Abstract method to create a handler based on the type of the provided configuration object. - -**Arguments**: - -- `config`: Configuration object for loading or saving checks. - -**Returns**: - -An instance of the corresponding BaseChecksStorageHandler. - -## ChecksStorageHandlerFactory Objects - -```python -class ChecksStorageHandlerFactory(BaseChecksStorageHandlerFactory) -``` - -#### create - -```python -def create(config: BaseChecksStorageConfig) -> ChecksStorageHandler -``` - -Factory method to create a handler based on the type of the provided configuration object. - -**Arguments**: - -- `config`: Configuration object for loading or saving checks. - -**Raises**: - -- `ValueError`: If the configuration type is unsupported. - -**Returns**: - -An instance of the corresponding BaseChecksStorageHandler. - diff --git a/docs/dqx/docs/reference/api/checks_validator.md b/docs/dqx/docs/reference/api/checks_validator.md deleted file mode 100644 index 0b064a4cf..000000000 --- a/docs/dqx/docs/reference/api/checks_validator.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -sidebar_label: checks_validator -title: databricks.labs.dqx.checks_validator ---- - -## ChecksValidationStatus Objects - -```python -@dataclass(frozen=True) -class ChecksValidationStatus() -``` - -Class to represent the validation status. - -#### add\_error - -```python -def add_error(error: str) -``` - -Add an error to the validation status. - -#### add\_errors - -```python -def add_errors(errors: list[str]) -``` - -Add an error to the validation status. - -#### has\_errors - -```python -@property -def has_errors() -> bool -``` - -Check if there are any errors in the validation status. - -#### errors - -```python -@property -def errors() -> list[str] -``` - -Get the list of errors in the validation status. - -#### to\_string - -```python -def to_string() -> str -``` - -Convert the validation status to a string. - -#### \_\_str\_\_ - -```python -def __str__() -> str -``` - -String representation of the ValidationStatus class. - -## ChecksValidator Objects - -```python -class ChecksValidator() -``` - -Class to validate quality rules (checks). - diff --git a/docs/dqx/docs/reference/api/cli.md b/docs/dqx/docs/reference/api/cli.md deleted file mode 100644 index 276eb9130..000000000 --- a/docs/dqx/docs/reference/api/cli.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -sidebar_label: cli -title: databricks.labs.dqx.cli ---- - -#### open\_remote\_config - -```python -@dqx.command -def open_remote_config(w: WorkspaceClient, - *, - ctx: WorkspaceContext | None = None) -``` - -Opens remote configuration in the browser. - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `ctx`: The WorkspaceContext instance to use for accessing the workspace. - -#### open\_dashboards - -```python -@dqx.command -def open_dashboards(w: WorkspaceClient, - *, - ctx: WorkspaceContext | None = None) -``` - -Opens remote dashboard directory in the browser. - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `ctx`: The WorkspaceContext instance to use for accessing the workspace. - -#### installations - -```python -@dqx.command -def installations(w: WorkspaceClient, - *, - product_name: str = "dqx") -> list[dict] -``` - -Show installations by different users on the same workspace. - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `product_name`: The name of the product to search for in the installation folder. - -#### validate\_checks - -```python -@dqx.command -def validate_checks(w: WorkspaceClient, - *, - run_config: str = "default", - validate_custom_check_functions: bool = True, - ctx: WorkspaceContext | None = None) -> list[dict] -``` - -Validate checks stored in the installation directory as a file. - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `run_config`: The name of the run configuration to use. -- `validate_custom_check_functions`: Whether to validate custom check functions (default is True). -- `ctx`: The WorkspaceContext instance to use for accessing the workspace. - -#### profile - -```python -@dqx.command -def profile(w: WorkspaceClient, - *, - run_config: str = "default", - ctx: WorkspaceContext | None = None) -> None -``` - -Profile input data and generate quality rule (checks) candidates. - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `run_config`: The name of the run configuration to use. -- `ctx`: The WorkspaceContext instance to use for accessing the workspace. - -#### workflows - -```python -@dqx.command -def workflows(w: WorkspaceClient, *, ctx: WorkspaceContext | None = None) -``` - -Show deployed workflows and their state - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `ctx`: The WorkspaceContext instance to use for accessing the workspace. - -#### logs - -```python -@dqx.command -def logs(w: WorkspaceClient, - *, - workflow: str | None = None, - ctx: WorkspaceContext | None = None) -``` - -Show logs of the latest job run. - -**Arguments**: - -- `w`: The WorkspaceClient instance to use for accessing the workspace. -- `workflow`: The name of the workflow to show logs for. -- `ctx`: The WorkspaceContext instance to use for accessing the workspace - diff --git a/docs/dqx/docs/reference/api/config.md b/docs/dqx/docs/reference/api/config.md deleted file mode 100644 index da4df925a..000000000 --- a/docs/dqx/docs/reference/api/config.md +++ /dev/null @@ -1,198 +0,0 @@ ---- -sidebar_label: config -title: databricks.labs.dqx.config ---- - -## InputConfig Objects - -```python -@dataclass -class InputConfig() -``` - -Configuration class for input data sources (e.g. tables or files). - -## OutputConfig Objects - -```python -@dataclass -class OutputConfig() -``` - -Configuration class for output data sinks (e.g. tables or files). - -## ProfilerConfig Objects - -```python -@dataclass -class ProfilerConfig() -``` - -Configuration class for profiler. - -#### summary\_stats\_file - -file containing profile summary statistics - -#### sample\_fraction - -fraction of data to sample (30%) - -#### sample\_seed - -seed for sampling - -#### limit - -limit the number of records to profile - -## RunConfig Objects - -```python -@dataclass -class RunConfig() -``` - -Configuration class for the data quality checks - -#### name - -name of the run configuration - -#### quarantine\_config - -quarantined data table - -#### checks\_location - -relative workspace file path or table containing quality rules / checks - -#### warehouse\_id - -warehouse id to use in the dashboard - -## WorkspaceConfig Objects - -```python -@dataclass -class WorkspaceConfig() -``` - -Configuration class for the workspace - -#### get\_run\_config - -```python -def get_run_config(run_config_name: str | None = "default") -> RunConfig -``` - -Get the run configuration for a given run name, or the default configuration if no run name is provided. - -**Arguments**: - -- `run_config_name`: The name of the run configuration to get. - -**Raises**: - -- `ValueError`: If no run configurations are available or if the specified run configuration name is -not found. - -**Returns**: - -The run configuration. - -## BaseChecksStorageConfig Objects - -```python -@dataclass -class BaseChecksStorageConfig(abc.ABC) -``` - -Marker base class for storage configuration. - -## FileChecksStorageConfig Objects - -```python -@dataclass -class FileChecksStorageConfig(BaseChecksStorageConfig) -``` - -Configuration class for storing checks in a file. - -**Arguments**: - -- `location`: The file path where the checks are stored. - -## WorkspaceFileChecksStorageConfig Objects - -```python -@dataclass -class WorkspaceFileChecksStorageConfig(BaseChecksStorageConfig) -``` - -Configuration class for storing checks in a workspace file. - -**Arguments**: - -- `location`: The workspace file path where the checks are stored. - -## TableChecksStorageConfig Objects - -```python -@dataclass -class TableChecksStorageConfig(BaseChecksStorageConfig) -``` - -Configuration class for storing checks in a table. - -**Arguments**: - -- `location`: The table name where the checks are stored. -- `run_config_name`: The name of the run configuration to use for checks (default is 'default'). -- `mode`: The mode for writing checks to a table (e.g., 'append' or 'overwrite'). -The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. - -#### run\_config\_name - -to filter checks by run config - -## VolumeFileChecksStorageConfig Objects - -```python -@dataclass -class VolumeFileChecksStorageConfig(BaseChecksStorageConfig) -``` - -Configuration class for storing checks in a Unity Catalog volume file. - -**Arguments**: - -- `location`: The Unity Catalog volume file path where the checks are stored. - -## InstallationChecksStorageConfig Objects - -```python -@dataclass -class InstallationChecksStorageConfig(WorkspaceFileChecksStorageConfig, - TableChecksStorageConfig, - VolumeFileChecksStorageConfig) -``` - -Configuration class for storing checks in an installation. - -**Arguments**: - -- `location`: The installation path where the checks are stored (e.g., table name, file path). -Not used when using installation method, as it is retrieved from the installation config. -- `run_config_name`: The name of the run configuration to use for checks (default is 'default'). -- `product_name`: The product name for retrieving checks from the installation (default is 'dqx'). -- `assume_user`: Whether to assume the user is the owner of the checks (default is True). - -#### location - -retrieved from the installation config - -#### run\_config\_name - -to retrieve run config - diff --git a/docs/dqx/docs/reference/api/config_loader.md b/docs/dqx/docs/reference/api/config_loader.md deleted file mode 100644 index d40a97d5e..000000000 --- a/docs/dqx/docs/reference/api/config_loader.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -sidebar_label: config_loader -title: databricks.labs.dqx.config_loader ---- - -## RunConfigLoader Objects - -```python -class RunConfigLoader() -``` - -Class to handle loading of run configurations from the installation. - -#### load\_run\_config - -```python -def load_run_config(run_config_name: str | None, - assume_user: bool = True, - product_name: str = "dqx") -> RunConfig -``` - -Load run configuration from the installation. - -**Arguments**: - -- `run_config_name`: name of the run configuration to use -- `assume_user`: if True, assume user installation -- `product_name`: name of the product - -#### get\_installation - -```python -def get_installation(assume_user: bool, product_name: str) -> Installation -``` - -Get the installation for the given product name. - -**Arguments**: - -- `assume_user`: if True, assume user installation -- `product_name`: name of the product - diff --git a/docs/dqx/docs/reference/api/contexts/application.md b/docs/dqx/docs/reference/api/contexts/application.md deleted file mode 100644 index 34894cf5f..000000000 --- a/docs/dqx/docs/reference/api/contexts/application.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -sidebar_label: application -title: databricks.labs.dqx.contexts.application ---- - -## GlobalContext Objects - -```python -class GlobalContext(abc.ABC) -``` - -Returns the parent run ID. - -**Returns**: - -The parent run ID as an integer. - -#### replace - -```python -def replace(**kwargs) -``` - -Replace cached properties. - -**Arguments**: - -- `kwargs`: Key-value pairs of properties to replace. - -**Returns**: - -The updated GlobalContext instance. - -## CliContext Objects - -```python -class CliContext(GlobalContext, abc.ABC) -``` - -Abstract base class for global context, providing common properties and methods for workspace management. - diff --git a/docs/dqx/docs/reference/api/contexts/workflows.md b/docs/dqx/docs/reference/api/contexts/workflows.md deleted file mode 100644 index ac5e76e9a..000000000 --- a/docs/dqx/docs/reference/api/contexts/workflows.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -sidebar_label: workflows -title: databricks.labs.dqx.contexts.workflows ---- - -## RuntimeContext Objects - -```python -class RuntimeContext(GlobalContext) -``` - -#### config - -```python -@cached_property -def config() -> WorkspaceConfig -``` - -Loads and returns the workspace configuration. - -#### run\_config - -```python -@cached_property -def run_config() -> RunConfig -``` - -Loads and returns the run configuration. - -#### connect\_config - -```python -@cached_property -def connect_config() -> core.Config -``` - -Returns the connection configuration. - -**Raises**: - -- `AssertionError`: If the connect configuration is not provided. - -**Returns**: - -The core.Config instance. - -#### workspace\_client - -```python -@cached_property -def workspace_client() -> WorkspaceClient -``` - -Returns the WorkspaceClient instance. - -#### installation - -```python -@cached_property -def installation() -> Installation -``` - -Returns the installation instance for the runtime. - -#### workspace\_id - -```python -@cached_property -def workspace_id() -> int -``` - -Returns the workspace ID. - -#### parent\_run\_id - -```python -@cached_property -def parent_run_id() -> int -``` - -Returns the parent run ID. - -#### profiler - -```python -@cached_property -def profiler() -> ProfilerRunner -``` - -Returns the ProfilerRunner instance. - diff --git a/docs/dqx/docs/reference/api/contexts/workspace.md b/docs/dqx/docs/reference/api/contexts/workspace.md deleted file mode 100644 index e0cc24aee..000000000 --- a/docs/dqx/docs/reference/api/contexts/workspace.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -sidebar_label: workspace -title: databricks.labs.dqx.contexts.workspace ---- - -## WorkspaceContext Objects - -```python -class WorkspaceContext(CliContext) -``` - -WorkspaceContext class that extends CliContext to provide workspace-specific functionality. - -**Arguments**: - -- `ws`: The WorkspaceClient instance to use for accessing the workspace. -- `named_parameters`: Optional dictionary of named parameters. - -#### workspace\_client - -```python -@cached_property -def workspace_client() -> WorkspaceClient -``` - -Returns the WorkspaceClient instance. - diff --git a/docs/dqx/docs/reference/api/engine.md b/docs/dqx/docs/reference/api/engine.md deleted file mode 100644 index 15eb1d961..000000000 --- a/docs/dqx/docs/reference/api/engine.md +++ /dev/null @@ -1,318 +0,0 @@ ---- -sidebar_label: engine -title: databricks.labs.dqx.engine ---- - -## DQEngineCore Objects - -```python -class DQEngineCore(DQEngineCoreBase) -``` - -Data Quality Engine Core class to apply data quality checks to a given dataframe. - -**Arguments**: - -- `workspace_client` _WorkspaceClient_ - WorkspaceClient instance to use for accessing the workspace. -- `extra_params` _ExtraParams_ - Extra parameters for the DQEngine. - -## DQEngine Objects - -```python -class DQEngine(DQEngineBase) -``` - -Data Quality Engine class to apply data quality checks to a given dataframe. - -#### apply\_checks - -```python -def apply_checks(df: DataFrame, - checks: list[DQRule], - ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame -``` - -Applies data quality checks to a given dataframe. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. -- `ref_dfs`: reference dataframes to use in the checks, if applicable - -**Returns**: - -dataframe with errors and warning result columns - -#### apply\_checks\_and\_split - -```python -def apply_checks_and_split( - df: DataFrame, - checks: list[DQRule], - ref_dfs: dict[str, DataFrame] | None = None -) -> tuple[DataFrame, DataFrame] -``` - -Applies data quality checks to a given dataframe and split it into two ("good" and "bad"), - -according to the data quality checks. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. -- `ref_dfs`: reference dataframes to use in the checks, if applicable - -**Returns**: - -two dataframes - "good" which includes warning rows but no result columns, and "data" having -error and warning rows and corresponding result columns - -#### apply\_checks\_by\_metadata\_and\_split - -```python -def apply_checks_by_metadata_and_split( - df: DataFrame, - checks: list[dict], - custom_check_functions: dict[str, Any] | None = None, - ref_dfs: dict[str, DataFrame] | None = None -) -> tuple[DataFrame, DataFrame] -``` - -Wrapper around `apply_checks_and_split` for use in the metadata-driven pipelines. The main difference - -is how the checks are specified - instead of using functions directly, they are described as function name plus -arguments. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - -it will be used as an error/warning message, or `null` if it's evaluated to `false` -* `name` - name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), -and `warn` (data is going into both dataframes) -* `df`0 (optional) - Expression for filtering data quality checks -* `df`1 (optional) - User-defined key-value pairs added to metadata generated by the check. -- `df`2: dictionary with custom check functions (eg. `df`3globals()`df`3 of the calling module). -If not specified, then only built-in functions are used for the checks. -- `df`5: reference dataframes to use in the checks, if applicable - -**Returns**: - -two dataframes - "good" which includes warning rows but no result columns, and "bad" having -error and warning rows and corresponding result columns - -#### apply\_checks\_by\_metadata - -```python -def apply_checks_by_metadata( - df: DataFrame, - checks: list[dict], - custom_check_functions: dict[str, Any] | None = None, - ref_dfs: dict[str, DataFrame] | None = None) -> DataFrame -``` - -Wrapper around `apply_checks` for use in the metadata-driven pipelines. The main difference - -is how the checks are specified - instead of using functions directly, they are described as function name plus -arguments. - -**Arguments**: - -- `df`: dataframe to check -- `checks`: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - -it will be used as an error/warning message, or `null` if it's evaluated to `false` -* `name` - name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), -and `warn` (data is going into both dataframes) -* `df`0 (optional) - Expression for filtering data quality checks -* `df`1 (optional) - User-defined key-value pairs added to metadata generated by the check. -- `df`2: dictionary with custom check functions (eg. `df`3globals()`df`3 of calling module). -- `df`5: reference dataframes to use in the checks, if applicable -If not specified, then only built-in functions are used for the checks. - -**Returns**: - -dataframe with errors and warning result columns - -#### apply\_checks\_and\_save\_in\_table - -```python -def apply_checks_and_save_in_table( - checks: list[DQRule], - input_config: InputConfig, - output_config: OutputConfig, - quarantine_config: OutputConfig | None = None, - ref_dfs: dict[str, DataFrame] | None = None) -> None -``` - -Apply data quality checks to a table or view and write the result to table(s). - -If quarantine_config is provided, the data will be split into good and bad records, -with good records written to the output table and bad records to the quarantine table. -If quarantine_config is not provided, all records (with error/warning columns) -will be written to the output table. - -**Arguments**: - -- `checks`: list of checks to apply to the dataframe. Each check is an instance of DQRule class. -- `input_config`: Input data configuration (e.g. table name or file location, read options) -- `output_config`: Output data configuration (e.g. table name, output mode, write options) -- `quarantine_config`: Optional quarantine data configuration (e.g. table name, output mode, write options) -- `ref_dfs`: Reference dataframes to use in the checks, if applicable - -#### apply\_checks\_by\_metadata\_and\_save\_in\_table - -```python -def apply_checks_by_metadata_and_save_in_table( - checks: list[dict], - input_config: InputConfig, - output_config: OutputConfig, - quarantine_config: OutputConfig | None = None, - custom_check_functions: dict[str, Any] | None = None, - ref_dfs: dict[str, DataFrame] | None = None) -> None -``` - -Apply data quality checks to a table or view and write the result to table(s). - -If quarantine_config is provided, the data will be split into good and bad records, -with good records written to the output table and bad records to the quarantine table. -If quarantine_config is not provided, all records (with error/warning columns) -will be written to the output table. - -**Arguments**: - -- `checks`: List of dictionaries describing checks. Each check is a dictionary consisting of following fields: -* `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - -it will be used as an error/warning message, or `null` if it's evaluated to `false` -* `name` - Name that will be given to a resulting column. Autogenerated if not provided -* `criticality` (optional) -Possible values are `error` (data going only into "bad" dataframe), -and `warn` (data is going into both dataframes) -- `input_config`: Input data configuration (e.g. table name or file location, read options) -- `output_config`: Output data configuration (e.g. table name, output mode, write options) -- `check`0: Optional quarantine data configuration (e.g. table name, output mode, write options) -- `check`1: Dictionary with custom check functions (eg. `check`2globals()`check`2 of calling module). -- `check`4: Reference dataframes to use in the checks, if applicable - -#### validate\_checks - -```python -@staticmethod -def validate_checks( - checks: list[dict], - custom_check_functions: dict[str, Any] | None = None, - validate_custom_check_functions: bool = True -) -> ChecksValidationStatus -``` - -Validate the input dict to ensure they conform to expected structure and types. - -Each check can be a dictionary. The function validates -the presence of required keys, the existence and callability of functions, and the types -of arguments passed to these functions. - -**Arguments**: - -- `checks`: List of checks to apply to the dataframe. Each check should be a dictionary. -- `custom_check_functions`: Optional dictionary with custom check functions. -- `validate_custom_check_functions`: If True, validate custom check functions. - -**Returns**: - -`ValidationStatus`: The validation status. - -#### get\_invalid - -```python -def get_invalid(df: DataFrame) -> DataFrame -``` - -Get records that violate data quality checks (records with warnings and errors). -@param df: input DataFrame. -@return: dataframe with error and warning rows and corresponding result columns. - -#### get\_valid - -```python -def get_valid(df: DataFrame) -> DataFrame -``` - -Get records that don't violate data quality checks (records with warnings but no errors). -@param df: input DataFrame. -@return: dataframe with warning rows but no result columns. - -#### save\_results\_in\_table - -```python -def save_results_in_table(output_df: DataFrame | None = None, - quarantine_df: DataFrame | None = None, - output_config: OutputConfig | None = None, - quarantine_config: OutputConfig | None = None, - run_config_name: str | None = "default", - product_name: str = "dqx", - assume_user: bool = True) -``` - -Save quarantine and output data to the `quarantine_table` and `output_table`. - -**Arguments**: - -- `quarantine_df`: Optional Dataframe containing the quarantine data -- `output_df`: Optional Dataframe containing the output data. If not provided, use run config -- `output_config`: Optional configuration for saving the output data. If not provided, use run config -- `quarantine_config`: Optional configuration for saving the quarantine data. If not provided, use run config -- `run_config_name`: Optional name of the run (config) to use -- `product_name`: name of the product/installation directory -- `assume_user`: if True, assume user installation - -#### load\_checks - -```python -def load_checks(config: BaseChecksStorageConfig) -> list[dict] -``` - -Load checks (dq rules) from the specified source type (file or table). - -**Arguments**: - -- `config`: storage configuration -Allowed configs are: -- `FileChecksStorageConfig`: for loading checks from a file in the local filesystem -- `WorkspaceFileChecksStorageConfig`: for loading checks from a workspace file -- `TableChecksStorageConfig`: for loading checks from a table -- `InstallationChecksStorageConfig`: for loading checks from the installation directory -- `VolumeFileChecksStorageConfig`: for loading checks from a Unity Catalog volume file -- ... - -**Raises**: - -- `ValueError`: if the source type is unknown - -#### save\_checks - -```python -def save_checks(checks: list[dict], config: BaseChecksStorageConfig) -> None -``` - -Save checks (dq rules) to the specified storage type (file or table). - -**Arguments**: - -- `checks`: list of dq rules to save -- `config`: storage configuration -Allowed configs are: -- `FileChecksStorageConfig`: for saving checks in a file in the local filesystem -- `WorkspaceFileChecksStorageConfig`: for saving checks in a workspace file -- `TableChecksStorageConfig`: for saving checks in a table -- `InstallationChecksStorageConfig`: for saving checks in the installation directory -- `VolumeFileChecksStorageConfig`: for saving checks in a Unity Catalog volume file -- ... - -**Raises**: - -- `ValueError`: if the storage type is unknown - diff --git a/docs/dqx/docs/reference/api/executor.md b/docs/dqx/docs/reference/api/executor.md deleted file mode 100644 index a1b6157f2..000000000 --- a/docs/dqx/docs/reference/api/executor.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -sidebar_label: executor -title: databricks.labs.dqx.executor ---- - -## DQCheckResult Objects - -```python -@dataclass(frozen=True) -class DQCheckResult() -``` - -Represents the result of applying a data quality rule to a DataFrame. - -This class holds: -- The Spark Column containing the result of the data quality check - (typically a struct with metadata and the outcome of the rule). -- The DataFrame that was evaluated or generated by the check. - -## DQRuleExecutor Objects - -```python -class DQRuleExecutor(abc.ABC) -``` - -Abstract base class for executing a data quality rule on a DataFrame. - -The executor is responsible for: -- Applying the rule's logic to the provided DataFrame (and optional reference DataFrames). -- Returning the raw result of the check, without applying filters or attaching metadata. - -Executors are specialized for different types of rules: -- DQRowRuleExecutor: Handles row-level rules (rules that produce a condition per row). -- DQDatasetRuleExecutor: Handles dataset-level rules (rules that may aggregate or join DataFrames). - -Subclasses must implement the `apply` method, which performs the rule check and returns -a DQCheckResult containing: - - The raw condition produced by the rule. - - The DataFrame used for reporting (original or transformed). - -#### apply - -```python -@abc.abstractmethod -def apply(df: DataFrame, - spark: SparkSession, - ref_dfs: dict[str, DataFrame] | None = None) -> DQCheckResult -``` - -Apply a rule and return results - -## DQRowRuleExecutor Objects - -```python -class DQRowRuleExecutor(DQRuleExecutor) -``` - -Executor for row-level data quality rules. - -This executor applies a DQRowRule to the provided DataFrame. Row-level rules generate -a condition (Spark Column) that evaluates to a message string if the rule is violated, -or null otherwise, for each row of the DataFrame. - -Responsibilities: -- Obtain the condition column. -- Return a DQCheckResult containing: - - The condition column. - - The input DataFrame. - -#### apply - -```python -def apply(df: DataFrame, - spark: SparkSession, - ref_dfs: dict[str, DataFrame] | None = None) -> DQCheckResult -``` - -Apply the row-level data quality rule to the provided DataFrame. - -The rule produces a condition (Spark Column) that indicates whether each row satisfies -or violates the check. The condition contains a message when the check fails, -or null when it passes. - -**Arguments**: - -- `df`: The input DataFrame to which the rule is applied. -- `spark`: The SparkSession used for executing the rule (unused for row rules). -- `ref_dfs`: Optional dictionary of reference DataFrames (unused for row rules). - -**Returns**: - -DQCheckResult containing: -- condition: Spark Column representing the check condition. -- check_df: The input (main) DataFrame (used for downstream processing). - -## DQDatasetRuleExecutor Objects - -```python -class DQDatasetRuleExecutor(DQRuleExecutor) -``` - -Executor for dataset-level data quality rules. - -This executor applies a DQDatasetRule to the provided DataFrame (and optional reference DataFrames). -Dataset-level rules can produce conditions that involve multiple rows, aggregations, -or comparisons across datasets. - -Responsibilities: -- Obtain condition column and check function closure containing computation logic. -- Return a DQCheckResult containing: - - The condition column. - - The resulting DataFrame produced by the rule. - -#### apply - -```python -def apply(df: DataFrame, - spark: SparkSession, - ref_dfs: dict[str, DataFrame] | None = None) -> DQCheckResult -``` - -Apply the dataset-level data quality rule to the provided DataFrame. - -This method executes the rules logic by executing check function closure. -By convention the first argument to the closure is the DataFrame -and the second argument is optional reference DataFrames. - -The rule produces: -- A condition (Spark Column) that represents the overall dataset check status. -- A DataFrame with the results of the dataset-level evaluation. - -**Arguments**: - -- `df`: The input DataFrame to which the rule is applied. -- `spark`: The SparkSession used for executing the rule. -- `ref_dfs`: Optional dictionary of reference DataFrames for dataset-level checks. - -**Returns**: - -DQCheckResult containing: -- condition: Spark Column representing the check condition. -- check_df: DataFrame produced by the check, containing evaluation results. - -## DQRuleExecutorFactory Objects - -```python -class DQRuleExecutorFactory() -``` - -Factory for creating the appropriate DQRuleExecutor instance for a given DQRule. - -This class encapsulates the logic for selecting the correct executor type -(row-level or dataset-level) based on the rule instance provided. - -Responsibilities: -- Determine the type of rule (DQRowRule or DQDatasetRule). -- Return the corresponding executor (DQRowRuleExecutor or DQDatasetRuleExecutor). -- Raise an error if the rule type is unsupported. - diff --git a/docs/dqx/docs/reference/api/installer/install.md b/docs/dqx/docs/reference/api/installer/install.md deleted file mode 100644 index 36d982bd4..000000000 --- a/docs/dqx/docs/reference/api/installer/install.md +++ /dev/null @@ -1,188 +0,0 @@ ---- -sidebar_label: install -title: databricks.labs.dqx.installer.install ---- - -## WorkspaceInstaller Objects - -```python -class WorkspaceInstaller(WorkspaceContext) -``` - -Installer for DQX workspace. - -**Arguments**: - -- `ws`: The WorkspaceClient instance. -- `environ`: Optional dictionary of environment variables. - -#### upgrades - -```python -@cached_property -def upgrades() -``` - -Returns the Upgrades instance for the product. - -**Returns**: - -An Upgrades instance. - -#### installation - -```python -@cached_property -def installation() -``` - -Returns the current installation for the product. - -**Raises**: - -- `NotFound`: If the installation is not found. - -**Returns**: - -An Installation instance. - -#### run - -```python -def run(default_config: WorkspaceConfig | None = None) -> WorkspaceConfig -``` - -Runs the installation process. - -**Arguments**: - -- `default_config`: Optional default configuration. - -**Raises**: - -- `ManyError`: If multiple errors occur during installation. -- `TimeoutError`: If a timeout occurs during installation. - -**Returns**: - -The final WorkspaceConfig used for the installation. - -#### extract\_major\_minor - -```python -@staticmethod -def extract_major_minor(version_string: str) -``` - -Extracts the major and minor version from a version string. - -**Arguments**: - -- `version_string`: The version string to extract from. - -**Returns**: - -The major.minor version as a string, or None if not found. - -#### configure - -```python -def configure( - default_config: WorkspaceConfig | None = None) -> WorkspaceConfig -``` - -Configures the workspace. - -Notes: -1. Connection errors are not handled within this configure method. - -**Arguments**: - -- `default_config`: Optional default configuration. - -**Raises**: - -- `NotFound`: If the previous installation is not found. -- `RuntimeWarning`: If the existing installation is corrupted. - -**Returns**: - -The final WorkspaceConfig used for the installation. - -#### replace\_config - -```python -def replace_config(**changes: Any) -> WorkspaceConfig | None -``` - -Persist the list of workspaces where UCX is successfully installed in the config - -## WorkspaceInstallation Objects - -```python -class WorkspaceInstallation() -``` - -#### current - -```python -@classmethod -def current(cls, ws: WorkspaceClient) -``` - -Creates a current WorkspaceInstallation instance based on the current workspace client. - -**Arguments**: - -- `ws`: The WorkspaceClient instance. - -**Returns**: - -A WorkspaceInstallation instance. - -#### config - -```python -@property -def config() -``` - -Returns the configuration of the workspace installation. - -**Returns**: - -The WorkspaceConfig instance. - -#### folder - -```python -@property -def folder() -``` - -Returns the installation folder path. - -**Returns**: - -The installation folder path as a string. - -#### run - -```python -def run() -> bool -``` - -Runs the workflow installation. - -**Returns**: - -True if the installation finished successfully, False otherwise. - -#### uninstall - -```python -def uninstall() -``` - -Uninstalls DQX from the workspace, including project folder, dashboards, and jobs. - diff --git a/docs/dqx/docs/reference/api/installer/logs.md b/docs/dqx/docs/reference/api/installer/logs.md deleted file mode 100644 index 85464d538..000000000 --- a/docs/dqx/docs/reference/api/installer/logs.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -sidebar_label: logs -title: databricks.labs.dqx.installer.logs ---- - -## PartialLogRecord Objects - -```python -@dataclass -class PartialLogRecord() -``` - -The information found within a log file record. - -#### peak\_multi\_line\_message - -```python -def peak_multi_line_message( - log: TextIO, pattern: re.Pattern) -> tuple[str, re.Match | None, str] -``` - -A single log record message may span multiple log lines. In this case, the regex on -subsequent lines do not match. - -**Arguments**: - -- `log` _TextIO_ - The log file IO. -- `pattern` _re.Pattern_ - The regex pattern for a log line. - -#### parse\_logs - -```python -def parse_logs(log: TextIO) -> Iterator[PartialLogRecord] -``` - -Parse the logs to retrieve values for PartialLogRecord fields. - -**Arguments**: - -- `log` _TextIO_ - The log file IO. - diff --git a/docs/dqx/docs/reference/api/installer/workflow_task.md b/docs/dqx/docs/reference/api/installer/workflow_task.md deleted file mode 100644 index a7b835d97..000000000 --- a/docs/dqx/docs/reference/api/installer/workflow_task.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -sidebar_label: workflow_task -title: databricks.labs.dqx.installer.workflow_task ---- - -## Task Objects - -```python -@dataclass -class Task() -``` - -#### dependencies - -```python -def dependencies() -``` - -List of dependencies - -## Workflow Objects - -```python -class Workflow() -``` - -#### name - -```python -@property -def name() -``` - -Name of the workflow - -#### spark\_conf - -```python -@property -def spark_conf() -> dict[str, str] | None -``` - -Spark configuration for the workflow - -#### override\_clusters - -```python -@property -def override_clusters() -> dict[str, str] | None -``` - -Override clusters for the workflow - -#### tasks - -```python -def tasks() -> Iterable[Task] -``` - -List of tasks - -#### workflow\_task - -```python -def workflow_task( - fn=None, - *, - depends_on=None, - job_cluster=Task.job_cluster) -> Callable[[Callable], Callable] -``` - -Decorator to register a task in a workflow. - -#### remove\_extra\_indentation - -```python -def remove_extra_indentation(doc: str) -> str -``` - -Remove extra indentation from docstring. - -**Arguments**: - -- `doc`: Docstring - diff --git a/docs/dqx/docs/reference/api/installer/workflows_installer.md b/docs/dqx/docs/reference/api/installer/workflows_installer.md deleted file mode 100644 index 68891f960..000000000 --- a/docs/dqx/docs/reference/api/installer/workflows_installer.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -sidebar_label: workflows_installer -title: databricks.labs.dqx.installer.workflows_installer ---- - -#### TEST\_NIGHTLY\_CI\_RESOURCES\_PURGE\_TIMEOUT - -Buffer for debugging nightly integration test runs - -## MaxedStreamHandler Objects - -```python -class MaxedStreamHandler(logging.StreamHandler) -``` - -#### MAX\_STREAM\_SIZE - -1 Mb minus some buffer - diff --git a/docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md b/docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md deleted file mode 100644 index c7ce4ac8d..000000000 --- a/docs/dqx/docs/reference/api/llm/extract_yaml_checks_examples.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -sidebar_label: extract_yaml_checks_examples -title: databricks.labs.dqx.llm.extract_yaml_checks_examples ---- - -#### extract\_yaml\_checks\_from\_content - -```python -def extract_yaml_checks_from_content(content: str, - source_name: str = "content" - ) -> list[dict[str, Any]] -``` - -Extract all YAML examples from MDX content string. - -**Arguments**: - -- `content`: The MDX content string to extract YAML from -- `source_name`: Name of the source for logging purposes (default: "content") - -**Returns**: - -List of parsed YAML objects from all valid blocks - -#### extract\_yaml\_checks\_from\_mdx - -```python -def extract_yaml_checks_from_mdx(mdx_file_path: str) -> list[dict[str, Any]] -``` - -Extract all YAML examples from a given MDX file. - -**Arguments**: - -- `mdx_file_path`: Path to the MDX file to extract YAML from - -**Raises**: - -- `FileNotFoundError`: If the MDX file does not exist - -**Returns**: - -List of parsed YAML objects from all valid blocks - -#### extract\_yaml\_checks\_examples - -```python -def extract_yaml_checks_examples(output_file_path: Path | None = None) -> bool -``` - -Extract all YAML examples from both quality_rules.mdx and quality_checks.mdx. - -Creates a combined YAML file with all examples from the documentation files -in the LLM resources directory for use in language model processing. - -**Returns**: - -True if extraction was successful, False otherwise - diff --git a/docs/dqx/docs/reference/api/llm/utils.md b/docs/dqx/docs/reference/api/llm/utils.md deleted file mode 100644 index edcf4177a..000000000 --- a/docs/dqx/docs/reference/api/llm/utils.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -sidebar_label: utils -title: databricks.labs.dqx.llm.utils ---- - -#### get\_check\_function\_definition - -```python -def get_check_function_definition( - custom_check_functions: dict[str, Any] | None = None -) -> list[dict[str, str]] -``` - -A utility function to get the definition of all check functions. - -This function is primarily used to generate a prompt for the LLM to generate check functions. - -**Arguments**: - -- `custom_check_functions`: A dictionary of custom check functions. -If provided, the function will use the custom check functions to resolve the check function. - If not provided, the function will use only the built-in check functions. - -Returns: - list[dict]: A list of dictionaries, each containing the definition of a check function. - -#### load\_yaml\_checks\_examples - -```python -def load_yaml_checks_examples() -> str -``` - -Load yaml_checks_examples.yml file from the llm/resources folder. - -**Returns**: - -checks examples as yaml string. - diff --git a/docs/dqx/docs/reference/api/manager.md b/docs/dqx/docs/reference/api/manager.md deleted file mode 100644 index c2191200f..000000000 --- a/docs/dqx/docs/reference/api/manager.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -sidebar_label: manager -title: databricks.labs.dqx.manager ---- - -## DQRuleManager Objects - -```python -@dataclass(frozen=True) -class DQRuleManager() -``` - -Orchestrates the application of a data quality rule to a DataFrame and builds the final check result. - -The manager is responsible for: -- Executing the rule using the appropriate row or dataset executor. -- Applying any filter condition specified in the rule to the check result. -- Combining user-defined and engine-provided metadata into the result. -- Constructing the final structured output (including check name, function, columns, metadata, etc.) -as a DQCheckResult. - -The manager does not implement the logic of individual checks. Instead, it delegates -rule application to the appropriate DQRuleExecutor based on the rule type (row-level or dataset-level). - -**Attributes**: - -- `check` - The DQRule instance that defines the check to apply. -- `df` - The DataFrame on which to apply the check. -- `engine_user_metadata` - Metadata provided by the engine (overridden by check.user_metadata if present). -- `run_time` - The timestamp when the check is executed. -- `ref_dfs` - Optional reference DataFrames for dataset-level checks. - -#### user\_metadata - -```python -@cached_property -def user_metadata() -> dict[str, str] -``` - -Returns user metadata as a dictionary. - -#### filter\_condition - -```python -@cached_property -def filter_condition() -``` - -Returns the filter condition for the check. - -#### process - -```python -def process() -> DQCheckResult -``` - -Process the data quality rule and return results as DQCheckResult containing: -- Column with the check result -- optional DataFrame with the results of the check - diff --git a/docs/dqx/docs/reference/api/profiler/common.md b/docs/dqx/docs/reference/api/profiler/common.md deleted file mode 100644 index 8e4d74159..000000000 --- a/docs/dqx/docs/reference/api/profiler/common.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -sidebar_label: common -title: databricks.labs.dqx.profiler.common ---- - -#### val\_to\_str - -```python -def val_to_str(value: Any, include_sql_quotes: bool = True) -``` - -Converts a value to a string. - -**Arguments**: - -- `value`: The value to convert. Can be a datetime, date, int, float, or other type. -- `include_sql_quotes`: Whether to include quotes around the value. Default is True. - -**Returns**: - -The string representation of the value - -#### val\_maybe\_to\_str - -```python -def val_maybe_to_str(value: Any, include_sql_quotes: bool = True) -``` - -Converts a value to a string if it is a datetime or date. - -**Arguments**: - -- `value`: The value to convert. Can be a datetime, date, or other type. -- `include_sql_quotes`: Whether to include quotes around the value. Default is True. - -**Returns**: - -The string representation of the value. - diff --git a/docs/dqx/docs/reference/api/profiler/dlt_generator.md b/docs/dqx/docs/reference/api/profiler/dlt_generator.md deleted file mode 100644 index ed9bd0c30..000000000 --- a/docs/dqx/docs/reference/api/profiler/dlt_generator.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -sidebar_label: dlt_generator -title: databricks.labs.dqx.profiler.dlt_generator ---- - -## DQDltGenerator Objects - -```python -class DQDltGenerator(DQEngineBase) -``` - -#### generate\_dlt\_rules - -```python -def generate_dlt_rules(rules: list[DQProfile], - action: str | None = None, - language: str = "SQL") -> list[str] | str | dict -``` - -Generates Lakeflow Pipelines (formerly Delta Live Table (DLT)) rules in the specified language. - -**Arguments**: - -- `rules`: A list of data quality profiles to generate rules for. -- `action`: The action to take on rule violation (e.g., "drop", "fail"). -- `language`: The language to generate the rules in ("SQL", "Python" or "Python_Dict"). - -**Raises**: - -- `ValueError`: If the specified language is not supported. - -**Returns**: - -A list of strings representing the Lakeflow Pipelines rules in SQL, a string representing -the Lakeflow Pipelines rules in Python, or dictionary with expressions. - diff --git a/docs/dqx/docs/reference/api/profiler/generator.md b/docs/dqx/docs/reference/api/profiler/generator.md deleted file mode 100644 index 40d06dafa..000000000 --- a/docs/dqx/docs/reference/api/profiler/generator.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -sidebar_label: generator -title: databricks.labs.dqx.profiler.generator ---- - -## DQGenerator Objects - -```python -class DQGenerator(DQEngineBase) -``` - -#### generate\_dq\_rules - -```python -def generate_dq_rules(profiles: list[DQProfile] | None = None, - level: str = "error") -> list[dict] -``` - -Generates a list of data quality rules based on the provided dq profiles. - -**Arguments**: - -- `profiles`: A list of data quality profiles to generate rules for. -- `level`: The criticality level of the rules (default is "error"). - -**Returns**: - -A list of dictionaries representing the data quality rules. - -#### dq\_generate\_is\_in - -```python -@staticmethod -def dq_generate_is_in(column: str, level: str = "error", **params: dict) -``` - -Generates a data quality rule to check if a column's value is in a specified list. - -**Arguments**: - -- `column`: The name of the column to check. -- `level`: The criticality level of the rule (default is "error"). -- `params`: Additional parameters, including the list of values to check against. - -**Returns**: - -A dictionary representing the data quality rule. - -#### dq\_generate\_min\_max - -```python -@staticmethod -def dq_generate_min_max(column: str, level: str = "error", **params: dict) -``` - -Generates a data quality rule to check if a column's value is within a specified range. - -**Arguments**: - -- `column`: The name of the column to check. -- `level`: The criticality level of the rule (default is "error"). -- `params`: Additional parameters, including the minimum and maximum values. - -**Returns**: - -A dictionary representing the data quality rule, or None if no limits are provided. - -#### dq\_generate\_is\_not\_null - -```python -@staticmethod -def dq_generate_is_not_null(column: str, level: str = "error", **params: dict) -``` - -Generates a data quality rule to check if a column's value is not null. - -**Arguments**: - -- `column`: The name of the column to check. -- `level`: The criticality level of the rule (default is "error"). -- `params`: Additional parameters. - -**Returns**: - -A dictionary representing the data quality rule. - -#### dq\_generate\_is\_not\_null\_or\_empty - -```python -@staticmethod -def dq_generate_is_not_null_or_empty(column: str, - level: str = "error", - **params: dict) -``` - -Generates a data quality rule to check if a column's value is not null or empty. - -**Arguments**: - -- `column`: The name of the column to check. -- `level`: The criticality level of the rule (default is "error"). -- `params`: Additional parameters, including whether to trim strings. - -**Returns**: - -A dictionary representing the data quality rule. - diff --git a/docs/dqx/docs/reference/api/profiler/profiler.md b/docs/dqx/docs/reference/api/profiler/profiler.md deleted file mode 100644 index 6fa584e5d..000000000 --- a/docs/dqx/docs/reference/api/profiler/profiler.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -sidebar_label: profiler -title: databricks.labs.dqx.profiler.profiler ---- - -## DQProfiler Objects - -```python -class DQProfiler(DQEngineBase) -``` - -Data Quality Profiler class to profile input data. - -#### get\_columns\_or\_fields - -```python -@staticmethod -def get_columns_or_fields(columns: list[T.StructField]) -> list[T.StructField] -``` - -Extracts all fields from a list of StructField objects, including nested fields from StructType columns. - -**Arguments**: - -- `columns`: A list of StructField objects to process. - -**Returns**: - -A list of StructField objects, including nested fields with prefixed names. - -#### profile - -```python -def profile( - df: DataFrame, - columns: list[str] | None = None, - options: dict[str, Any] | None = None -) -> tuple[dict[str, Any], list[DQProfile]] -``` - -Profiles a DataFrame to generate summary statistics and data quality rules. - -**Arguments**: - -- `df`: The DataFrame to profile. -- `columns`: An optional list of column names to include in the profile. If None, all columns are included. -- `options`: An optional dictionary of options for profiling. - -**Returns**: - -A tuple containing a dictionary of summary statistics and a list of data quality profiles. - -#### profile\_table - -```python -def profile_table( - table: str, - columns: list[str] | None = None, - options: dict[str, Any] | None = None -) -> tuple[dict[str, Any], list[DQProfile]] -``` - -Profiles a table to generate summary statistics and data quality rules. - -**Arguments**: - -- `table`: The fully-qualified table name (`catalog.schema.table`) to be profiled -- `columns`: An optional list of column names to include in the profile. If None, all columns are included. -- `options`: An optional dictionary of options for profiling. - -**Returns**: - -A tuple containing a dictionary of summary statistics and a list of data quality profiles. - -#### profile\_tables - -```python -def profile_tables( - tables: list[str] | None = None, - patterns: list[str] | None = None, - exclude_matched: bool = False, - columns: dict[str, list[str]] | None = None, - options: list[dict[str, Any]] | None = None -) -> dict[str, tuple[dict[str, Any], list[DQProfile]]] -``` - -Profiles Delta tables in Unity Catalog to generate summary statistics and data quality rules. - -**Arguments**: - -- `tables`: An optional list of table names to include. -- `patterns`: An optional list of table names or filesystem-style wildcards (e.g. 'schema.*') to include. -If None, all tables are included. By default, tables matching the pattern are included. -- `exclude_matched`: Specifies whether to include tables matched by the pattern. If True, matched tables -are excluded. If False, matched tables are included. -- `columns`: A dictionary with column names to include in the profile. Keys should be fully-qualified table -names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. -- `options`: A dictionary with options for profiling each table. Keys should be fully-qualified table names -(e.g. `catalog.schema.table`) and values should be options for profiling. - -**Returns**: - -A dictionary mapping table names to tuples containing summary statistics and data quality profiles. - diff --git a/docs/dqx/docs/reference/api/profiler/runner.md b/docs/dqx/docs/reference/api/profiler/runner.md deleted file mode 100644 index caab474b8..000000000 --- a/docs/dqx/docs/reference/api/profiler/runner.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -sidebar_label: runner -title: databricks.labs.dqx.profiler.runner ---- - -## ProfilerRunner Objects - -```python -class ProfilerRunner() -``` - -Runs the DQX profiler on the input data and saves the generated checks and profile summary stats. - -#### run - -```python -def run(input_config: InputConfig, - profiler_config: ProfilerConfig) -> tuple[list[dict], dict[str, Any]] -``` - -Run the DQX profiler on the input data and return the generated checks and profile summary stats. - -**Arguments**: - -- `input_config`: Input data configuration (e.g. table name or file location, read options). -- `profiler_config`: Profiler configuration. - -**Returns**: - -A tuple containing the generated checks and profile summary statistics. - -#### save - -```python -def save(checks: list[dict], summary_stats: dict[str, Any], - checks_location: str | None, - profile_summary_stats_file: str | None) -> None -``` - -Save the generated checks and profile summary statistics to the specified files. - -**Arguments**: - -- `checks`: The generated checks. -- `summary_stats`: The profile summary statistics. -- `checks_location`: The file to save the checks to. -- `profile_summary_stats_file`: The file to save the profile summary statistics to. - diff --git a/docs/dqx/docs/reference/api/profiler/workflow.md b/docs/dqx/docs/reference/api/profiler/workflow.md deleted file mode 100644 index ea2f2a558..000000000 --- a/docs/dqx/docs/reference/api/profiler/workflow.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -sidebar_label: workflow -title: databricks.labs.dqx.profiler.workflow ---- - -## ProfilerWorkflow Objects - -```python -class ProfilerWorkflow(Workflow) -``` - -#### profile - -```python -@workflow_task -def profile(ctx: RuntimeContext) -``` - -Profile the input data and save the generated checks and profile summary stats. - -**Arguments**: - -- `ctx`: Runtime context. - diff --git a/docs/dqx/docs/reference/api/rule.md b/docs/dqx/docs/reference/api/rule.md deleted file mode 100644 index 9aab514dc..000000000 --- a/docs/dqx/docs/reference/api/rule.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -sidebar_label: rule -title: databricks.labs.dqx.rule ---- - -## Criticality Objects - -```python -class Criticality(Enum) -``` - -Enum class to represent criticality of the check. - -## DefaultColumnNames Objects - -```python -class DefaultColumnNames(Enum) -``` - -Enum class to represent columns in the dataframe that will be used for error and warning reporting. - -## ColumnArguments Objects - -```python -class ColumnArguments(Enum) -``` - -Enum class that is used as input parsing for custom column naming. - -## ExtraParams Objects - -```python -@dataclass(frozen=True) -class ExtraParams() -``` - -Class to represent extra parameters for DQEngine. - -## SingleColumnMixin Objects - -```python -class SingleColumnMixin() -``` - -Mixin to handle column-related functionalities. - -## MultipleColumnsMixin Objects - -```python -class MultipleColumnsMixin() -``` - -Mixin to handle columns-related functionalities. - -## DQRule Objects - -```python -@dataclass(frozen=True) -class DQRule(abc.ABC, DQRuleTypeMixin, SingleColumnMixin, - MultipleColumnsMixin) -``` - -Represents a data quality rule that applies a quality check function to column(s) or -column expression(s). This class includes the following attributes: -* `check_func` - The function used to perform the quality check. -* `name` (optional) - A custom name for the check; autogenerated if not provided. -* `criticality` (optional) - Defines the severity level of the check: - - `error`: Critical issues. - - `warn`: Potential issues. -* `column` (optional) - A single column to which the check function is applied. -* `columns` (optional) - A list of columns to which the check function is applied. -* `filter` (optional) - A filter expression to apply the check only to rows meeting specific conditions. -* `check_func_args` (optional) - Positional arguments for the check function (excluding `column`). -* `name`0 (optional) - Keyword arguments for the check function (excluding `column`). -* `name`2 (optional) - User-defined key-value pairs added to metadata generated by the check. - -#### columns - -some checks require list of columns instead of column - -#### get\_check\_condition - -```python -@abc.abstractmethod -def get_check_condition() -> Column -``` - -Compute the check condition for the rule. - -**Returns**: - -The Spark Column representing the check condition. - -#### columns\_as\_string\_expr - -```python -@ft.cached_property -def columns_as_string_expr() -> Column -``` - -Spark Column expression representing the column(s) as a string (not normalized). - -**Returns**: - -A Spark Column object representing the column(s) as a string (not normalized). - -#### prepare\_check\_func\_args\_and\_kwargs - -```python -def prepare_check_func_args_and_kwargs() -> tuple[list, dict] -``` - -Prepares positional arguments and keyword arguments for the check function. -Includes only arguments supported by the check function and skips empty values. - -#### to\_dict - -```python -def to_dict() -> dict -``` - -Converts a DQRule instance into a structured dictionary. - -## DQRowRule Objects - -```python -@dataclass(frozen=True) -class DQRowRule(DQRule) -``` - -Represents a row-level data quality rule that applies a quality check function to a column or column expression. -Works with check functions that take a single column or no column as input. - -#### get\_check\_condition - -```python -def get_check_condition() -> Column -``` - -Compute the check condition for this rule. - -**Returns**: - -The Spark Column representing the check condition. - -## DQDatasetRule Objects - -```python -@dataclass(frozen=True) -class DQDatasetRule(DQRule) -``` - -Represents a dataset-level data quality rule that applies a quality check function to a column or -column expression or list of columns depending on the check function. -Either column or columns can be provided but not both. The rules are applied to the entire dataset or group of rows -rather than individual rows. Failed checks are appended to the result columns in the same way as row-level rules. - -#### get\_check\_condition - -```python -def get_check_condition() -> Column -``` - -Compute the check condition for this rule. - -**Returns**: - -The Spark Column representing the check condition. - -## DQForEachColRule Objects - -```python -@dataclass(frozen=True) -class DQForEachColRule(DQRuleTypeMixin) -``` - -Represents a data quality rule that applies to a quality check function -repeatedly on each specified column of the provided list of columns. -This class includes the following attributes: -* `columns` - A list of column names or expressions to which the check function should be applied. -* `check_func` - The function used to perform the quality check. -* `name` (optional) - A custom name for the check; autogenerated if not provided. -* `criticality` - The severity level of the check: - - `'warn'` for potential issues. - - `'error'` for critical issues. -* `filter` (optional) - A filter expression to apply the check only to rows meeting specific conditions. -* `check_func_args` (optional) - Positional arguments for the check function (excluding column names). -* `check_func_kwargs` (optional) - Keyword arguments for the check function (excluding column names). -* `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - -#### get\_rules - -```python -def get_rules() -> list[DQRule] -``` - -Build a list of rules for a set of columns. - -**Returns**: - -list of dq rules - diff --git a/docs/dqx/docs/reference/api/runtime.md b/docs/dqx/docs/reference/api/runtime.md deleted file mode 100644 index f4bd44d20..000000000 --- a/docs/dqx/docs/reference/api/runtime.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -sidebar_label: runtime -title: databricks.labs.dqx.runtime ---- - -## Workflows Objects - -```python -class Workflows() -``` - -#### all - -```python -@classmethod -def all(cls, config: WorkspaceConfig) -``` - -Return all workflows. - -#### tasks - -```python -def tasks() -> list[Task] -``` - -Return all tasks. - -#### trigger - -```python -def trigger(*argv) -``` - -Trigger a workflow. - -#### main - -```python -def main(*argv) -``` - -Main entry point. - diff --git a/docs/dqx/docs/reference/api/sidebar.json b/docs/dqx/docs/reference/api/sidebar.json deleted file mode 100644 index bf6cc09a0..000000000 --- a/docs/dqx/docs/reference/api/sidebar.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "items": [ - { - "items": [ - { - "items": [ - { - "items": [ - { - "items": [ - "reference/api/databricks/labs/dqx/contexts/application", - "reference/api/databricks/labs/dqx/contexts/workflows", - "reference/api/databricks/labs/dqx/contexts/workspace" - ], - "label": "databricks.labs.dqx.contexts", - "type": "category" - }, - { - "items": [ - "reference/api/databricks/labs/dqx/installer/install", - "reference/api/databricks/labs/dqx/installer/logs", - "reference/api/databricks/labs/dqx/installer/workflow_task", - "reference/api/databricks/labs/dqx/installer/workflows_installer" - ], - "label": "databricks.labs.dqx.installer", - "type": "category" - }, - { - "items": [ - "reference/api/databricks/labs/dqx/llm/extract_yaml_checks_examples", - "reference/api/databricks/labs/dqx/llm/utils" - ], - "label": "databricks.labs.dqx.llm", - "type": "category" - }, - { - "items": [ - "reference/api/databricks/labs/dqx/profiler/common", - "reference/api/databricks/labs/dqx/profiler/dlt_generator", - "reference/api/databricks/labs/dqx/profiler/generator", - "reference/api/databricks/labs/dqx/profiler/profiler", - "reference/api/databricks/labs/dqx/profiler/runner", - "reference/api/databricks/labs/dqx/profiler/workflow" - ], - "label": "databricks.labs.dqx.profiler", - "type": "category" - }, - "reference/api/databricks/labs/dqx/__main__", - "reference/api/databricks/labs/dqx/base", - "reference/api/databricks/labs/dqx/check_funcs", - "reference/api/databricks/labs/dqx/checks_resolver", - "reference/api/databricks/labs/dqx/checks_serializer", - "reference/api/databricks/labs/dqx/checks_storage", - "reference/api/databricks/labs/dqx/checks_validator", - "reference/api/databricks/labs/dqx/cli", - "reference/api/databricks/labs/dqx/config", - "reference/api/databricks/labs/dqx/config_loader", - "reference/api/databricks/labs/dqx/engine", - "reference/api/databricks/labs/dqx/executor", - "reference/api/databricks/labs/dqx/manager", - "reference/api/databricks/labs/dqx/rule", - "reference/api/databricks/labs/dqx/runtime", - "reference/api/databricks/labs/dqx/utils" - ], - "label": "databricks.labs.dqx", - "type": "category" - } - ], - "label": "databricks.labs", - "type": "category" - } - ], - "label": "databricks", - "type": "category" - } - ], - "label": "Reference", - "type": "category" -} \ No newline at end of file diff --git a/docs/dqx/docs/reference/api/utils.md b/docs/dqx/docs/reference/api/utils.md deleted file mode 100644 index 89f6846b0..000000000 --- a/docs/dqx/docs/reference/api/utils.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -sidebar_label: utils -title: databricks.labs.dqx.utils ---- - -#### get\_column\_name\_or\_alias - -```python -def get_column_name_or_alias( - column: str | Column | ConnectColumn, - normalize: bool = False, - allow_simple_expressions_only: bool = False) -> str -``` - -Extracts the column alias or name from a PySpark Column or ConnectColumn expression. - -PySpark does not provide direct access to the alias of an unbound column, so this function -parses the alias from the column's string representation. - -- Supports columns with one or multiple aliases. -- Ensures the extracted expression is truncated to 255 characters. -- Provides an optional normalization step for consistent naming. - -**Arguments**: - -- `column`: Column, ConnectColumn or string representing a column. -- `normalize`: If True, normalizes the column name (removes special characters, converts to lowercase). -- `allow_simple_expressions_only`: If True, raises an error if the column expression is not a simple expression. -Complex PySpark expressions (e.g., conditionals, arithmetic, or nested transformations), cannot be fully -reconstructed correctly when converting to string (e.g. F.col("a") + F.lit(1)). -However, in certain situations this is acceptable, e.g. when using the output for reporting purposes. - -**Raises**: - -- `ValueError`: If the column expression is invalid. -- `TypeError`: If the column type is unsupported. - -**Returns**: - -The extracted column alias or name. - -#### get\_columns\_as\_strings - -```python -def get_columns_as_strings( - columns: list[str | Column], - allow_simple_expressions_only: bool = True) -> list[str] -``` - -Extracts column names from a list of PySpark Column or ConnectColumn expressions. - -This function processes each column, ensuring that only valid column names are returned. - -**Arguments**: - -- `columns`: List of columns, ConnectColumns or strings representing columns. -- `allow_simple_expressions_only`: If True, raises an error if the column expression is not a simple expression. - -**Returns**: - -List of column names as strings. - -#### is\_simple\_column\_expression - -```python -def is_simple_column_expression(col_name: str) -> bool -``` - -Returns True if the column name does not contain any disallowed characters: - -space, comma, semicolon, curly braces, parentheses, newline, tab, or equals sign. - -**Arguments**: - -- `col_name`: Column name to validate. - -**Returns**: - -True if the column name is valid, False otherwise. - -#### normalize\_bound\_args - -```python -def normalize_bound_args(val: Any) -> Any -``` - -Normalize a value or collection of values for consistent processing. - -Handles primitives, dates, and column-like objects. Lists, tuples, and sets are -recursively normalized with type preserved. - -**Arguments**: - -- `val`: Value or collection of values to normalize. - -**Raises**: - -- `ValueError`: If a column resolves to an invalid name. -- `TypeError`: If a column type is unsupported. - -**Returns**: - -Normalized value or collection. - -#### normalize\_col\_str - -```python -def normalize_col_str(col_str: str) -> str -``` - -Normalizes string to be compatible with metastore column names by applying the following transformations: - -* remove special characters -* convert to lowercase -* limit the length to 255 characters to be compatible with metastore column names - -**Arguments**: - -- `col_str`: Column or string representing a column. - -**Returns**: - -Normalized column name. - -#### read\_input\_data - -```python -def read_input_data(spark: SparkSession, - input_config: InputConfig) -> DataFrame -``` - -Reads input data from the specified location and format. - -**Arguments**: - -- `spark`: SparkSession -- `input_config`: InputConfig with source location/table name, format, and options - -**Returns**: - -DataFrame with values read from the input data - -#### save\_dataframe\_as\_table - -```python -def save_dataframe_as_table(df: DataFrame, output_config: OutputConfig) -``` - -Helper method to save a DataFrame to a Delta table. - -**Arguments**: - -- `df`: The DataFrame to save -- `output_config`: Output table name, write mode, and options - -#### safe\_json\_load - -```python -def safe_json_load(value: str) -``` - -Safely load a JSON string, returning the original value if it fails to parse. - -This allows to specify string value without a need to escape the quotes. - -**Arguments**: - -- `value`: The value to parse as JSON. - From 467c9e8ef0bf742cf308b729aa4ec6c9011b33a6 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 13 Aug 2025 13:00:51 +0200 Subject: [PATCH 04/37] remove uv --- uv.lock | 303 -------------------------------------------------------- 1 file changed, 303 deletions(-) delete mode 100644 uv.lock diff --git a/uv.lock b/uv.lock deleted file mode 100644 index 08b20a3c1..000000000 --- a/uv.lock +++ /dev/null @@ -1,303 +0,0 @@ -version = 1 -revision = 2 -requires-python = ">=3.10" - -[[package]] -name = "cachetools" -version = "5.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, -] - -[[package]] -name = "certifi" -version = "2025.8.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, - { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, - { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, - { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, - { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, - { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, - { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, - { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, - { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, - { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, - { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, - { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, - { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, - { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, - { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, - { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, - { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, - { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, - { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, - { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, - { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, - { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, - { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, - { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, - { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, - { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, - { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, - { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, - { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, - { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, - { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, - { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, - { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, - { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, - { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, - { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, - { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, - { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, - { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, - { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, - { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, - { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, - { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, -] - -[[package]] -name = "databricks-labs-blueprint" -version = "0.9.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "databricks-sdk" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/d0/9d818b50dc4fa86a9b5fdc0d74b96eacbd06410d7bd10b9e5f75dc416e35/databricks_labs_blueprint-0.9.3.tar.gz", hash = "sha256:a40628c0d58b6a9c8cf776b3ffa31237a8eec2f4d7a21142464cd2c285a2cd61", size = 76667, upload-time = "2024-11-14T13:33:23.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/f7/4e77bdcd83fb5e53d79526f4532dd05af53e5dcbb2c2854ae536baecf133/databricks_labs_blueprint-0.9.3-py3-none-any.whl", hash = "sha256:0e640953deef5e41bc324d1035ce8c5d549023178ce50708700ab34c438451f3", size = 61549, upload-time = "2024-11-14T13:33:21.818Z" }, -] - -[package.optional-dependencies] -yaml = [ - { name = "pyyaml" }, -] - -[[package]] -name = "databricks-labs-dqx" -source = { editable = "." } -dependencies = [ - { name = "databricks-labs-blueprint" }, - { name = "databricks-labs-lsql" }, - { name = "databricks-sdk" }, -] - -[package.optional-dependencies] -cli = [ - { name = "pyspark" }, -] - -[package.metadata] -requires-dist = [ - { name = "databricks-labs-blueprint", specifier = ">=0.9.1,<0.10" }, - { name = "databricks-labs-lsql", specifier = ">=0.5,<0.15" }, - { name = "databricks-sdk", specifier = "~=0.57" }, - { name = "pyspark", marker = "extra == 'cli'", specifier = "~=3.5.0" }, -] -provides-extras = ["cli"] - -[[package]] -name = "databricks-labs-lsql" -version = "0.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "databricks-labs-blueprint", extra = ["yaml"] }, - { name = "databricks-sdk" }, - { name = "sqlglot" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/05/0b12ab6efc3dd737ae06a509f513241a1d893f12149e8ebdefcdd23d2099/databricks_labs_lsql-0.14.2.tar.gz", hash = "sha256:4c4dddc12b27bab0587830419ef744565128a97fceec10f02b451b1ee007bb3e", size = 52776, upload-time = "2025-02-27T16:34:05.897Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/67/bc65eed315c45bfeb2df1fb3204894c45f16bef34c4cbbebbb0c528f9d71/databricks_labs_lsql-0.14.2-py3-none-any.whl", hash = "sha256:2f97c2bf633aa1569db77ffa234f9e456438687975d78bba568cdbb3e6fe86cb", size = 47965, upload-time = "2025-02-27T16:34:03.569Z" }, -] - -[[package]] -name = "databricks-sdk" -version = "0.62.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/16/1ad8be298b3135a3be0875afb97ce07a4fecd32489f8351bc4419e5e083b/databricks_sdk-0.62.0.tar.gz", hash = "sha256:12f8da735f74cba5265dcb0620c30941628f63787e6ea7e57cc5ed22d9e1b987", size = 725602, upload-time = "2025-08-06T11:31:06.415Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/cd/5fe949ec0b160130066d0ba82b4f499dc494a7405723836ebc37511f6bfc/databricks_sdk-0.62.0-py3-none-any.whl", hash = "sha256:79d4abe60306239985a5b718583923e13316c5417204017b0dadf74bccfcc6e1", size = 681789, upload-time = "2025-08-06T11:31:04.492Z" }, -] - -[[package]] -name = "google-auth" -version = "2.40.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "pyasn1-modules" }, - { name = "rsa" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/9b/e92ef23b84fa10a64ce4831390b7a4c2e53c0132568d99d4ae61d04c8855/google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77", size = 281029, upload-time = "2025-06-04T18:04:57.577Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" }, -] - -[[package]] -name = "idna" -version = "3.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, -] - -[[package]] -name = "py4j" -version = "0.10.9.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b", size = 200481, upload-time = "2022-08-12T22:49:07.05Z" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322, upload-time = "2024-09-10T22:41:42.55Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - -[[package]] -name = "pyspark" -version = "3.5.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py4j" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2e/62/36e50d38e5fe158e97cddec983b44f9417b1e205b02320e3c463b5f802fa/pyspark-3.5.6.tar.gz", hash = "sha256:f8b1c4360e41ab398c64904fae08740503bcb6bd389457d659fa6d9f2952cc48", size = 317359167, upload-time = "2025-05-27T08:24:20.82Z" } - -[[package]] -name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, - { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, - { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, - { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, - { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, - { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, - { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, - { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, - { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, - { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, -] - -[[package]] -name = "requests" -version = "2.32.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, -] - -[[package]] -name = "rsa" -version = "4.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, -] - -[[package]] -name = "sqlglot" -version = "27.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/aa/0c93a9974c0dc0726aa1a3ebc956f88bc5f6694d811b23d9a31085bb812b/sqlglot-27.6.0.tar.gz", hash = "sha256:413bc6c9978710bd97110e9a4363c517dfb92954d0a9fd2922e87c5707395deb", size = 5409152, upload-time = "2025-08-01T16:03:12.516Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/58/8b267b6b3c27ccb16367b49db2ac8c6032939f4fbffdc66dd0831833a7b9/sqlglot-27.6.0-py3-none-any.whl", hash = "sha256:828e6eea4e608f221c14f9e5dbe2f9f73142f11ba289d84f58c56ca44ee49d23", size = 496769, upload-time = "2025-08-01T16:03:09.914Z" }, -] - -[[package]] -name = "urllib3" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, -] From 8a1fbebdaa16847ede2a55f07c7a19a4885a7cf8 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 13 Aug 2025 13:01:40 +0200 Subject: [PATCH 05/37] remove unused scripts --- docs/dqx/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/dqx/package.json b/docs/dqx/package.json index e8e2e09af..eda7ffc5e 100644 --- a/docs/dqx/package.json +++ b/docs/dqx/package.json @@ -12,8 +12,7 @@ "serve": "docusaurus serve", "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", - "typecheck": "tsc", - "generate-api-docs": "tsx src/scripts/generate-api-docs.ts" + "typecheck": "tsc" }, "dependencies": { "@docusaurus/core": "^3.8.1", From 16fb4c06d4a939ab652e8fcbb135a7390f3eb9fb Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 14:35:39 +0200 Subject: [PATCH 06/37] fix headers leveling --- pyproject.toml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44e96161e..a281fa296 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,10 +115,33 @@ extract_yaml_checks_examples = "python src/databricks/labs/dqx/llm/extract_yaml_ dependencies = ["pydoc-markdown>=4.8.2"] [tool.pydoc-markdown] -loaders = [{ type = "python", search_path = ["./src"], packages = ["databricks.labs.dqx"] }] -processors = [{ type = "smart" }, { type = "crossref" }, {type = "filter", skip_empty_modules = true}] -renderer = { type = "docusaurus", docs_base_path = "docs/dqx/docs", relative_output_path = "reference/api", relative_sidebar_path = "sidebar.json" } -hooks = {post-render = ["cp -r docs/dqx/docs/reference/api/databricks/labs/dqx/* docs/dqx/docs/reference/api", "rm -rf docs/dqx/docs/reference/api/databricks"]} +loaders = [ + { type = "python", search_path = [ + "./src", + ], packages = [ + "databricks.labs.dqx", + ] }, +] +processors = [ + { type = "smart" }, + { type = "crossref" }, + { type = "filter", skip_empty_modules = true }, +] +hooks = { post-render = [ + "cp -r docs/dqx/docs/reference/api/databricks/labs/dqx/* docs/dqx/docs/reference/api", + "rm -rf docs/dqx/docs/reference/api/databricks", +] } + +[tool.pydoc-markdown.renderer] + +type = "docusaurus" +docs_base_path = "docs/dqx/docs" +relative_output_path = "reference/api" +relative_sidebar_path = "sidebar.json" + +# by default method and function are on level 4 which is too deep for docusaurus +# we want to move them to level 3 +markdown = { header_level_by_type = { Method = 3, Function = 3, Class = 2, Module = 1 } } [tool.isort] profile = "black" From bc5d84de25dbb545f79d55d594fbf80a98b7104c Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 15:59:38 +0200 Subject: [PATCH 07/37] address comments --- docs/dqx/sidebars.ts | 5 +++-- tests/e2e/test_run_demos.py | 15 --------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/docs/dqx/sidebars.ts b/docs/dqx/sidebars.ts index 5956b79c4..8c61f1dc4 100644 --- a/docs/dqx/sidebars.ts +++ b/docs/dqx/sidebars.ts @@ -1,4 +1,4 @@ -import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; +import type { SidebarsConfig } from '@docusaurus/plugin-content-docs'; // This runs in Node.js - Don't use client-side code here (browser APIs, JSX...) @@ -14,7 +14,8 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; */ const sidebars: SidebarsConfig = { // By default, Docusaurus generates a sidebar from the docs folder structure - tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], + tutorialSidebar: [ + { type: 'autogenerated', dirName: '.' }], }; export default sidebars; diff --git a/tests/e2e/test_run_demos.py b/tests/e2e/test_run_demos.py index b2d4a2968..cca88efc0 100644 --- a/tests/e2e/test_run_demos.py +++ b/tests/e2e/test_run_demos.py @@ -170,21 +170,6 @@ def test_run_dqx_demo_tool(installation_ctx, make_schema, make_notebook, make_jo ) logging.info(f"Job run {run.run_id} completed successfully for dqx_demo_tool") - -def validate_demo_run_status(run: Run, client: WorkspaceClient) -> None: - """ - Validates that a demo run completed successfully. - :param run: `Run` object returned from a `WorkspaceClient.jobs.submit(...)` command - :param client: `WorkspaceClient` object for getting task output - """ - task = run.tasks[0] - termination_details = run.status.termination_details - - assert ( - termination_details.type == TerminationTypeType.SUCCESS - ), f"Run of '{task.task_key}' failed with message: {client.jobs.get_run_output(task.run_id).error}" - - def test_run_dqx_streaming_demo_native(make_notebook, make_schema, make_job, tmp_path, library_ref): ws = WorkspaceClient() path = Path(__file__).parent.parent.parent / "demos" / "dqx_streaming_demo_native.py" From a88c597bc9ca11d5c51a82c969b2f47f6fe6506d Mon Sep 17 00:00:00 2001 From: Ivan Trusov Date: Thu, 14 Aug 2025 16:00:23 +0200 Subject: [PATCH 08/37] Update .gitignore Co-authored-by: Marcin Wojtyczka --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dc2b4d2d4..0e39c25c4 100644 --- a/.gitignore +++ b/.gitignore @@ -160,7 +160,7 @@ dev/cleanup.py .databricks-login.json .local-dev -# Cursor IDE specific files (not all developers use Cursor) +# Cursor IDE specific files .cursorrules .cursor/ From e1d7404263813e9478b33045664cdae0796e7aaf Mon Sep 17 00:00:00 2001 From: Ivan Trusov Date: Thu, 14 Aug 2025 16:01:01 +0200 Subject: [PATCH 09/37] Update src/databricks/labs/dqx/check_funcs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/databricks/labs/dqx/check_funcs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index a1e46a15c..c51a7f9c2 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -1119,6 +1119,18 @@ def compare_datasets( The log containing detailed differences is written to the message field of the check result as JSON string. Example: `{"row_missing":false,"row_extra":true,"changed":{"val":{"df":"val1"}}}` + Example: + ```json + { + "row_missing": false, + "row_extra": true, + "changed": { + "val": { + "df": "val1" + } + } + } + ``` :param columns: List of columns to use for row matching with the reference DataFrame (can be a list of string column names or column expressions). Only simple column expressions are supported, e.g. F.col("col_name") From 0b375c5d9309024e1e1aee1d1e3f4fd82258d260 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 16:05:58 +0200 Subject: [PATCH 10/37] minimize pyproject changes --- pyproject.toml | 109 ++++++++----------------- src/databricks/labs/dqx/check_funcs.py | 1 - 2 files changed, 36 insertions(+), 74 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a281fa296..81f52fcc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,10 +27,9 @@ classifiers = [ "Topic :: Software Development :: Libraries", "Topic :: Utilities", ] -dependencies = [ - "databricks-labs-blueprint>=0.9.1,<0.10", - "databricks-sdk~=0.57", - "databricks-labs-lsql>=0.5,<0.15", +dependencies = ["databricks-labs-blueprint>=0.9.1,<0.10", + "databricks-sdk~=0.57", + "databricks-labs-lsql>=0.5,<0.15", ] [project.optional-dependencies] @@ -86,28 +85,24 @@ dependencies = [ "pyspark~=3.5.0" ] -python = "3.10" +python="3.10" # store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better path = ".venv" [tool.hatch.envs.default.scripts] -test = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20" -coverage = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20" +test = "pytest tests/unit/ -n 10 --cov --cov-report=xml:coverage-unit.xml --timeout 30 --durations 20" +coverage = "pytest tests/ -n 10 --cov --cov-report=html --timeout 600 --durations 20" integration = "pytest tests/integration/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20" -e2e = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20" -fmt = [ - "black .", - "ruff check . --fix", - "mypy .", - "pylint --output-format=colorized -j 0 src tests", -] -verify = [ - "black --check .", - "ruff check .", - "mypy .", - "pylint --output-format=colorized -j 0 src tests", -] +e2e = "pytest tests/e2e/ -n 10 --cov --cov-report=xml --timeout 600 --durations 20" +fmt = ["black .", + "ruff check . --fix", + "mypy .", + "pylint --output-format=colorized -j 0 src tests"] +verify = ["black --check .", + "ruff check .", + "mypy .", + "pylint --output-format=colorized -j 0 src tests"] update_github_urls = "python docs/dqx/update_github_urls.py" extract_yaml_checks_examples = "python src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py" @@ -175,7 +170,11 @@ parallel = true [tool.coverage.report] omit = ["*/working-copy/*", 'src/databricks/labs/dqx/__main__.py'] -exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] [tool.pylint.main] # PyLint configuration is adapted from Google Python Style Guide with modifications. @@ -354,20 +353,15 @@ function-rgx = "[A-Za-z_][A-Za-z0-9_]{2,}$" # Good variable names which should always be accepted, separated by a comma. good-names = [ - "f", # use for file handles - "i", - "j", - "k", # use for loops - "df", # use for pyspark.sql.DataFrame - "ex", - "e", # use for exceptions - "fn", - "cb", # use for callbacks - "_", # use for ignores - "a", # use for databricks.sdk.AccountClient - "w", - "ws", # use for databricks.sdk.WorkspaceClient - "me", # use for current user + "f", # use for file handles + "i", "j", "k", # use for loops + "df", # use for pyspark.sql.DataFrame + "ex", "e", # use for exceptions + "fn", "cb", # use for callbacks + "_", # use for ignores + "a", # use for databricks.sdk.AccountClient + "w", "ws", # use for databricks.sdk.WorkspaceClient + "me" # use for current user ] # Good variable names regexes, separated by a comma. If names match any regex, @@ -569,13 +563,7 @@ logging-modules = ["logging"] [tool.pylint."messages control"] # Only show warnings with the listed confidence levels. Leave empty to show all. # Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence = [ - "HIGH", - "CONTROL_FLOW", - "INFERENCE", - "INFERENCE_FAILURE", - "UNDEFINED", -] +confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] # Disable the message, report, category or checker with the given id(s). You can # either give multiple identifiers separated by comma (,) or put this option @@ -607,7 +595,7 @@ disable = [ "unnecessary-default-type-args", "mock-no-usage", "broad-exception-caught", - "rewrite-as-for-loop", + "rewrite-as-for-loop" ] # Enable the message, report, category or checker with the given id(s). You can @@ -619,16 +607,7 @@ enable = ["useless-suppression", "use-symbolic-message-instead"] [tool.pylint.method_args] # List of qualified names (i.e., library.method) which require a timeout # parameter e.g. 'requests.api.get,requests.api.post' -timeout-methods = [ - "requests.api.delete", - "requests.api.get", - "requests.api.head", - "requests.api.options", - "requests.api.patch", - "requests.api.post", - "requests.api.put", - "requests.api.request", -] +timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"] [tool.pylint.miscellaneous] # List of note tags to take in consideration, separated by a comma. @@ -754,22 +733,12 @@ ignore-none = true ignore-on-opaque-inference = true # List of symbolic message names to ignore for Mixin members. -ignored-checks-for-mixins = [ - "no-member", - "not-async-context-manager", - "not-context-manager", - "attribute-defined-outside-init", -] +ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes = [ - "SQLObject", - "optparse.Values", - "thread._local", - "_thread._local", -] +ignored-classes = ["SQLObject", "optparse.Values", "thread._local", "_thread._local"] # Show a hint with possible names when a member name was not found. The aspect of # finding the hint is based on edit distance. @@ -816,13 +785,7 @@ ignored-argument-names = "_.*|^ignored_|^unused_" # List of qualified module names which can have objects that can redefine # builtins. -redefining-builtins-modules = [ - "six.moves", - "past.builtins", - "future.builtins", - "builtins", - "io", -] +redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] [tool.hatch.metadata] -allow-direct-references = true +allow-direct-references = true \ No newline at end of file diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index c51a7f9c2..df368c1e7 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -1117,7 +1117,6 @@ def compare_datasets( The comparison does not support Map types (any column comparison on map type is skipped automatically). The log containing detailed differences is written to the message field of the check result as JSON string. - Example: `{"row_missing":false,"row_extra":true,"changed":{"val":{"df":"val1"}}}` Example: ```json From fa81c905b58b9fae302a5fb3640ccf7388e73083 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 16:09:52 +0200 Subject: [PATCH 11/37] update gh pipeline --- .github/workflows/docs-release.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 41fdd567a..38f9d230e 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -13,13 +13,23 @@ jobs: group: databrickslabs-protected-runner-group labels: linux-ubuntu-latest environment: release - defaults: - run: - working-directory: docs/dqx + steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Install Python + uses: actions/setup-python@v5 + with: + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' + python-version: 3.10 + + - name: Install Hatch + run: | + pip install hatch==1.9.4 + - uses: actions/setup-node@v4 with: node-version: 20 @@ -27,9 +37,14 @@ jobs: cache-dependency-path: docs/dqx/yarn.lock # need to put the lockfile path explicitly - name: Install dependencies - run: yarn install --frozen-lockfile + run: yarn --cwd docs/dqx install --frozen-lockfile + + + - name: Generate API docs + run: hatch pydoc-markdown + - name: Build website - run: yarn build + run: yarn --cwd docs/dqx build - name: Upload Build Artifact uses: actions/upload-pages-artifact@v3 From b53348e62a0ccaec62ba25c760ca10063325b37a Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 16:15:03 +0200 Subject: [PATCH 12/37] add api doc authoring --- docs/dqx/docs/dev/contributing.mdx | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 67221940d..e4670084b 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -297,4 +297,40 @@ Example: ```python def viz_type(self) -> str: return self.viz.get("type", "UNKNOWN") -``` \ No newline at end of file +``` + +## Writing API docs + +API docs are generated using [pydoc-markdown](https://github.com/pypa/pydoc-markdown). + +To generate API docs, run the following command: +```shell +hatch run docs:pydoc-markdown +``` + +The API docs are generated in the `docs/dqx/docs/reference/api` directory. + +To write API docs, use the following format: + +```python + +def method(self, arg1: str, arg2: int) -> str: + """ + Short method description in Markdown format. + + Very long method description in Markdown format. + + :param arg1: Argument 1 description + :param arg2: Argument 2 description + :return: Return value description + + Please note that double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. + If you need a code example, use triple backticks, e.g.: + ```python + print("Hello, world!") + ``` + + """ + return "Hello, world!" +``` + From 4f87bce3233c6efa99b7bca691918dbfdcb4389e Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 16:30:44 +0200 Subject: [PATCH 13/37] fix formatting --- tests/e2e/test_run_demos.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_run_demos.py b/tests/e2e/test_run_demos.py index cca88efc0..cdac759cb 100644 --- a/tests/e2e/test_run_demos.py +++ b/tests/e2e/test_run_demos.py @@ -170,6 +170,7 @@ def test_run_dqx_demo_tool(installation_ctx, make_schema, make_notebook, make_jo ) logging.info(f"Job run {run.run_id} completed successfully for dqx_demo_tool") + def test_run_dqx_streaming_demo_native(make_notebook, make_schema, make_job, tmp_path, library_ref): ws = WorkspaceClient() path = Path(__file__).parent.parent.parent / "demos" / "dqx_streaming_demo_native.py" From 0014418422c3130452292954ab4a43e9f9aabdfb Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 14 Aug 2025 17:48:27 +0200 Subject: [PATCH 14/37] Update docs/dqx/docs/dev/contributing.mdx --- docs/dqx/docs/dev/contributing.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index e4670084b..717475cff 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -310,7 +310,7 @@ hatch run docs:pydoc-markdown The API docs are generated in the `docs/dqx/docs/reference/api` directory. -To write API docs, use the following format: +To write API docs, use the following format in the source code: ```python From 1f2aba585d5d229f828d8d6f2ad6f7a8ea1d72ce Mon Sep 17 00:00:00 2001 From: Ivan Trusov Date: Thu, 14 Aug 2025 18:53:44 +0200 Subject: [PATCH 15/37] Update .github/workflows/docs-release.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/workflows/docs-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 38f9d230e..10361ce66 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -41,7 +41,7 @@ jobs: - name: Generate API docs - run: hatch pydoc-markdown + run: hatch run docs:pydoc-markdown - name: Build website run: yarn --cwd docs/dqx build From 80eb9b689fdeed19d4541700ffab316f73ae23df Mon Sep 17 00:00:00 2001 From: renardeinside Date: Thu, 14 Aug 2025 18:55:37 +0200 Subject: [PATCH 16/37] pin py version --- .github/workflows/docs-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 10361ce66..d4097edd7 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -24,7 +24,7 @@ jobs: with: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - python-version: 3.10 + python-version: '3.11' - name: Install Hatch run: | From 76b0a48ece214e5c3393aa0087babd82a0e07c21 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 11:16:59 +0200 Subject: [PATCH 17/37] added missing docstrings --- src/databricks/labs/dqx/base.py | 89 ++++++++-------- src/databricks/labs/dqx/engine.py | 170 ++++++++++++++++++++++-------- 2 files changed, 171 insertions(+), 88 deletions(-) diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index 29eb4b747..6a1cd0191 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -49,94 +49,93 @@ class DQEngineCoreBase(DQEngineBase): def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> DataFrame: - """Applies data quality checks to a given dataframe. + """Applies data quality checks to a given DataFrame. - :param df: dataframe to check - :param checks: list of checks to apply to the dataframe. Each check is an instance of DQRule class. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: dataframe with errors and warning result columns + :param df: DataFrame to check + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: DataFrame with errors and warning result columns """ @abc.abstractmethod def apply_checks_and_split( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks to a given dataframe and split it into two ("good" and "bad"), + """Applies data quality checks to a given DataFrame and split it into two ("good" and "bad"), according to the data quality checks. - :param df: dataframe to check - :param checks: list of checks to apply to the dataframe. Each check is an instance of DQRule class. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: two dataframes - "good" which includes warning rows but no result columns, and "data" having + :param df: DataFrame to check + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: two DataFrames - "good" which includes warning rows but no result columns, and "data" having error and warning rows and corresponding result columns """ @abc.abstractmethod - def apply_checks_by_metadata_and_split( + def apply_checks_by_metadata( self, df: DataFrame, checks: list[dict], custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, - ) -> tuple[DataFrame, DataFrame]: - """Wrapper around `apply_checks_and_split` for use in the metadata-driven pipelines. The main difference - is how the checks are specified - instead of using functions directly, they are described as function name plus - arguments. + ) -> DataFrame: + """ + Applies data quality checks defined as metadata to a given DataFrame. - :param df: dataframe to check + :param df: DataFrame to check :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - it will be used as an error/warning message, or `null` if it's evaluated to `false` * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). + * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) + :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: two dataframes - "good" which includes warning rows but no result columns, and "bad" having - error and warning rows and corresponding result columns + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: DataFrame with errors and warning result columns """ @abc.abstractmethod - def apply_checks_by_metadata( + def apply_checks_by_metadata_and_split( self, df: DataFrame, checks: list[dict], custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, - ) -> DataFrame: - """Wrapper around `apply_checks` for use in the metadata-driven pipelines. The main difference - is how the checks are specified - instead of using functions directly, they are described as function name plus - arguments. + ) -> tuple[DataFrame, DataFrame]: + """Applies data quality checks defined as metadata to a given DataFrame and split it into two ("good" and "bad"), + according to the data quality checks. - :param df: dataframe to check + :param df: DataFrame to check :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - it will be used as an error/warning message, or `null` if it's evaluated to `false` * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). + * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) + :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: dataframe with errors and warning result columns + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having + error and warning rows and corresponding result columns """ @staticmethod @abc.abstractmethod def validate_checks( - checks: list[dict], custom_check_functions: dict[str, Any] | None = None + checks: list[dict], + custom_check_functions: dict[str, Any] | None = None, + validate_custom_check_functions: bool = True, ) -> ChecksValidationStatus: """ - Validate the input dict to ensure they conform to expected structure and types. + Validates checks defined as metadata to ensure they conform to expected structure and types. - Each check can be a dictionary. The function validates - the presence of required keys, the existence and callability of functions, and the types - of arguments passed to these functions. - - :param checks: List of checks to apply to the dataframe. Each check should be a dictionary. - :param custom_check_functions: Optional dictionary with custom check functions. + The function validates the presence of required keys, the existence and callability of functions, + and the types of arguments passed to these functions. + :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. + :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). + :param validate_custom_check_functions: If True, validates custom check functions. :return ValidationStatus: The validation status. """ @@ -145,7 +144,7 @@ def get_invalid(self, df: DataFrame) -> DataFrame: """ Get records that violate data quality checks (records with warnings and errors). @param df: input DataFrame. - @return: dataframe with error and warning rows and corresponding result columns. + @return: DataFrame with error and warning rows and corresponding result columns. """ @abc.abstractmethod @@ -153,7 +152,7 @@ def get_valid(self, df: DataFrame) -> DataFrame: """ Get records that don't violate data quality checks (records with warnings but no errors). @param df: input DataFrame. - @return: dataframe with warning rows but no result columns. + @return: DataFrame with warning rows but no result columns. """ @staticmethod @@ -164,7 +163,7 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: This does not require installation of DQX in the workspace. The returning checks can be used as input for `apply_checks_by_metadata` function. - :param filepath: path to a file containing the checks. + :param filepath: path to a file containing checks definitions. :return: list of dq rules """ @@ -175,5 +174,5 @@ def save_checks_in_local_file(checks: list[dict], filepath: str): Save checks (dq rules) to a file (yaml or json) in the local file system. :param checks: list of dq rules to save - :param filepath: path to a file containing the checks. + :param filepath: path to a file where the checks definitions will be saved. """ diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index ef6dec372..3a922219c 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -40,7 +40,7 @@ class DQEngineCore(DQEngineCoreBase): - """Data Quality Engine Core class to apply data quality checks to a given dataframe. + """Data Quality Engine Core class to apply data quality checks to a given DataFrame. Args: workspace_client (WorkspaceClient): WorkspaceClient instance to use for accessing the workspace. extra_params (ExtraParams): Extra parameters for the DQEngine. @@ -72,6 +72,13 @@ def __init__( def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> DataFrame: + """Applies data quality checks to a given DataFrame. + + :param df: DataFrame to check + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: DataFrame with errors and warning result columns + """ if not checks: return self._append_empty_checks(df) @@ -95,6 +102,15 @@ def apply_checks( def apply_checks_and_split( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> tuple[DataFrame, DataFrame]: + """Applies data quality checks to a given DataFrame and split it into two ("good" and "bad"), + according to the data quality checks. + + :param df: DataFrame to check + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having + error and warning rows and corresponding result columns + """ if not checks: return df, self._append_empty_checks(df).limit(0) @@ -110,28 +126,58 @@ def apply_checks_and_split( return good_df, bad_df - def apply_checks_by_metadata_and_split( + def apply_checks_by_metadata( self, df: DataFrame, checks: list[dict], custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, - ) -> tuple[DataFrame, DataFrame]: + ) -> DataFrame: + """Applies data quality checks defined as metadata to a given DataFrame. + + :param df: DataFrame to check + :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: + * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - + it will be used as an error/warning message, or `null` if it's evaluated to `false` + * `name` - name that will be given to a resulting column. Autogenerated if not provided + * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) + :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). + If not specified, then only built-in functions are used for the checks. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: DataFrame with errors and warning result columns + """ dq_rule_checks = deserialize_checks(checks, custom_check_functions) - good_df, bad_df = self.apply_checks_and_split(df, dq_rule_checks, ref_dfs) - return good_df, bad_df + return self.apply_checks(df, dq_rule_checks, ref_dfs) - def apply_checks_by_metadata( + def apply_checks_by_metadata_and_split( self, df: DataFrame, checks: list[dict], custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, - ) -> DataFrame: + ) -> tuple[DataFrame, DataFrame]: + """Applies data quality checks defined as metadata to a given DataFrame and split it into two ("good" and "bad"), + according to the data quality checks. + + :param df: DataFrame to check + :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: + * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - + it will be used as an error/warning message, or `null` if it's evaluated to `false` + * `name` - name that will be given to a resulting column. Autogenerated if not provided + * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) + :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). + If not specified, then only built-in functions are used for the checks. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having + error and warning rows and corresponding result columns + """ dq_rule_checks = deserialize_checks(checks, custom_check_functions) - return self.apply_checks(df, dq_rule_checks, ref_dfs) + good_df, bad_df = self.apply_checks_and_split(df, dq_rule_checks, ref_dfs) + return good_df, bad_df @staticmethod def validate_checks( @@ -139,32 +185,70 @@ def validate_checks( custom_check_functions: dict[str, Any] | None = None, validate_custom_check_functions: bool = True, ) -> ChecksValidationStatus: + """ + Validates checks defined as metadata to ensure they conform to expected structure and types. + + The function validates the presence of required keys, the existence and callability of functions, + and the types of arguments passed to these functions. + + :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. + :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). + :param validate_custom_check_functions: If True, validates custom check functions. + :return ValidationStatus: The validation status. + """ return ChecksValidator.validate_checks(checks, custom_check_functions, validate_custom_check_functions) def get_invalid(self, df: DataFrame) -> DataFrame: + """ + Get records that violate data quality checks (records with warnings and errors). + + :param df: input DataFrame. + :return: DataFrame with error and warning rows and corresponding result columns. + """ return df.where( F.col(self._result_column_names[ColumnArguments.ERRORS]).isNotNull() | F.col(self._result_column_names[ColumnArguments.WARNINGS]).isNotNull() ) def get_valid(self, df: DataFrame) -> DataFrame: + """ + Get records that don't violate data quality checks (records with warnings but no errors). + + :param df: input DataFrame. + :return: DataFrame with warning rows but no result columns. + """ return df.where(F.col(self._result_column_names[ColumnArguments.ERRORS]).isNull()).drop( self._result_column_names[ColumnArguments.ERRORS], self._result_column_names[ColumnArguments.WARNINGS] ) @staticmethod def load_checks_from_local_file(filepath: str) -> list[dict]: + """ + Load checks (dq rules) from a file (json or yaml) in the local file system. + This does not require installation of DQX in the workspace. + The returning checks can be used as input for `apply_checks_by_metadata` function. + + :param filepath: path to a file containing checks definitions. + :return: list of dq rules + """ return FileChecksStorageHandler().load(FileChecksStorageConfig(location=filepath)) @staticmethod def save_checks_in_local_file(checks: list[dict], filepath: str): + """ + Save checks (dq rules) to a file (json or yaml) in the local file system. + This does not require installation of DQX in the workspace. + + :param checks: list of dq rules to save + :param filepath: path to a file where the checks definitions will be saved. + """ return FileChecksStorageHandler().save(checks, FileChecksStorageConfig(location=filepath)) @staticmethod def _get_check_columns(checks: list[DQRule], criticality: str) -> list[DQRule]: """Get check columns based on criticality. - :param checks: list of checks to apply to the dataframe + :param checks: list of checks to apply to the DataFrame :param criticality: criticality :return: list of check columns """ @@ -176,10 +260,10 @@ def _all_are_dq_rules(checks: list[DQRule]) -> bool: return all(isinstance(check, DQRule) for check in checks) def _append_empty_checks(self, df: DataFrame) -> DataFrame: - """Append empty checks at the end of dataframe. + """Append empty checks at the end of DataFrame. - :param df: dataframe without checks - :return: dataframe with checks + :param df: DataFrame without checks + :return: DataFrame with checks """ return df.select( "*", @@ -243,7 +327,7 @@ def _create_results_array( class DQEngine(DQEngineBase): - """Data Quality Engine class to apply data quality checks to a given dataframe.""" + """Data Quality Engine class to apply data quality checks to a given DataFrame.""" def __init__( self, @@ -266,25 +350,25 @@ def __init__( def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> DataFrame: - """Applies data quality checks to a given dataframe. + """Applies data quality checks to a given DataFrame. - :param df: dataframe to check - :param checks: list of checks to apply to the dataframe. Each check is an instance of DQRule class. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: dataframe with errors and warning result columns + :param df: DataFrame to check + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: DataFrame with errors and warning result columns """ return self._engine.apply_checks(df, checks, ref_dfs) def apply_checks_and_split( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks to a given dataframe and split it into two ("good" and "bad"), + """Applies data quality checks to a given DataFrame and split it into two ("good" and "bad"), according to the data quality checks. - :param df: dataframe to check - :param checks: list of checks to apply to the dataframe. Each check is an instance of DQRule class. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: two dataframes - "good" which includes warning rows but no result columns, and "data" having + :param df: DataFrame to check + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: two DataFrames - "good" which includes warning rows but no result columns, and "data" having error and warning rows and corresponding result columns """ return self._engine.apply_checks_and_split(df, checks, ref_dfs) @@ -300,19 +384,19 @@ def apply_checks_by_metadata_and_split( is how the checks are specified - instead of using functions directly, they are described as function name plus arguments. - :param df: dataframe to check + :param df: DataFrame to check :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - it will be used as an error/warning message, or `null` if it's evaluated to `false` * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) + * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) * `filter` (optional) - Expression for filtering data quality checks * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference dataframes to use in the checks, if applicable - :return: two dataframes - "good" which includes warning rows but no result columns, and "bad" having + :param ref_dfs: reference DataFrames to use in the checks, if applicable + :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having error and warning rows and corresponding result columns """ return self._engine.apply_checks_by_metadata_and_split(df, checks, custom_check_functions, ref_dfs) @@ -328,19 +412,19 @@ def apply_checks_by_metadata( is how the checks are specified - instead of using functions directly, they are described as function name plus arguments. - :param df: dataframe to check + :param df: DataFrame to check :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - it will be used as an error/warning message, or `null` if it's evaluated to `false` * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) + * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) * `filter` (optional) - Expression for filtering data quality checks * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - :param ref_dfs: reference dataframes to use in the checks, if applicable + :param ref_dfs: reference DataFrames to use in the checks, if applicable If not specified, then only built-in functions are used for the checks. - :return: dataframe with errors and warning result columns + :return: DataFrame with errors and warning result columns """ return self._engine.apply_checks_by_metadata(df, checks, custom_check_functions, ref_dfs) @@ -360,11 +444,11 @@ def apply_checks_and_save_in_table( If quarantine_config is not provided, all records (with error/warning columns) will be written to the output table. - :param checks: list of checks to apply to the dataframe. Each check is an instance of DQRule class. + :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. :param input_config: Input data configuration (e.g. table name or file location, read options) :param output_config: Output data configuration (e.g. table name, output mode, write options) :param quarantine_config: Optional quarantine data configuration (e.g. table name, output mode, write options) - :param ref_dfs: Reference dataframes to use in the checks, if applicable + :param ref_dfs: Reference DataFrames to use in the checks, if applicable """ # Read data from the specified table df = read_input_data(self.spark, input_config) @@ -400,13 +484,13 @@ def apply_checks_by_metadata_and_save_in_table( * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - it will be used as an error/warning message, or `null` if it's evaluated to `false` * `name` - Name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) -Possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) + * `criticality` (optional) -Possible values are `error` (data going only into "bad" DataFrame), + and `warn` (data is going into both DataFrames) :param input_config: Input data configuration (e.g. table name or file location, read options) :param output_config: Output data configuration (e.g. table name, output mode, write options) :param quarantine_config: Optional quarantine data configuration (e.g. table name, output mode, write options) :param custom_check_functions: Dictionary with custom check functions (eg. ``globals()`` of calling module). - :param ref_dfs: Reference dataframes to use in the checks, if applicable + :param ref_dfs: Reference DataFrames to use in the checks, if applicable """ # Read data from the specified table df = read_input_data(self.spark, input_config) @@ -434,7 +518,7 @@ def validate_checks( the presence of required keys, the existence and callability of functions, and the types of arguments passed to these functions. - :param checks: List of checks to apply to the dataframe. Each check should be a dictionary. + :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. :param custom_check_functions: Optional dictionary with custom check functions. :param validate_custom_check_functions: If True, validate custom check functions. @@ -446,7 +530,7 @@ def get_invalid(self, df: DataFrame) -> DataFrame: """ Get records that violate data quality checks (records with warnings and errors). @param df: input DataFrame. - @return: dataframe with error and warning rows and corresponding result columns. + @return: DataFrame with error and warning rows and corresponding result columns. """ return self._engine.get_invalid(df) @@ -454,7 +538,7 @@ def get_valid(self, df: DataFrame) -> DataFrame: """ Get records that don't violate data quality checks (records with warnings but no errors). @param df: input DataFrame. - @return: dataframe with warning rows but no result columns. + @return: DataFrame with warning rows but no result columns. """ return self._engine.get_valid(df) @@ -471,8 +555,8 @@ def save_results_in_table( """ Save quarantine and output data to the `quarantine_table` and `output_table`. - :param quarantine_df: Optional Dataframe containing the quarantine data - :param output_df: Optional Dataframe containing the output data. If not provided, use run config + :param quarantine_df: Optional DataFrame containing the quarantine data + :param output_df: Optional DataFrame containing the output data. If not provided, use run config :param output_config: Optional configuration for saving the output data. If not provided, use run config :param quarantine_config: Optional configuration for saving the quarantine data. If not provided, use run config :param run_config_name: Optional name of the run (config) to use From 32839b0557624873f84c09bb9795cdf61f61ca32 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 12:11:19 +0200 Subject: [PATCH 18/37] updated docstrings --- pyproject.toml | 3 +- src/databricks/labs/dqx/base.py | 130 ++++---- src/databricks/labs/dqx/engine.py | 494 ++++++++++++++++++------------ 3 files changed, 359 insertions(+), 268 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81f52fcc1..342111fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -595,7 +595,8 @@ disable = [ "unnecessary-default-type-args", "mock-no-usage", "broad-exception-caught", - "rewrite-as-for-loop" + "rewrite-as-for-loop", + "redundant-returns-doc", ] # Enable the message, report, category or checker with the given id(s). You can diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index 6a1cd0191..58911d9a2 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -16,15 +16,19 @@ def __init__(self, workspace_client: WorkspaceClient): @cached_property def ws(self) -> WorkspaceClient: - """ - Cached property to verify and return the workspace client. + """Return a verified `WorkspaceClient` configured for DQX. + + Ensures workspace connectivity and sets the product info used for + telemetry so that requests are attributed to `dqx`. """ return self._verify_workspace_client(self._workspace_client) @cached_property def spark(self) -> SparkSession: - """ - Cached property return the SparkSession. + """Return the `SparkSession` associated with this engine. + + The session is created during initialization using + `SparkSession.builder.getOrCreate()`. """ return self._spark @@ -32,7 +36,7 @@ def spark(self) -> SparkSession: @final def _verify_workspace_client(ws: WorkspaceClient) -> WorkspaceClient: """ - Verifies the Databricks workspace client configuration. + Verify the Databricks WorkspaceClient configuration and connectivity. """ # Using reflection to set right value for _product_info as dqx for telemetry product_info = getattr(ws.config, '_product_info') @@ -49,26 +53,26 @@ class DQEngineCoreBase(DQEngineBase): def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> DataFrame: - """Applies data quality checks to a given DataFrame. + """Apply data quality checks to the given DataFrame. - :param df: DataFrame to check - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: DataFrame with errors and warning result columns + :param df: Input DataFrame to check. + :param checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame that includes errors and warnings result columns. """ @abc.abstractmethod def apply_checks_and_split( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks to a given DataFrame and split it into two ("good" and "bad"), - according to the data quality checks. + """Apply data quality checks to the given DataFrame and split the results into two DataFrames + ("good" and "bad"). - :param df: DataFrame to check - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: two DataFrames - "good" which includes warning rows but no result columns, and "data" having - error and warning rows and corresponding result columns + :param df: Input DataFrame to check. + :param checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ @abc.abstractmethod @@ -80,19 +84,17 @@ def apply_checks_by_metadata( ref_dfs: dict[str, DataFrame] | None = None, ) -> DataFrame: """ - Applies data quality checks defined as metadata to a given DataFrame. + Apply data quality checks defined as metadata to the given DataFrame. - :param df: DataFrame to check - :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: DataFrame with errors and warning result columns + :param df: Input DataFrame to check. + :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check`: A check definition including check function and arguments to use. + - `name`: Optional name for the resulting column. Auto-generated if not provided. + - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame that includes errors and warnings result columns. """ @abc.abstractmethod @@ -103,21 +105,19 @@ def apply_checks_by_metadata_and_split( custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks defined as metadata to a given DataFrame and split it into two ("good" and "bad"), - according to the data quality checks. - - :param df: DataFrame to check - :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). - If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having - error and warning rows and corresponding result columns + """Apply data quality checks defined as metadata to the given DataFrame and split the results into + two DataFrames ("good" and "bad"). + + :param df: Input DataFrame to check. + :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check`: A check definition including check function and arguments to use. + - `name`: Optional name for the resulting column. Auto-generated if not provided. + - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ @staticmethod @@ -128,51 +128,53 @@ def validate_checks( validate_custom_check_functions: bool = True, ) -> ChecksValidationStatus: """ - Validates checks defined as metadata to ensure they conform to expected structure and types. + Validate checks defined as metadata to ensure they conform to the expected structure and types. - The function validates the presence of required keys, the existence and callability of functions, - and the types of arguments passed to these functions. + This method validates the presence of required keys, the existence and callability of functions, + and the types of arguments passed to those functions. :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - :param validate_custom_check_functions: If True, validates custom check functions. - :return ValidationStatus: The validation status. + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param validate_custom_check_functions: If True, validate custom check functions. + :return: ChecksValidationStatus indicating the validation result. """ @abc.abstractmethod def get_invalid(self, df: DataFrame) -> DataFrame: """ - Get records that violate data quality checks (records with warnings and errors). - @param df: input DataFrame. - @return: DataFrame with error and warning rows and corresponding result columns. + Return records that violate data quality checks (rows with warnings or errors). + + :param df: Input DataFrame. + :return: DataFrame with rows that have errors or warnings and the corresponding result columns. """ @abc.abstractmethod def get_valid(self, df: DataFrame) -> DataFrame: """ - Get records that don't violate data quality checks (records with warnings but no errors). - @param df: input DataFrame. - @return: DataFrame with warning rows but no result columns. + Return records that do not violate data quality checks (rows with warnings but no errors). + + :param df: Input DataFrame. + :return: DataFrame with warning rows but without the results columns. """ @staticmethod @abc.abstractmethod def load_checks_from_local_file(filepath: str) -> list[dict]: """ - Load checks (dq rules) from a file (json or yaml) in the local file system. - This does not require installation of DQX in the workspace. - The returning checks can be used as input for `apply_checks_by_metadata` function. + Load DQ rules (checks) from a local JSON or YAML file. + + The returned checks can be used as input to `apply_checks_by_metadata`. - :param filepath: path to a file containing checks definitions. - :return: list of dq rules + :param filepath: Path to a file containing checks definitions. + :return: List of DQ rules (checks). """ @staticmethod @abc.abstractmethod def save_checks_in_local_file(checks: list[dict], filepath: str): """ - Save checks (dq rules) to a file (yaml or json) in the local file system. + Save DQ rules (checks) to a local YAML or JSON file. - :param checks: list of dq rules to save - :param filepath: path to a file where the checks definitions will be saved. + :param checks: List of DQ rules (checks) to save. + :param filepath: Path to a file where the checks definitions will be saved. """ diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 3a922219c..b8402394f 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -40,10 +40,11 @@ class DQEngineCore(DQEngineCoreBase): - """Data Quality Engine Core class to apply data quality checks to a given DataFrame. - Args: - workspace_client (WorkspaceClient): WorkspaceClient instance to use for accessing the workspace. - extra_params (ExtraParams): Extra parameters for the DQEngine. + """Core engine to apply data quality checks to a DataFrame. + + :param workspace_client: WorkspaceClient instance used to access the workspace. + :param spark: Optional SparkSession to use. If not provided, the active session is used. + :param extra_params: Optional extra parameters for the engine, such as result column names and run metadata. """ def __init__( @@ -72,12 +73,12 @@ def __init__( def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> DataFrame: - """Applies data quality checks to a given DataFrame. + """Apply data quality checks to the given DataFrame. - :param df: DataFrame to check - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: DataFrame with errors and warning result columns + :param df: Input DataFrame to check. + :param checks: List of checks to apply. Each check must be a `DQRule` instance. + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame with errors and warnings result columns. """ if not checks: return self._append_empty_checks(df) @@ -102,14 +103,14 @@ def apply_checks( def apply_checks_and_split( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks to a given DataFrame and split it into two ("good" and "bad"), - according to the data quality checks. + """Apply data quality checks to the given DataFrame and split the results into two DataFrames + ("good" and "bad"). - :param df: DataFrame to check - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having - error and warning rows and corresponding result columns + :param df: Input DataFrame to check. + :param checks: List of checks to apply. Each check must be a `DQRule` instance. + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ if not checks: return df, self._append_empty_checks(df).limit(0) @@ -133,19 +134,17 @@ def apply_checks_by_metadata( custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, ) -> DataFrame: - """Applies data quality checks defined as metadata to a given DataFrame. - - :param df: DataFrame to check - :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: DataFrame with errors and warning result columns + """Apply data quality checks defined as metadata to the given DataFrame. + + :param df: Input DataFrame to check. + :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check`: A check definition including check function and arguments to use. + - `name`: Optional name for the resulting column. Auto-generated if not provided. + - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame with errors and warnings result columns. """ dq_rule_checks = deserialize_checks(checks, custom_check_functions) @@ -158,21 +157,18 @@ def apply_checks_by_metadata_and_split( custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks defined as metadata to a given DataFrame and split it into two ("good" and "bad"), - according to the data quality checks. - - :param df: DataFrame to check - :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having - error and warning rows and corresponding result columns + """Apply data quality checks defined as metadata to the given DataFrame and split the results into + two DataFrames ("good" and "bad"). + + :param df: Input DataFrame to check. + :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check`: A check definition including check function and arguments to use. + - `name`: Optional name for the resulting column. Auto-generated if not provided. + - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame that includes errors and warnings result columns. """ dq_rule_checks = deserialize_checks(checks, custom_check_functions) @@ -186,24 +182,24 @@ def validate_checks( validate_custom_check_functions: bool = True, ) -> ChecksValidationStatus: """ - Validates checks defined as metadata to ensure they conform to expected structure and types. + Validate checks defined as metadata to ensure they conform to the expected structure and types. - The function validates the presence of required keys, the existence and callability of functions, - and the types of arguments passed to these functions. + This method validates the presence of required keys, the existence and callability of functions, + and the types of arguments passed to those functions. :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - :param validate_custom_check_functions: If True, validates custom check functions. - :return ValidationStatus: The validation status. + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param validate_custom_check_functions: If True, validate custom check functions. + :return: ChecksValidationStatus indicating the validation result. """ return ChecksValidator.validate_checks(checks, custom_check_functions, validate_custom_check_functions) def get_invalid(self, df: DataFrame) -> DataFrame: """ - Get records that violate data quality checks (records with warnings and errors). + Return records that violate data quality checks (rows with warnings or errors). - :param df: input DataFrame. - :return: DataFrame with error and warning rows and corresponding result columns. + :param df: Input DataFrame. + :return: DataFrame with rows that have errors or warnings and the corresponding result columns. """ return df.where( F.col(self._result_column_names[ColumnArguments.ERRORS]).isNotNull() @@ -212,10 +208,10 @@ def get_invalid(self, df: DataFrame) -> DataFrame: def get_valid(self, df: DataFrame) -> DataFrame: """ - Get records that don't violate data quality checks (records with warnings but no errors). + Return records that do not violate data quality checks (rows with warnings but no errors). - :param df: input DataFrame. - :return: DataFrame with warning rows but no result columns. + :param df: Input DataFrame. + :return: DataFrame with warning rows but without the results columns. """ return df.where(F.col(self._result_column_names[ColumnArguments.ERRORS]).isNull()).drop( self._result_column_names[ColumnArguments.ERRORS], self._result_column_names[ColumnArguments.WARNINGS] @@ -224,23 +220,22 @@ def get_valid(self, df: DataFrame) -> DataFrame: @staticmethod def load_checks_from_local_file(filepath: str) -> list[dict]: """ - Load checks (dq rules) from a file (json or yaml) in the local file system. - This does not require installation of DQX in the workspace. - The returning checks can be used as input for `apply_checks_by_metadata` function. + Load DQ rules (checks) from a local JSON or YAML file. + + The returned checks can be used as input to `apply_checks_by_metadata`. - :param filepath: path to a file containing checks definitions. - :return: list of dq rules + :param filepath: Path to a file containing checks definitions. + :return: List of DQ rules. """ return FileChecksStorageHandler().load(FileChecksStorageConfig(location=filepath)) @staticmethod def save_checks_in_local_file(checks: list[dict], filepath: str): """ - Save checks (dq rules) to a file (json or yaml) in the local file system. - This does not require installation of DQX in the workspace. + Save DQ rules (checks) to a local YAML or JSON file. - :param checks: list of dq rules to save - :param filepath: path to a file where the checks definitions will be saved. + :param checks: List of DQ rules (checks) to save. + :param filepath: Path to a file where the checks definitions will be saved. """ return FileChecksStorageHandler().save(checks, FileChecksStorageConfig(location=filepath)) @@ -327,7 +322,11 @@ def _create_results_array( class DQEngine(DQEngineBase): - """Data Quality Engine class to apply data quality checks to a given DataFrame.""" + """High-level engine to apply data quality checks and manage IO. + + This class delegates core checking logic to `DQEngineCore` while providing helpers to + read inputs, persist results, and work with different storage backends for checks. + """ def __init__( self, @@ -350,83 +349,71 @@ def __init__( def apply_checks( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> DataFrame: - """Applies data quality checks to a given DataFrame. + """Apply data quality checks to the given DataFrame. - :param df: DataFrame to check - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: DataFrame with errors and warning result columns + :param df: Input DataFrame to check. + :param checks: List of checks to apply. Each check must be a `DQRule` instance. + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame with errors and warnings result columns. """ return self._engine.apply_checks(df, checks, ref_dfs) def apply_checks_and_split( self, df: DataFrame, checks: list[DQRule], ref_dfs: dict[str, DataFrame] | None = None ) -> tuple[DataFrame, DataFrame]: - """Applies data quality checks to a given DataFrame and split it into two ("good" and "bad"), - according to the data quality checks. + """Apply data quality checks to the given DataFrame and split the results into two DataFrames + ("good" and "bad"). - :param df: DataFrame to check - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: two DataFrames - "good" which includes warning rows but no result columns, and "data" having - error and warning rows and corresponding result columns + :param df: Input DataFrame to check. + :param checks: List of checks to apply. Each check must be a `DQRule` instance. + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ return self._engine.apply_checks_and_split(df, checks, ref_dfs) - def apply_checks_by_metadata_and_split( + def apply_checks_by_metadata( self, df: DataFrame, checks: list[dict], custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, - ) -> tuple[DataFrame, DataFrame]: - """Wrapper around `apply_checks_and_split` for use in the metadata-driven pipelines. The main difference - is how the checks are specified - instead of using functions directly, they are described as function name plus - arguments. - - :param df: DataFrame to check - :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - * `filter` (optional) - Expression for filtering data quality checks - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). - If not specified, then only built-in functions are used for the checks. - :param ref_dfs: reference DataFrames to use in the checks, if applicable - :return: two DataFrames - "good" which includes warning rows but no result columns, and "bad" having - error and warning rows and corresponding result columns + ) -> DataFrame: + """Apply data quality checks defined as metadata to the given DataFrame. + + :param df: Input DataFrame to check. + :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check`: A check definition including check function and arguments to use. + - `name`: Optional name for the resulting column. Auto-generated if not provided. + - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame with errors and warnings result columns. """ - return self._engine.apply_checks_by_metadata_and_split(df, checks, custom_check_functions, ref_dfs) + return self._engine.apply_checks_by_metadata(df, checks, custom_check_functions, ref_dfs) - def apply_checks_by_metadata( + def apply_checks_by_metadata_and_split( self, df: DataFrame, checks: list[dict], custom_check_functions: dict[str, Any] | None = None, ref_dfs: dict[str, DataFrame] | None = None, - ) -> DataFrame: - """Wrapper around `apply_checks` for use in the metadata-driven pipelines. The main difference - is how the checks are specified - instead of using functions directly, they are described as function name plus - arguments. - - :param df: DataFrame to check - :param checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - * `filter` (optional) - Expression for filtering data quality checks - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of calling module). - :param ref_dfs: reference DataFrames to use in the checks, if applicable - If not specified, then only built-in functions are used for the checks. - :return: DataFrame with errors and warning result columns + ) -> tuple[DataFrame, DataFrame]: + """Apply data quality checks defined as metadata to the given DataFrame and split the results into + two DataFrames ("good" and "bad"). + + :param df: Input DataFrame to check. + :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check`: A check definition including check function and arguments to use. + - `name`: Optional name for the resulting column. Auto-generated if not provided. + - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + :param ref_dfs: Optional reference DataFrames to use in the checks. + :return: DataFrame that includes errors and warnings result columns. """ - return self._engine.apply_checks_by_metadata(df, checks, custom_check_functions, ref_dfs) + return self._engine.apply_checks_by_metadata_and_split(df, checks, custom_check_functions, ref_dfs) def apply_checks_and_save_in_table( self, @@ -437,18 +424,19 @@ def apply_checks_and_save_in_table( ref_dfs: dict[str, DataFrame] | None = None, ) -> None: """ - Apply data quality checks to a table or view and write the result to table(s). + Apply data quality checks to input data and save results. + + If `quarantine_config` is provided, split the data into valid and invalid records: + - valid records are written using `output_config` + - invalid records are written using `quarantine_config` - If quarantine_config is provided, the data will be split into good and bad records, - with good records written to the output table and bad records to the quarantine table. - If quarantine_config is not provided, all records (with error/warning columns) - will be written to the output table. + If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. - :param checks: list of checks to apply to the DataFrame. Each check is an instance of DQRule class. - :param input_config: Input data configuration (e.g. table name or file location, read options) - :param output_config: Output data configuration (e.g. table name, output mode, write options) - :param quarantine_config: Optional quarantine data configuration (e.g. table name, output mode, write options) - :param ref_dfs: Reference DataFrames to use in the checks, if applicable + :param checks: List of `DQRule` checks to apply. + :param input_config: Input configuration (e.g., table/view or file location and read options). + :param output_config: Output configuration (e.g., table name, mode, and write options). + :param quarantine_config: Optional configuration for writing invalid records. + :param ref_dfs: Optional reference DataFrames used by checks. """ # Read data from the specified table df = read_input_data(self.spark, input_config) @@ -473,24 +461,23 @@ def apply_checks_by_metadata_and_save_in_table( ref_dfs: dict[str, DataFrame] | None = None, ) -> None: """ - Apply data quality checks to a table or view and write the result to table(s). - - If quarantine_config is provided, the data will be split into good and bad records, - with good records written to the output table and bad records to the quarantine table. - If quarantine_config is not provided, all records (with error/warning columns) - will be written to the output table. - - :param checks: List of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - Name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) -Possible values are `error` (data going only into "bad" DataFrame), - and `warn` (data is going into both DataFrames) - :param input_config: Input data configuration (e.g. table name or file location, read options) - :param output_config: Output data configuration (e.g. table name, output mode, write options) - :param quarantine_config: Optional quarantine data configuration (e.g. table name, output mode, write options) - :param custom_check_functions: Dictionary with custom check functions (eg. ``globals()`` of calling module). - :param ref_dfs: Reference DataFrames to use in the checks, if applicable + Apply metadata-defined data quality checks to input data and save results. + + If `quarantine_config` is provided, split the data into valid and invalid records: + - valid records are written using `output_config` + - invalid records are written using `quarantine_config` + + If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. + + :param checks: List of dictionaries describing checks. Each item must include: + - `check`: Check definition (function name plus arguments) to evaluate. + - `name` (optional): Result column name; auto-generated if omitted. + - `criticality` (optional): One of `error` (only in invalid output) or `warn` (also appears in valid output). + :param input_config: Input configuration (e.g., table/view or file location and read options). + :param output_config: Output configuration (e.g., table name, mode, and write options). + :param quarantine_config: Optional configuration for writing invalid records. + :param custom_check_functions: Optional mapping of custom check function names to callables/modules (e.g., globals()). + :param ref_dfs: Optional reference DataFrames used by checks. """ # Read data from the specified table df = read_input_data(self.spark, input_config) @@ -512,33 +499,33 @@ def validate_checks( validate_custom_check_functions: bool = True, ) -> ChecksValidationStatus: """ - Validate the input dict to ensure they conform to expected structure and types. + Validate checks defined as metadata to ensure they conform to the expected structure and types. - Each check can be a dictionary. The function validates - the presence of required keys, the existence and callability of functions, and the types - of arguments passed to these functions. + This method validates the presence of required keys, the existence and callability of functions, + and the types of arguments passed to those functions. :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - :param custom_check_functions: Optional dictionary with custom check functions. + :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param validate_custom_check_functions: If True, validate custom check functions. - - :return ValidationStatus: The validation status. + :return: ChecksValidationStatus indicating the validation result. """ return DQEngineCore.validate_checks(checks, custom_check_functions, validate_custom_check_functions) def get_invalid(self, df: DataFrame) -> DataFrame: """ - Get records that violate data quality checks (records with warnings and errors). - @param df: input DataFrame. - @return: DataFrame with error and warning rows and corresponding result columns. + Return records that violate data quality checks (rows with warnings or errors). + + :param df: Input DataFrame. + :return: DataFrame with rows that have errors or warnings and the corresponding result columns. """ return self._engine.get_invalid(df) def get_valid(self, df: DataFrame) -> DataFrame: """ - Get records that don't violate data quality checks (records with warnings but no errors). - @param df: input DataFrame. - @return: DataFrame with warning rows but no result columns. + Return records that do not violate data quality checks (rows with warnings but no errors). + + :param df: Input DataFrame. + :return: DataFrame with warning rows but without the results columns. """ return self._engine.get_valid(df) @@ -552,16 +539,21 @@ def save_results_in_table( product_name: str = "dqx", assume_user: bool = True, ): - """ - Save quarantine and output data to the `quarantine_table` and `output_table`. - - :param quarantine_df: Optional DataFrame containing the quarantine data - :param output_df: Optional DataFrame containing the output data. If not provided, use run config - :param output_config: Optional configuration for saving the output data. If not provided, use run config - :param quarantine_config: Optional configuration for saving the quarantine data. If not provided, use run config - :param run_config_name: Optional name of the run (config) to use - :param product_name: name of the product/installation directory - :param assume_user: if True, assume user installation + """Persist result DataFrames using explicit configs or the named run configuration. + + Behavior: + - If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`. + - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`. + - A write occurs only when both a DataFrame and its corresponding config are available. + + :param output_df: DataFrame with valid rows to be saved (optional). + :param quarantine_df: DataFrame with invalid rows to be saved (optional). + :param output_config: Configuration describing where/how to write the valid rows. If omitted, falls back to the run config. + :param quarantine_config: Configuration describing where/how to write the invalid rows. If omitted, falls back to the run config. + :param run_config_name: Name of the run configuration to load when a config parameter is omitted. + :param product_name: Product/installation identifier used to resolve installation paths for config loading. + :param assume_user: Whether to assume a per-user installation when loading the run configuration. + :return: None """ if output_df is not None and output_config is None: run_config = self._run_config_loader.load_run_config(run_config_name, assume_user, product_name) @@ -578,34 +570,45 @@ def save_results_in_table( save_dataframe_as_table(quarantine_df, quarantine_config) def load_checks(self, config: BaseChecksStorageConfig) -> list[dict]: - """ - Load checks (dq rules) from the specified source type (file or table). - :param config: storage configuration - Allowed configs are: - - `FileChecksStorageConfig`: for loading checks from a file in the local filesystem - - `WorkspaceFileChecksStorageConfig`: for loading checks from a workspace file - - `TableChecksStorageConfig`: for loading checks from a table - - `InstallationChecksStorageConfig`: for loading checks from the installation directory - - `VolumeFileChecksStorageConfig`: for loading checks from a Unity Catalog volume file - - ... - :raises ValueError: if the source type is unknown + """Load DQ rules (checks) from the storage backend described by `config`. + + This method delegates to a storage handler selected by the factory + based on the concrete type of `config` and returns the parsed list + of checks (as dictionaries) ready for `apply_checks_by_metadata`. + + Supported storage configurations include, for example: + - `FileChecksStorageConfig` (local file) + - `WorkspaceFileChecksStorageConfig` (Databricks workspace file) + - `TableChecksStorageConfig` (table-backed storage) + - `InstallationChecksStorageConfig` (installation directory) + - `VolumeFileChecksStorageConfig` (Unity Catalog volume file) + + :param config: Configuration object describing the storage backend. + :return: List of DQ rules (checks) represented as dictionaries. + :raises ValueError: If the configuration type is unsupported. """ handler = self._checks_handler_factory.create(config) return handler.load(config) def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> None: - """ - Save checks (dq rules) to the specified storage type (file or table). - :param checks: list of dq rules to save - :param config: storage configuration - Allowed configs are: - - `FileChecksStorageConfig`: for saving checks in a file in the local filesystem - - `WorkspaceFileChecksStorageConfig`: for saving checks in a workspace file - - `TableChecksStorageConfig`: for saving checks in a table - - `InstallationChecksStorageConfig`: for saving checks in the installation directory - - `VolumeFileChecksStorageConfig`: for saving checks in a Unity Catalog volume file - - ... - :raises ValueError: if the storage type is unknown + """Persist DQ rules (checks) to the storage backend described by `config`. + + The appropriate storage handler is resolved from the configuration + type and used to write the provided checks. Any write semantics + (e.g., append/overwrite) are controlled by fields on `config` + such as `mode` where applicable. + + Supported storage configurations include, for example: + - `FileChecksStorageConfig` (local file) + - `WorkspaceFileChecksStorageConfig` (Databricks workspace file) + - `TableChecksStorageConfig` (table-backed storage) + - `InstallationChecksStorageConfig` (installation directory) + - `VolumeFileChecksStorageConfig` (Unity Catalog volume file) + + :param checks: List of DQ rules (checks) to save (as dictionaries). + :param config: Configuration object describing the storage backend and write options. + :return: None + :raises ValueError: If the configuration type is unsupported. """ handler = self._checks_handler_factory.create(config) handler.save(checks, config) @@ -615,8 +618,16 @@ def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> No # @staticmethod def load_checks_from_local_file(filepath: str) -> list[dict]: + """Deprecated: Use `load_checks` with `FileChecksStorageConfig` instead. + + Load DQ rules (checks) from a local JSON or YAML file. + + :param filepath: Path to a file containing checks definitions. + :return: List of DQ rules (checks) represented as dictionaries. + """ warnings.warn( - "Use `load_checks` method instead. This method will be removed in future versions.", + "Use `load_checks` method with `FileChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) @@ -624,32 +635,67 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: @staticmethod def save_checks_in_local_file(checks: list[dict], path: str): + """Deprecated: Use `save_checks` with `FileChecksStorageConfig` instead. + + Save DQ rules (checks) to a local YAML or JSON file. + + :param checks: List of DQ rules (checks) to save. + :param path: File path where the checks definitions will be saved. + :return: None + """ warnings.warn( - "Use `save_checks` method instead. This method will be removed in future versions.", + "Use `save_checks` method with `FileChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) return DQEngineCore.save_checks_in_local_file(checks, path) def load_checks_from_workspace_file(self, workspace_path: str) -> list[dict]: + """Deprecated: Use `load_checks` with `WorkspaceFileChecksStorageConfig` instead. + + Load checks stored in a Databricks workspace file. + + :param workspace_path: Path to the workspace file containing checks definitions. + :return: List of DQ rules (checks) represented as dictionaries. + """ warnings.warn( - "Use `load_checks` method instead. This method will be removed in future versions.", + "Use `load_checks` method with `WorkspaceFileChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) return self.load_checks(WorkspaceFileChecksStorageConfig(location=workspace_path)) def save_checks_in_workspace_file(self, checks: list[dict], workspace_path: str): + """Deprecated: Use `save_checks` with `WorkspaceFileChecksStorageConfig` instead. + + Save checks to a Databricks workspace file. + + :param checks: List of DQ rules (checks) to save. + :param workspace_path: Path to the workspace file where checks will be saved. + :return: None + """ warnings.warn( - "Use `save_checks` method instead. This method will be removed in future versions.", + "Use `save_checks` method with `WorkspaceFileChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) return self.save_checks(checks, WorkspaceFileChecksStorageConfig(location=workspace_path)) def load_checks_from_table(self, table_name: str, run_config_name: str = "default") -> list[dict]: + """Deprecated: Use `load_checks` with `TableChecksStorageConfig` instead. + + Load checks from a table. + + :param table_name: Fully qualified table name where checks are stored. + :param run_config_name: Name of the run configuration (used by the storage handler if needed). + :return: List of DQ rules (checks) represented as dictionaries. + """ warnings.warn( - "Use `load_checks` method instead. This method will be removed in future versions.", + "Use `load_checks` method with `TableChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) @@ -658,8 +704,19 @@ def load_checks_from_table(self, table_name: str, run_config_name: str = "defaul def save_checks_in_table( self, checks: list[dict], table_name: str, run_config_name: str = "default", mode: str = "append" ): + """Deprecated: Use `save_checks` with `TableChecksStorageConfig` instead. + + Save checks to a table. + + :param checks: List of DQ rules (checks) to save. + :param table_name: Fully qualified table name where checks will be written. + :param run_config_name: Name of the run configuration (used by the storage handler if needed). + :param mode: Write mode, e.g., "append" or "overwrite". + :return: None + """ warnings.warn( - "Use `save_checks` method instead. This method will be removed in future versions.", + "Use `save_checks` method with `TableChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) @@ -674,8 +731,19 @@ def load_checks_from_installation( product_name: str = "dqx", assume_user: bool = True, ) -> list[dict]: + """Deprecated: Use `load_checks` with `InstallationChecksStorageConfig` instead. + + Load checks from the installation directory. + + :param run_config_name: Named run configuration to resolve installation paths and defaults. + :param method: Deprecated parameter; ignored. + :param product_name: Product/installation identifier (e.g., "dqx"). + :param assume_user: Whether to assume a per-user installation layout. + :return: List of DQ rules (checks) represented as dictionaries. + """ warnings.warn( - "Use `load_checks` method instead. This method will be removed in future versions.", + "Use `load_checks` method with `InstallationChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) @@ -694,8 +762,20 @@ def save_checks_in_installation( product_name: str = "dqx", assume_user: bool = True, ): + """Deprecated: Use `save_checks` with `InstallationChecksStorageConfig` instead. + + Save checks to the installation directory. + + :param checks: List of DQ rules (checks) to save. + :param run_config_name: Named run configuration to resolve installation paths and defaults. + :param method: Deprecated parameter; ignored. + :param product_name: Product/installation identifier (e.g., "dqx"). + :param assume_user: Whether to assume a per-user installation layout. + :return: None + """ warnings.warn( - "Use `save_checks` method instead. This method will be removed in future versions.", + "Use `save_checks` method with `InstallationChecksStorageConfig` instead. " + "This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) @@ -710,9 +790,17 @@ def save_checks_in_installation( def load_run_config( self, run_config_name: str = "default", assume_user: bool = True, product_name: str = "dqx" ) -> RunConfig: + """Deprecated: Use `RunConfigLoader.load_run_config` directly. + + Load a run configuration by name. This wrapper will be removed in a future version. + + :param run_config_name: Name of the run configuration to load. + :param assume_user: Whether to assume a per-user installation when resolving paths. + :param product_name: Product/installation identifier (e.g., "dqx"). + :return: Loaded `RunConfig` instance. + """ warnings.warn( - "Use `load_run_config` method from `config_loader.RunConfigLoader` class. " - "This method will be removed in future versions.", + "Use `RunConfigLoader.load_run_config` method instead. This method will be removed in future versions.", category=DeprecationWarning, stacklevel=2, ) From a5e62883f607be0920b42a7c60e3e7698686580f Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 12:12:19 +0200 Subject: [PATCH 19/37] fmt --- src/databricks/labs/dqx/engine.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index b8402394f..de13d7024 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -85,7 +85,8 @@ def apply_checks( if not DQEngineCore._all_are_dq_rules(checks): raise TypeError( - "All elements in the 'checks' list must be instances of DQRule. Use 'apply_checks_by_metadata' to pass checks as list of dicts instead." + "All elements in the 'checks' list must be instances of DQRule. " + "Use 'apply_checks_by_metadata' to pass checks as list of dicts instead." ) warning_checks = self._get_check_columns(checks, Criticality.WARN.value) @@ -117,7 +118,8 @@ def apply_checks_and_split( if not DQEngineCore._all_are_dq_rules(checks): raise TypeError( - "All elements in the 'checks' list must be instances of DQRule. Use 'apply_checks_by_metadata_and_split' to pass checks as list of dicts instead." + "All elements in the 'checks' list must be instances of DQRule. " + "Use 'apply_checks_by_metadata_and_split' to pass checks as list of dicts instead." ) checked_df = self.apply_checks(df, checks, ref_dfs) From 1446f6edf7ede793351b639f5d18cefa96292d54 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 12:13:43 +0200 Subject: [PATCH 20/37] Update src/databricks/labs/dqx/check_funcs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/databricks/labs/dqx/check_funcs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index df368c1e7..d219692f8 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -1348,7 +1348,10 @@ def _match_rows( """ Perform a null-safe join between two DataFrames based on primary key columns. Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols. - Use eq null safe join to ensure that: `1 == 1 matches; NULL <=> NULL matches; 1 <=> NULL` does not match + Use eq null safe join to ensure that: + - 1 == 1 matches + - NULL <=> NULL matches + - 1 <=> NULL does not match :param df: The input DataFrame to join. :param ref_df: The reference DataFrame to join against. From 6ac3d3cd582d341dbaff850305880c7152caf86a Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 12:33:40 +0200 Subject: [PATCH 21/37] removed lists from docstrings for cleaner api reference docs --- src/databricks/labs/dqx/base.py | 16 +++--- src/databricks/labs/dqx/check_funcs.py | 11 ++-- src/databricks/labs/dqx/engine.py | 72 +++++++++++++------------- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index 58911d9a2..eb5421928 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -88,10 +88,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check`: A check definition including check function and arguments to use. - - `name`: Optional name for the resulting column. Auto-generated if not provided. - - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + `check`: A check definition including check function and arguments to use; + `name`: Optional name for the resulting column. Auto-generated if not provided; + `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -110,10 +110,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check`: A check definition including check function and arguments to use. - - `name`: Optional name for the resulting column. Auto-generated if not provided. - - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + `check`: A check definition including check function and arguments to use; + `name`: Optional name for the resulting column. Auto-generated if not provided; + `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index d219692f8..ce1361c6e 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -32,8 +32,8 @@ def make_condition(condition: Column, message: Column | str, alias: str) -> Colu """Helper function to create a condition column. :param condition: condition expression. - Pass the check if the condition evaluates to False. - Fail the check if condition evaluates to True. + Pass the check if the condition evaluates to False; + Fail the check if condition evaluates to True; :param message: message to output - it could be either `Column` object, or string constant :param alias: name for the resulting column :return: an instance of `Column` type, that either returns string if condition is evaluated to `true`, @@ -1349,9 +1349,9 @@ def _match_rows( Perform a null-safe join between two DataFrames based on primary key columns. Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols. Use eq null safe join to ensure that: - - 1 == 1 matches - - NULL <=> NULL matches - - 1 <=> NULL does not match + 1 == 1 matches; + NULL <=> NULL matches; + 1 <=> NULL does not match; :param df: The input DataFrame to join. :param ref_df: The reference DataFrame to join against. @@ -1472,6 +1472,7 @@ def _add_compare_condition( This function adds a new column (`condition_col`) to the DataFrame, which contains structured information about mismatched records. The mismatches are determined based on the presence of missing rows, extra rows, and differences in specified columns. + :param df: The input DataFrame containing the comparison results. :param condition_col: The name of the column to add, which will store mismatch information. :param row_missing_col: The name of the column indicating missing rows. diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index de13d7024..57047bf32 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -140,10 +140,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check`: A check definition including check function and arguments to use. - - `name`: Optional name for the resulting column. Auto-generated if not provided. - - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + `check`: A check definition including check function and arguments to use; + `name`: Optional name for the resulting column. Auto-generated if not provided; + `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame with errors and warnings result columns. @@ -164,10 +164,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check`: A check definition including check function and arguments to use. - - `name`: Optional name for the resulting column. Auto-generated if not provided. - - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + `check`: A check definition including check function and arguments to use; + `name`: Optional name for the resulting column. Auto-generated if not provided; + `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -385,10 +385,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check`: A check definition including check function and arguments to use. - - `name`: Optional name for the resulting column. Auto-generated if not provided. - - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + `check`: A check definition including check function and arguments to use; + `name`: Optional name for the resulting column. Auto-generated if not provided; + `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame with errors and warnings result columns. @@ -407,10 +407,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check`: A check definition including check function and arguments to use. - - `name`: Optional name for the resulting column. Auto-generated if not provided. - - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + `check`: A check definition including check function and arguments to use; + `name`: Optional name for the resulting column. Auto-generated if not provided; + `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -429,8 +429,8 @@ def apply_checks_and_save_in_table( Apply data quality checks to input data and save results. If `quarantine_config` is provided, split the data into valid and invalid records: - - valid records are written using `output_config` - - invalid records are written using `quarantine_config` + valid records are written using `output_config`; + invalid records are written using `quarantine_config`; If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. @@ -466,15 +466,15 @@ def apply_checks_by_metadata_and_save_in_table( Apply metadata-defined data quality checks to input data and save results. If `quarantine_config` is provided, split the data into valid and invalid records: - - valid records are written using `output_config` - - invalid records are written using `quarantine_config` + valid records are written using `output_config`; + invalid records are written using `quarantine_config`; If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. :param checks: List of dictionaries describing checks. Each item must include: - - `check`: Check definition (function name plus arguments) to evaluate. - - `name` (optional): Result column name; auto-generated if omitted. - - `criticality` (optional): One of `error` (only in invalid output) or `warn` (also appears in valid output). + `check`: Check definition (function name plus arguments) to evaluate; + `name` (optional): Result column name; auto-generated if omitted; + `criticality` (optional): One of `error` (only in invalid output) or `warn` (also appears in valid output); :param input_config: Input configuration (e.g., table/view or file location and read options). :param output_config: Output configuration (e.g., table name, mode, and write options). :param quarantine_config: Optional configuration for writing invalid records. @@ -544,9 +544,9 @@ def save_results_in_table( """Persist result DataFrames using explicit configs or the named run configuration. Behavior: - - If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`. - - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`. - - A write occurs only when both a DataFrame and its corresponding config are available. + If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`; + If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`; + A write occurs only when both a DataFrame and its corresponding config are available; :param output_df: DataFrame with valid rows to be saved (optional). :param quarantine_df: DataFrame with invalid rows to be saved (optional). @@ -579,11 +579,11 @@ def load_checks(self, config: BaseChecksStorageConfig) -> list[dict]: of checks (as dictionaries) ready for `apply_checks_by_metadata`. Supported storage configurations include, for example: - - `FileChecksStorageConfig` (local file) - - `WorkspaceFileChecksStorageConfig` (Databricks workspace file) - - `TableChecksStorageConfig` (table-backed storage) - - `InstallationChecksStorageConfig` (installation directory) - - `VolumeFileChecksStorageConfig` (Unity Catalog volume file) + `FileChecksStorageConfig` (local file); + `WorkspaceFileChecksStorageConfig` (Databricks workspace file); + `TableChecksStorageConfig` (table-backed storage); + `InstallationChecksStorageConfig` (installation directory); + `VolumeFileChecksStorageConfig` (Unity Catalog volume file); :param config: Configuration object describing the storage backend. :return: List of DQ rules (checks) represented as dictionaries. @@ -601,11 +601,11 @@ def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> No such as `mode` where applicable. Supported storage configurations include, for example: - - `FileChecksStorageConfig` (local file) - - `WorkspaceFileChecksStorageConfig` (Databricks workspace file) - - `TableChecksStorageConfig` (table-backed storage) - - `InstallationChecksStorageConfig` (installation directory) - - `VolumeFileChecksStorageConfig` (Unity Catalog volume file) + `FileChecksStorageConfig` (local file); + `WorkspaceFileChecksStorageConfig` (Databricks workspace file); + `TableChecksStorageConfig` (table-backed storage); + `InstallationChecksStorageConfig` (installation directory); + `VolumeFileChecksStorageConfig` (Unity Catalog volume file); :param checks: List of DQ rules (checks) to save (as dictionaries). :param config: Configuration object describing the storage backend and write options. From 4958b7bcde74d5ddd415e3d857cbea43f0676e58 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 12:38:11 +0200 Subject: [PATCH 22/37] bumped black version to avoid compatibility issue with docspec-python --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 342111fc6..7d89e2eea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ path = "src/databricks/labs/dqx/__about__.py" [tool.hatch.envs.default] features = ["pii"] dependencies = [ - "black~=24.3.0", + "black~=24.8.0", "chispa~=0.10.1", "coverage[toml]~=7.4.4", "databricks-labs-pylint~=0.5", From 93f64acba846fd4978affc07545c48c47d3a3131 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 12:45:47 +0200 Subject: [PATCH 23/37] added docs clean command --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index af69e1a60..4fa704cd7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: clean dev lint fmt test integration coverage e2e -clean: +clean: docs-clean rm -fr .venv clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml rm -fr **/*.pyc @@ -44,4 +44,9 @@ docs-install: docs-serve: docs-build hatch run docs:pydoc-markdown - yarn --cwd docs/dqx serve \ No newline at end of file + yarn --cwd docs/dqx serve + +docs-clean: + rm -rf docs/dqx/build + rm -rf docs/dqx/.docusaurus docs/dqx/.cache + rm -rf docs/dqx/docs/reference/api/* From a360b23d34b9533ea54217fc1248a70794abe404 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 13:02:59 +0200 Subject: [PATCH 24/37] updated docstrings --- src/databricks/labs/dqx/base.py | 16 +++--- src/databricks/labs/dqx/check_funcs.py | 4 +- src/databricks/labs/dqx/engine.py | 73 ++++++++++++++------------ 3 files changed, 48 insertions(+), 45 deletions(-) diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index eb5421928..8382269e9 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -88,10 +88,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - `check`: A check definition including check function and arguments to use; - `name`: Optional name for the resulting column. Auto-generated if not provided; - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -110,10 +110,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - `check`: A check definition including check function and arguments to use; - `name`: Optional name for the resulting column. Auto-generated if not provided; - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index ce1361c6e..5b37ab361 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -32,8 +32,8 @@ def make_condition(condition: Column, message: Column | str, alias: str) -> Colu """Helper function to create a condition column. :param condition: condition expression. - Pass the check if the condition evaluates to False; - Fail the check if condition evaluates to True; + - Pass the check if the condition evaluates to False + - Fail the check if condition evaluates to True :param message: message to output - it could be either `Column` object, or string constant :param alias: name for the resulting column :return: an instance of `Column` type, that either returns string if condition is evaluated to `true`, diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 57047bf32..ca6a02c41 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -140,10 +140,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - `check`: A check definition including check function and arguments to use; - `name`: Optional name for the resulting column. Auto-generated if not provided; - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame with errors and warnings result columns. @@ -164,10 +164,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - `check`: A check definition including check function and arguments to use; - `name`: Optional name for the resulting column. Auto-generated if not provided; - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -385,9 +385,9 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - `check`: A check definition including check function and arguments to use; - `name`: Optional name for the resulting column. Auto-generated if not provided; - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. @@ -407,9 +407,9 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - `check`: A check definition including check function and arguments to use; - `name`: Optional name for the resulting column. Auto-generated if not provided; - `criticality`: Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. @@ -429,8 +429,9 @@ def apply_checks_and_save_in_table( Apply data quality checks to input data and save results. If `quarantine_config` is provided, split the data into valid and invalid records: - valid records are written using `output_config`; - invalid records are written using `quarantine_config`; + + - valid records are written using `output_config`; + - invalid records are written using `quarantine_config`. If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. @@ -466,15 +467,16 @@ def apply_checks_by_metadata_and_save_in_table( Apply metadata-defined data quality checks to input data and save results. If `quarantine_config` is provided, split the data into valid and invalid records: - valid records are written using `output_config`; - invalid records are written using `quarantine_config`; + + - valid records are written using `output_config`; + - invalid records are written using `quarantine_config`. If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. - :param checks: List of dictionaries describing checks. Each item must include: - `check`: Check definition (function name plus arguments) to evaluate; - `name` (optional): Result column name; auto-generated if omitted; - `criticality` (optional): One of `error` (only in invalid output) or `warn` (also appears in valid output); + :param checks: List of dictionaries describing checks. Each item must include + - `check` - Check definition (function name plus arguments) to evaluate. + - `name` (optional) - Result column name; auto-generated if omitted. + - `criticality` (optional) - One of `error` (only in invalid output) or `warn` (also appears in valid output). :param input_config: Input configuration (e.g., table/view or file location and read options). :param output_config: Output configuration (e.g., table name, mode, and write options). :param quarantine_config: Optional configuration for writing invalid records. @@ -544,9 +546,10 @@ def save_results_in_table( """Persist result DataFrames using explicit configs or the named run configuration. Behavior: - If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`; - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`; - A write occurs only when both a DataFrame and its corresponding config are available; + + - If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`. + - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`. + - A write occurs only when both a DataFrame and its corresponding config are available. :param output_df: DataFrame with valid rows to be saved (optional). :param quarantine_df: DataFrame with invalid rows to be saved (optional). @@ -579,11 +582,11 @@ def load_checks(self, config: BaseChecksStorageConfig) -> list[dict]: of checks (as dictionaries) ready for `apply_checks_by_metadata`. Supported storage configurations include, for example: - `FileChecksStorageConfig` (local file); - `WorkspaceFileChecksStorageConfig` (Databricks workspace file); - `TableChecksStorageConfig` (table-backed storage); - `InstallationChecksStorageConfig` (installation directory); - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); + - `FileChecksStorageConfig` (local file); + - `WorkspaceFileChecksStorageConfig` (Databricks workspace file); + - `TableChecksStorageConfig` (table-backed storage); + - `InstallationChecksStorageConfig` (installation directory); + - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); :param config: Configuration object describing the storage backend. :return: List of DQ rules (checks) represented as dictionaries. @@ -601,11 +604,11 @@ def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> No such as `mode` where applicable. Supported storage configurations include, for example: - `FileChecksStorageConfig` (local file); - `WorkspaceFileChecksStorageConfig` (Databricks workspace file); - `TableChecksStorageConfig` (table-backed storage); - `InstallationChecksStorageConfig` (installation directory); - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); + - `FileChecksStorageConfig` (local file); + - `WorkspaceFileChecksStorageConfig` (Databricks workspace file); + - `TableChecksStorageConfig` (table-backed storage); + - `InstallationChecksStorageConfig` (installation directory); + - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); :param checks: List of DQ rules (checks) to save (as dictionaries). :param config: Configuration object describing the storage backend and write options. From 3860d3530ba412c201bc497582e0d2806f84a104 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 13:04:50 +0200 Subject: [PATCH 25/37] updated docstrings --- src/databricks/labs/dqx/engine.py | 41 ++++++++++++++----------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index ca6a02c41..3f725acbf 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -140,10 +140,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame with errors and warnings result columns. @@ -164,10 +164,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -385,10 +385,10 @@ def apply_checks_by_metadata( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame with errors and warnings result columns. @@ -407,10 +407,10 @@ def apply_checks_by_metadata_and_split( :param df: Input DataFrame to check. :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). :param ref_dfs: Optional reference DataFrames to use in the checks. :return: DataFrame that includes errors and warnings result columns. @@ -429,7 +429,6 @@ def apply_checks_and_save_in_table( Apply data quality checks to input data and save results. If `quarantine_config` is provided, split the data into valid and invalid records: - - valid records are written using `output_config`; - invalid records are written using `quarantine_config`. @@ -467,16 +466,15 @@ def apply_checks_by_metadata_and_save_in_table( Apply metadata-defined data quality checks to input data and save results. If `quarantine_config` is provided, split the data into valid and invalid records: - - valid records are written using `output_config`; - invalid records are written using `quarantine_config`. If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. :param checks: List of dictionaries describing checks. Each item must include - - `check` - Check definition (function name plus arguments) to evaluate. - - `name` (optional) - Result column name; auto-generated if omitted. - - `criticality` (optional) - One of `error` (only in invalid output) or `warn` (also appears in valid output). + - `check` - Check definition (function name plus arguments) to evaluate. + - `name` (optional) - Result column name; auto-generated if omitted. + - `criticality` (optional) - One of `error` (only in invalid output) or `warn` (also appears in valid output). :param input_config: Input configuration (e.g., table/view or file location and read options). :param output_config: Output configuration (e.g., table name, mode, and write options). :param quarantine_config: Optional configuration for writing invalid records. @@ -546,7 +544,6 @@ def save_results_in_table( """Persist result DataFrames using explicit configs or the named run configuration. Behavior: - - If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`. - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`. - A write occurs only when both a DataFrame and its corresponding config are available. From 35fda1c9995cc414042c1ab919fc11656d774e86 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 13:22:55 +0200 Subject: [PATCH 26/37] updated docstrings and clean docs --- Makefile | 2 +- src/databricks/labs/dqx/engine.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4fa704cd7..b33b5513a 100644 --- a/Makefile +++ b/Makefile @@ -49,4 +49,4 @@ docs-serve: docs-build docs-clean: rm -rf docs/dqx/build rm -rf docs/dqx/.docusaurus docs/dqx/.cache - rm -rf docs/dqx/docs/reference/api/* + find docs/dqx/docs/reference/api -mindepth 1 -not -name 'index.mdx' -exec rm -rf {} + diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 3f725acbf..0f11742e6 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -471,14 +471,14 @@ def apply_checks_by_metadata_and_save_in_table( If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. - :param checks: List of dictionaries describing checks. Each item must include - - `check` - Check definition (function name plus arguments) to evaluate. - - `name` (optional) - Result column name; auto-generated if omitted. - - `criticality` (optional) - One of `error` (only in invalid output) or `warn` (also appears in valid output). + :param checks: List of dicts describing checks. Each item must include 'check' (function name plus + arguments) and may include 'name' (result column; auto-generated if omitted) and 'criticality' + ('error' or 'warn'). :param input_config: Input configuration (e.g., table/view or file location and read options). :param output_config: Output configuration (e.g., table name, mode, and write options). :param quarantine_config: Optional configuration for writing invalid records. - :param custom_check_functions: Optional mapping of custom check function names to callables/modules (e.g., globals()). + :param custom_check_functions: Optional mapping of custom check function names + to callables/modules (e.g., globals()). :param ref_dfs: Optional reference DataFrames used by checks. """ # Read data from the specified table From 4c174b14f5ac228f9a43237f571bcb2a2f534c14 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 14:42:54 +0200 Subject: [PATCH 27/37] updated docstring guidelines --- docs/dqx/docs/dev/contributing.mdx | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 717475cff..917425ea8 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -310,27 +310,29 @@ hatch run docs:pydoc-markdown The API docs are generated in the `docs/dqx/docs/reference/api` directory. -To write API docs, use the following format in the source code: +Use [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) format for the docstrings, +so that they are rendered correctly in the API docs. +Example: ```python - def method(self, arg1: str, arg2: int) -> str: """ Short method description in Markdown format. Very long method description in Markdown format. - :param arg1: Argument 1 description - :param arg2: Argument 2 description - :return: Return value description - - Please note that double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. - If you need a code example, use triple backticks, e.g.: - ```python - print("Hello, world!") - ``` + Args: + arg1 (str): Argument 1 description. + arg2 (int): Argument 2 description. + Returns: + str: Return value description. """ return "Hello, world!" ``` +Please note that double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. +If you need a code example, use triple backticks, e.g.: +```python +print("Hello, world!") +``` From 10a477e05cbf1082d1c4cfa6f1b2a85694063adc Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 15 Aug 2025 17:57:13 +0200 Subject: [PATCH 28/37] updated docstrings to use google style --- docs/dqx/docs/dev/contributing.mdx | 2 +- src/databricks/labs/dqx/base.py | 105 ++- src/databricks/labs/dqx/check_funcs.py | 706 +++++++++++------- src/databricks/labs/dqx/checks_resolver.py | 11 +- src/databricks/labs/dqx/checks_serializer.py | 81 +- src/databricks/labs/dqx/checks_storage.py | 105 ++- src/databricks/labs/dqx/checks_validator.py | 63 +- src/databricks/labs/dqx/cli.py | 43 +- src/databricks/labs/dqx/config.py | 43 +- src/databricks/labs/dqx/config_loader.py | 17 +- .../labs/dqx/contexts/application.py | 10 +- src/databricks/labs/dqx/contexts/workflows.py | 7 +- src/databricks/labs/dqx/contexts/workspace.py | 5 +- src/databricks/labs/dqx/engine.py | 398 ++++++---- src/databricks/labs/dqx/executor.py | 30 +- src/databricks/labs/dqx/installer/install.py | 66 +- .../labs/dqx/installer/workflow_task.py | 6 +- .../dqx/llm/extract_yaml_checks_examples.py | 35 +- src/databricks/labs/dqx/llm/utils.py | 19 +- .../labs/dqx/pii/pii_detection_funcs.py | 42 +- src/databricks/labs/dqx/profiler/common.py | 18 +- .../labs/dqx/profiler/dlt_generator.py | 73 +- src/databricks/labs/dqx/profiler/generator.py | 53 +- src/databricks/labs/dqx/profiler/profiler.py | 279 ++++--- src/databricks/labs/dqx/profiler/runner.py | 18 +- src/databricks/labs/dqx/profiler/workflow.py | 3 +- src/databricks/labs/dqx/rule.py | 24 +- src/databricks/labs/dqx/utils.py | 94 ++- tests/e2e/test_pii_detection_checks.py | 6 +- tests/e2e/test_run_demos.py | 6 +- tests/unit/test_build_rules.py | 7 +- 31 files changed, 1490 insertions(+), 885 deletions(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 917425ea8..649d6badf 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -45,7 +45,7 @@ Here are the example steps to submit your first contribution: 3. `git checkout main` (or `gcm` if you're using [ohmyzsh](https://ohmyz.sh/)). 4. `git pull` (or `gl` if you're using [ohmyzsh](https://ohmyz.sh/)). 5. `git checkout -b FEATURENAME` (or `gcb FEATURENAME` if you're using [ohmyzsh](https://ohmyz.sh/)). -6. .. do the work +6. .. do the work. Please make sure to document all your public methods, functions and classes using [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). We generate API documentation automatically using the docstrings. 7. `make fmt` (Note: If you have an issue with `make fmt`, ensure your IDE folder is ignored in .gitignore. Already added for .idea/ and .cursor/) 8. `make lint` 9. .. fix if any issues are reported diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index 8382269e9..ceba036e3 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -55,10 +55,13 @@ def apply_checks( ) -> DataFrame: """Apply data quality checks to the given DataFrame. - :param df: Input DataFrame to check. - :param checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame that includes errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame that includes errors and warnings result columns. """ @abc.abstractmethod @@ -68,11 +71,14 @@ def apply_checks_and_split( """Apply data quality checks to the given DataFrame and split the results into two DataFrames ("good" and "bad"). - :param df: Input DataFrame to check. - :param checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and - "bad" (rows with errors or warnings and the corresponding result columns). + Args: + df: Input DataFrame to check. + checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ @abc.abstractmethod @@ -86,15 +92,18 @@ def apply_checks_by_metadata( """ Apply data quality checks defined as metadata to the given DataFrame. - :param df: Input DataFrame to check. - :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame that includes errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame that includes errors and warnings result columns. """ @abc.abstractmethod @@ -108,16 +117,19 @@ def apply_checks_by_metadata_and_split( """Apply data quality checks defined as metadata to the given DataFrame and split the results into two DataFrames ("good" and "bad"). - :param df: Input DataFrame to check. - :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and - "bad" (rows with errors or warnings and the corresponding result columns). + Args: + df: Input DataFrame to check. + checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ @staticmethod @@ -133,10 +145,13 @@ def validate_checks( This method validates the presence of required keys, the existence and callability of functions, and the types of arguments passed to those functions. - :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param validate_custom_check_functions: If True, validate custom check functions. - :return: ChecksValidationStatus indicating the validation result. + Args: + checks: List of checks to apply to the DataFrame. Each check should be a dictionary. + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + validate_custom_check_functions: If True, validate custom check functions. + + Returns: + ChecksValidationStatus indicating the validation result. """ @abc.abstractmethod @@ -144,8 +159,11 @@ def get_invalid(self, df: DataFrame) -> DataFrame: """ Return records that violate data quality checks (rows with warnings or errors). - :param df: Input DataFrame. - :return: DataFrame with rows that have errors or warnings and the corresponding result columns. + Args: + df: Input DataFrame. + + Returns: + DataFrame with rows that have errors or warnings and the corresponding result columns. """ @abc.abstractmethod @@ -153,8 +171,11 @@ def get_valid(self, df: DataFrame) -> DataFrame: """ Return records that do not violate data quality checks (rows with warnings but no errors). - :param df: Input DataFrame. - :return: DataFrame with warning rows but without the results columns. + Args: + df: Input DataFrame. + + Returns: + DataFrame with warning rows but without the results columns. """ @staticmethod @@ -165,8 +186,11 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: The returned checks can be used as input to `apply_checks_by_metadata`. - :param filepath: Path to a file containing checks definitions. - :return: List of DQ rules (checks). + Args: + filepath: Path to a file containing checks definitions. + + Returns: + List of DQ rules (checks). """ @staticmethod @@ -175,6 +199,7 @@ def save_checks_in_local_file(checks: list[dict], filepath: str): """ Save DQ rules (checks) to a local YAML or JSON file. - :param checks: List of DQ rules (checks) to save. - :param filepath: Path to a file where the checks definitions will be saved. + Args: + checks: List of DQ rules (checks) to save. + filepath: Path to a file where the checks definitions will be saved. """ diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 5b37ab361..03ac0fc33 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -31,13 +31,16 @@ class DQPattern(Enum): def make_condition(condition: Column, message: Column | str, alias: str) -> Column: """Helper function to create a condition column. - :param condition: condition expression. - - Pass the check if the condition evaluates to False - - Fail the check if condition evaluates to True - :param message: message to output - it could be either `Column` object, or string constant - :param alias: name for the resulting column - :return: an instance of `Column` type, that either returns string if condition is evaluated to `true`, - or `null` if condition is evaluated to `false` + Args: + condition: condition expression. + - Pass the check if the condition evaluates to False + - Fail the check if condition evaluates to True + message: message to output - it could be either `Column` object, or string constant + alias: name for the resulting column + + Returns: + an instance of `Column` type, that either returns string if condition is evaluated to `true`, + or `null` if condition is evaluated to `false` """ if isinstance(message, str): msg_col = F.lit(message) @@ -50,9 +53,12 @@ def make_condition(condition: Column, message: Column | str, alias: str) -> Colu def matches_pattern(column: str | Column, pattern: DQPattern) -> Column: """Checks whether the values in the input column match a given pattern. - :param column: column to check; can be a string column name or a column expression - :param pattern: pattern to match against - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + pattern: pattern to match against + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) condition = ~col_expr.rlike(pattern.value) @@ -71,9 +77,12 @@ def matches_pattern(column: str | Column, pattern: DQPattern) -> Column: def is_not_null_and_not_empty(column: str | Column, trim_strings: bool | None = False) -> Column: """Checks whether the values in the input column are not null and not empty. - :param column: column to check; can be a string column name or a column expression - :param trim_strings: boolean flag to trim spaces from strings - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + trim_strings: boolean flag to trim spaces from strings + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) if trim_strings: @@ -88,8 +97,11 @@ def is_not_null_and_not_empty(column: str | Column, trim_strings: bool | None = def is_not_empty(column: str | Column) -> Column: """Checks whether the values in the input column are not empty (but may be null). - :param column: column to check; can be a string column name or a column expression - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) condition = col_expr.cast("string") == F.lit("") @@ -100,8 +112,11 @@ def is_not_empty(column: str | Column) -> Column: def is_not_null(column: str | Column) -> Column: """Checks whether the values in the input column are not null. - :param column: column to check; can be a string column name or a column expression - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) return make_condition(col_expr.isNull(), f"Column '{col_expr_str}' value is null", f"{col_str_norm}_is_null") @@ -111,9 +126,12 @@ def is_not_null(column: str | Column) -> Column: def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column: """Checks whether the values in the input column are not null and present in the list of allowed values. - :param column: column to check; can be a string column name or a column expression - :param allowed: list of allowed values (actual values or Column objects) - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + allowed: list of allowed values (actual values or Column objects) + + Returns: + Column object for condition """ if not allowed: raise ValueError("allowed list is not provided.") @@ -140,9 +158,12 @@ def is_in_list(column: str | Column, allowed: list) -> Column: """Checks whether the values in the input column are present in the list of allowed values (null values are allowed). - :param column: column to check; can be a string column name or a column expression - :param allowed: list of allowed values (actual values or Column objects) - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + allowed: list of allowed values (actual values or Column objects) + + Returns: + Column object for condition """ if not allowed: raise ValueError("allowed list is not provided.") @@ -174,13 +195,16 @@ def sql_expression( ) -> Column: """Checks whether the condition provided as an SQL expression is met. - :param expression: SQL expression. Fail if expression evaluates to True, pass if it evaluates to False. - :param msg: optional message of the `Column` type, automatically generated if None - :param name: optional name of the resulting column, automatically generated if None - :param negate: if the condition should be negated (true) or not. For example, "col is not null" will mark null - values as "bad". Although sometimes it's easier to specify it other way around "col is null" + negate set to False - :param columns: optional list of columns to be used for reporting. Unused in the actual logic. - :return: new Column + Args: + expression: SQL expression. Fail if expression evaluates to True, pass if it evaluates to False. + msg: optional message of the `Column` type, automatically generated if None + name: optional name of the resulting column, automatically generated if None + negate: if the condition should be negated (true) or not. For example, "col is not null" will mark null + values as "bad". Although sometimes it's easier to specify it other way around "col is null" + negate set to False + columns: optional list of columns to be used for reporting. Unused in the actual logic. + + Returns: + new Column """ expr_col = F.expr(expression) expr_msg = expression @@ -208,12 +232,15 @@ def is_older_than_col2_for_n_days( ) -> Column: """Checks whether the values in one input column are at least N days older than the values in another column. - :param column1: first column to check; can be a string column name or a column expression - :param column2: second column to check; can be a string column name or a column expression - :param days: number of days - :param negate: if the condition should be negated (true) or not; if negated, the check will fail when values in the - first column are at least N days older than values in the second column - :return: new Column + Args: + column1: first column to check; can be a string column name or a column expression + column2: second column to check; can be a string column name or a column expression + days: number of days + negate: if the condition should be negated (true) or not; if negated, the check will fail when values in the + first column are at least N days older than values in the second column + + Returns: + new Column """ col_str_norm1, col_expr_str1, col_expr1 = _get_normalized_column_and_expr(column1) col_str_norm2, col_expr_str2, col_expr2 = _get_normalized_column_and_expr(column2) @@ -255,12 +282,15 @@ def is_older_than_n_days( ) -> Column: """Checks whether the values in the input column are at least N days older than the current date. - :param column: column to check; can be a string column name or a column expression - :param days: number of days - :param curr_date: (optional) set current date - :param negate: if the condition should be negated (true) or not; if negated, the check will fail when values in the - first column are at least N days older than values in the second column - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + days: number of days + curr_date: (optional) set current date + negate: if the condition should be negated (true) or not; if negated, the check will fail when values in the + first column are at least N days older than values in the second column + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) if curr_date is None: @@ -302,10 +332,13 @@ def is_not_in_future(column: str | Column, offset: int = 0, curr_timestamp: Colu """Checks whether the values in the input column contain a timestamp that is not in the future, where 'future' is defined as current_timestamp + offset (in seconds). - :param column: column to check; can be a string column name or a column expression - :param offset: offset (in seconds) to add to the current timestamp at time of execution - :param curr_timestamp: (optional) set current timestamp - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + offset: offset (in seconds) to add to the current timestamp at time of execution + curr_timestamp: (optional) set current timestamp + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) if curr_timestamp is None: @@ -334,10 +367,13 @@ def is_not_in_near_future(column: str | Column, offset: int = 0, curr_timestamp: where 'near future' is defined as greater than the current timestamp but less than the current_timestamp + offset (in seconds). - :param column: column to check; can be a string column name or a column expression - :param offset: offset (in seconds) to add to the current timestamp at time of execution - :param curr_timestamp: (optional) set current timestamp - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + offset: offset (in seconds) to add to the current timestamp at time of execution + curr_timestamp: (optional) set current timestamp + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) if curr_timestamp is None: @@ -368,9 +404,12 @@ def is_not_less_than( ) -> Column: """Checks whether the values in the input column are not less than the provided limit. - :param column: column to check; can be a string column name or a column expression - :param limit: limit to use in the condition as number, date, timestamp, column name or sql expression - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + limit: limit to use in the condition as number, date, timestamp, column name or sql expression + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) limit_expr = _get_limit_expr(limit) @@ -395,9 +434,12 @@ def is_not_greater_than( ) -> Column: """Checks whether the values in the input column are not greater than the provided limit. - :param column: column to check; can be a string column name or a column expression - :param limit: limit to use in the condition as number, date, timestamp, column name or sql expression - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + limit: limit to use in the condition as number, date, timestamp, column name or sql expression + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) limit_expr = _get_limit_expr(limit) @@ -424,10 +466,13 @@ def is_in_range( ) -> Column: """Checks whether the values in the input column are in the provided limits (inclusive of both boundaries). - :param column: column to check; can be a string column name or a column expression - :param min_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression - :param max_limit: max limit to use in the condition as number, date, timestamp, column name or sql expression - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + min_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression + max_limit: max limit to use in the condition as number, date, timestamp, column name or sql expression + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) min_limit_expr = _get_limit_expr(min_limit) @@ -459,10 +504,13 @@ def is_not_in_range( ) -> Column: """Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries). - :param column: column to check; can be a string column name or a column expression - :param min_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression - :param max_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression - :return: new Column + Args: + column: column to check; can be a string column name or a column expression + min_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression + max_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression + + Returns: + new Column """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) min_limit_expr = _get_limit_expr(min_limit) @@ -490,10 +538,13 @@ def is_not_in_range( def regex_match(column: str | Column, regex: str, negate: bool = False) -> Column: """Checks whether the values in the input column matches a given regex. - :param column: column to check; can be a string column name or a column expression - :param regex: regex to check - :param negate: if the condition should be negated (true) or not - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + regex: regex to check + negate: if the condition should be negated (true) or not + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) if negate: @@ -510,8 +561,11 @@ def regex_match(column: str | Column, regex: str, negate: bool = False) -> Colum def is_not_null_and_not_empty_array(column: str | Column) -> Column: """Checks whether the values in the array input column are not null and not empty. - :param column: column to check; can be a string column name or a column expression - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) condition = col_expr.isNull() | (F.size(col_expr) == 0) @@ -524,9 +578,12 @@ def is_not_null_and_not_empty_array(column: str | Column) -> Column: def is_valid_date(column: str | Column, date_format: str | None = None) -> Column: """Checks whether the values in the input column have valid date formats. - :param column: column to check; can be a string column name or a column expression - :param date_format: date format (e.g. 'yyyy-mm-dd') - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + date_format: date format (e.g. 'yyyy-mm-dd') + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) date_col = F.try_to_timestamp(col_expr) if date_format is None else F.try_to_timestamp(col_expr, F.lit(date_format)) @@ -545,9 +602,12 @@ def is_valid_date(column: str | Column, date_format: str | None = None) -> Colum def is_valid_timestamp(column: str | Column, timestamp_format: str | None = None) -> Column: """Checks whether the values in the input column have valid timestamp formats. - :param column: column to check; can be a string column name or a column expression - :param timestamp_format: timestamp format (e.g. 'yyyy-mm-dd HH:mm:ss') - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + timestamp_format: timestamp format (e.g. 'yyyy-mm-dd HH:mm:ss') + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) ts_col = ( @@ -570,8 +630,11 @@ def is_valid_timestamp(column: str | Column, timestamp_format: str | None = None def is_valid_ipv4_address(column: str | Column) -> Column: """Checks whether the values in the input column have valid IPv4 address formats. - :param column: column to check; can be a string column name or a column expression - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression + + Returns: + Column object for condition """ return matches_pattern(column, DQPattern.IPV4_ADDRESS) @@ -581,11 +644,15 @@ def is_ipv4_address_in_cidr(column: str | Column, cidr_block: str) -> Column: """ Checks if an IP column value falls within the given CIDR block. - :param column: column to check; can be a string column name or a column expression - :param cidr_block: CIDR block string (e.g., '192.168.1.0/24') - :raises ValueError: If cidr_block is not a valid string in CIDR notation. + Args: + column: column to check; can be a string column name or a column expression + cidr_block: CIDR block string (e.g., '192.168.1.0/24') - :return: Column object for condition + Raises: + ValueError: If cidr_block is not a valid string in CIDR notation. + + Returns: + Column object for condition """ if not cidr_block: @@ -626,10 +693,13 @@ def is_data_fresh( This is useful for identifying stale data due to delayed pipelines and helps catch upstream issues early. - :param column: column to check; can be a string column name or a column expression containing timestamp values - :param max_age_minutes: maximum age in minutes before data is considered stale - :param base_timestamp: (optional) set base timestamp column from which the stale check is calculated, if not provided uses current_timestamp() - :return: Column object for condition + Args: + column: column to check; can be a string column name or a column expression containing timestamp values + max_age_minutes: maximum age in minutes before data is considered stale + base_timestamp: (optional) set base timestamp column from which the stale check is calculated, if not provided uses current_timestamp() + + Returns: + Column object for condition """ col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) if base_timestamp is None: @@ -671,13 +741,16 @@ def is_unique( In streaming, uniqueness is validated within individual micro-batches only. - :param columns: List of column names (str) or Spark Column expressions to validate for uniqueness. - :param nulls_distinct: Whether NULLs are treated as distinct (default: True). - :param row_filter: Optional SQL expression for filtering rows before checking uniqueness. - Auto-injected from the check filter. - :return: A tuple of: - - A Spark Column representing the condition for uniqueness violations. - - A closure that applies the uniqueness check and adds the necessary condition/count columns. + Args: + columns: List of column names (str) or Spark Column expressions to validate for uniqueness. + nulls_distinct: Whether NULLs are treated as distinct (default: True). + row_filter: Optional SQL expression for filtering rows before checking uniqueness. + Auto-injected from the check filter. + + Returns: + A tuple of: + - A Spark Column representing the condition for uniqueness violations. + - A closure that applies the uniqueness check and adds the necessary condition/count columns. """ if len(columns) == 1: single_key = columns[0] @@ -698,8 +771,11 @@ def apply(df: DataFrame) -> DataFrame: Adds columns indicating whether the row violates uniqueness, and how many duplicates exist. The condition is applied during check evaluation to flag duplicates. - :param df: The input DataFrame to validate for uniqueness. - :return: The DataFrame with additional condition and count columns for uniqueness validation. + Args: + df: The input DataFrame to validate for uniqueness. + + Returns: + The DataFrame with additional condition and count columns for uniqueness validation. """ window_count_col = f"__window_count_{col_str_norm}_{unique_str}" @@ -765,17 +841,20 @@ def foreign_key( NULL values in the foreign key columns are ignored (SQL ANSI behavior). - :param columns: List of column names (str) or Column expressions in the dataset (foreign key). - :param ref_columns: List of column names (str) or Column expressions in the reference dataset. - :param ref_df_name: Name of the reference DataFrame (used when passing DataFrames directly). - :param ref_table: Name of the reference table (used when reading from catalog). - :param row_filter: Optional SQL expression for filtering rows before checking the foreign key. - Auto-injected from the check filter. - :param negate: If True, the condition is negated (i.e., the check fails when the foreign key values exist in the - reference DataFrame/Table). If False, the check fails when the foreign key values do not exist in the reference. - :return: A tuple of: - - A Spark Column representing the condition for foreign key violations. - - A closure that applies the foreign key validation by joining against the reference. + Args: + columns: List of column names (str) or Column expressions in the dataset (foreign key). + ref_columns: List of column names (str) or Column expressions in the reference dataset. + ref_df_name: Name of the reference DataFrame (used when passing DataFrames directly). + ref_table: Name of the reference table (used when reading from catalog). + row_filter: Optional SQL expression for filtering rows before checking the foreign key. + Auto-injected from the check filter. + negate: If True, the condition is negated (i.e., the check fails when the foreign key values exist in the + reference DataFrame/Table). If False, the check fails when the foreign key values do not exist in the reference. + + Returns: + A tuple of: + - A Spark Column representing the condition for foreign key violations. + - A closure that applies the foreign key validation by joining against the reference. """ _validate_ref_params(columns, ref_columns, ref_df_name, ref_table) @@ -798,10 +877,13 @@ def apply(df: DataFrame, spark: SparkSession, ref_dfs: dict[str, DataFrame]) -> Joins the dataset with the reference DataFrame or table to verify the foreign key values. Adds a condition column indicating whether each row violates the foreign key constraint. - :param df: The input DataFrame to validate. - :param spark: SparkSession used if reading a reference table. - :param ref_dfs: Dictionary of reference DataFrames (by name), used for joins. - :return: The DataFrame with an additional condition column for foreign key validation. + Args: + df: The input DataFrame to validate. + spark: SparkSession used if reading a reference table. + ref_dfs: Dictionary of reference DataFrames (by name), used for joins. + + Returns: + The DataFrame with an additional condition column for foreign key validation. """ ref_df = _get_ref_df(ref_df_name, ref_table, ref_dfs, spark) @@ -859,20 +941,23 @@ def sql_query( """ Checks whether the condition column generated by SQL query is met. - :param query: SQL query that must return as a minimum a condition column and - all merge columns. The resulting DataFrame is automatically joined back to the input DataFrame. - using the merge_columns. Reference DataFrames when provided in the ref_dfs parameter are registered as temp view. - :param condition_column: Column name indicating violation (boolean). Fail the check if True, pass it if False - :param merge_columns: List of columns for join back to the input DataFrame. - They must provide a unique key for the join, otherwise a duplicate records may be produced. - :param msg: Optional custom message or Column expression. - :param name: Optional name for the result. - :param negate: If True, the condition is negated (i.e., the check fails when the condition is False). - :param input_placeholder: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the - input DataFrame on which the checks are applied. - :param row_filter: Optional SQL expression for filtering rows before checking the foreign key. - Auto-injected from the check filter. - :return: Tuple (condition column, apply function). + Args: + query: SQL query that must return as a minimum a condition column and + all merge columns. The resulting DataFrame is automatically joined back to the input DataFrame + using the merge_columns. Reference DataFrames when provided in the ref_dfs parameter are registered as temp view. + condition_column: Column name indicating violation (boolean). Fail the check if True, pass it if False + merge_columns: List of columns for join back to the input DataFrame. + They must provide a unique key for the join, otherwise a duplicate records may be produced. + msg: Optional custom message or Column expression. + name: Optional name for the result. + negate: If True, the condition is negated (i.e., the check fails when the condition is False). + input_placeholder: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the + input DataFrame on which the checks are applied. + row_filter: Optional SQL expression for filtering rows before checking the foreign key. + Auto-injected from the check filter. + + Returns: + Tuple (condition column, apply function). """ if not merge_columns: raise ValueError("merge_columns must contain at least one column.") @@ -968,14 +1053,17 @@ def is_aggr_not_greater_than( or group of columns does not exceed a specified limit. Rows where the aggregation result exceeds the limit are flagged. - :param column: Column name (str) or Column expression to aggregate. - :param limit: Numeric value, column name, or SQL expression for the limit. - :param aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). - :param group_by: Optional list of column names or Column expressions to group by. - :param row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - :return: A tuple of: - - A Spark Column representing the condition for aggregation limit violations. - - A closure that applies the aggregation check and adds the necessary condition/metric columns. + Args: + column: Column name (str) or Column expression to aggregate. + limit: Numeric value, column name, or SQL expression for the limit. + aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). + group_by: Optional list of column names or Column expressions to group by. + row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + + Returns: + A tuple of: + - A Spark Column representing the condition for aggregation limit violations. + - A closure that applies the aggregation check and adds the necessary condition/metric columns. """ return _is_aggr_compare( column, @@ -1004,14 +1092,17 @@ def is_aggr_not_less_than( or group of columns is not below a specified limit. Rows where the aggregation result is below the limit are flagged. - :param column: Column name (str) or Column expression to aggregate. - :param limit: Numeric value, column name, or SQL expression for the limit. - :param aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). - :param group_by: Optional list of column names or Column expressions to group by. - :param row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - :return: A tuple of: - - A Spark Column representing the condition for aggregation limit violations. - - A closure that applies the aggregation check and adds the necessary condition/metric columns. + Args: + column: Column name (str) or Column expression to aggregate. + limit: Numeric value, column name, or SQL expression for the limit. + aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). + group_by: Optional list of column names or Column expressions to group by. + row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + + Returns: + A tuple of: + - A Spark Column representing the condition for aggregation limit violations. + - A closure that applies the aggregation check and adds the necessary condition/metric columns. """ return _is_aggr_compare( column, @@ -1040,14 +1131,17 @@ def is_aggr_equal( or group of columns is equal to a specified limit. Rows where the aggregation result is not equal to the limit are flagged. - :param column: Column name (str) or Column expression to aggregate. - :param limit: Numeric value, column name, or SQL expression for the limit. - :param aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). - :param group_by: Optional list of column names or Column expressions to group by. - :param row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - :return: A tuple of: - - A Spark Column representing the condition for aggregation limit violations. - - A closure that applies the aggregation check and adds the necessary condition/metric columns. + Args: + column: Column name (str) or Column expression to aggregate. + limit: Numeric value, column name, or SQL expression for the limit. + aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). + group_by: Optional list of column names or Column expressions to group by. + row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + + Returns: + A tuple of: + - A Spark Column representing the condition for aggregation limit violations. + - A closure that applies the aggregation check and adds the necessary condition/metric columns. """ return _is_aggr_compare( column, @@ -1076,14 +1170,17 @@ def is_aggr_not_equal( or group of columns is not equal to a specified limit. Rows where the aggregation result is equal to the limit are flagged. - :param column: Column name (str) or Column expression to aggregate. - :param limit: Numeric value, column name, or SQL expression for the limit. - :param aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). - :param group_by: Optional list of column names or Column expressions to group by. - :param row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. - :return: A tuple of: - - A Spark Column representing the condition for aggregation limit violations. - - A closure that applies the aggregation check and adds the necessary condition/metric columns. + Args: + column: Column name (str) or Column expression to aggregate. + limit: Numeric value, column name, or SQL expression for the limit. + aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max' (default: 'count'). + group_by: Optional list of column names or Column expressions to group by. + row_filter: Optional SQL expression to filter rows before aggregation. Auto-injected from the check filter. + + Returns: + A tuple of: + - A Spark Column representing the condition for aggregation limit violations. + - A closure that applies the aggregation check and adds the necessary condition/metric columns. """ return _is_aggr_compare( column, @@ -1111,48 +1208,54 @@ def compare_datasets( ) -> tuple[Column, Callable]: """ Dataset-level check that compares two datasets and returns a condition for changed rows, - with details on a row and column-level differences. + with details on row and column-level differences. + Only columns that are common across both datasets will be compared. Mismatched columns are ignored. Detailed information about the differences is provided in the condition column. The comparison does not support Map types (any column comparison on map type is skipped automatically). - The log containing detailed differences is written to the message field of the check result as JSON string. - - Example: - ```json - { - "row_missing": false, - "row_extra": true, - "changed": { - "val": { - "df": "val1" - } - } + The log containing detailed differences is written to the message field of the check result as a JSON string. + + Examples: + ```json + { + "row_missing": false, + "row_extra": true, + "changed": { + "val": { + "df": "val1" } - ``` - :param columns: List of columns to use for row matching with the reference DataFrame - (can be a list of string column names or column expressions). - Only simple column expressions are supported, e.g. F.col("col_name") - :param ref_columns: List of columns in the reference DataFrame or Table to row match against the source DataFrame - (can be a list of string column names or column expressions). The `columns` parameter is matched with `ref_columns` - by position, so the order of the provided columns in both lists must be exactly aligned. - Only simple column expressions are supported, e.g. F.col("col_name") - :param ref_df_name: Name of the reference DataFrame (used when passing DataFrames directly). - :param ref_table: Name of the reference table (used when reading from catalog). - :param check_missing_records: Perform FULL OUTER JOIN between the DataFrames to find also records - that could be missing from the DataFrame. Use this with caution as it may produce output with more rows - than in the original DataFrame. - :param exclude_columns: List of columns to exclude from the value comparison but not from row matching - (can be a list of string column names or column expressions). - Only simple column expressions are supported, e.g. F.col("col_name") - The parameter does not alter the list of columns used to determine row matches, - it only controls which columns are skipped during the column value comparison. - :param null_safe_row_matching: If True, treats nulls as equal when matching rows. - :param null_safe_column_value_matching: If True, treats nulls as equal when matching column values. - If enabled (NULL, NULL) column values are equal and matching. - :param row_filter: Optional SQL expression to filter rows in the input DataFrame. - Auto-injected from the check filter. - :return: A tuple of: + } + } + ``` + + Args: + columns: List of columns to use for row matching with the reference DataFrame + (can be a list of string column names or column expressions). Only simple column + expressions are supported, e.g. F.col("col_name"). + ref_columns: List of columns in the reference DataFrame or Table to row match against + the source DataFrame (can be a list of string column names or column expressions). + The `columns` parameter is matched with `ref_columns` by position, so the order of + the provided columns in both lists must be exactly aligned. Only simple column + expressions are supported, e.g. F.col("col_name"). + ref_df_name: Name of the reference DataFrame (used when passing DataFrames directly). + ref_table: Name of the reference table (used when reading from catalog). + check_missing_records: Perform FULL OUTER JOIN between the DataFrames to also find + records that could be missing from the DataFrame. Use with caution as it may produce + output with more rows than in the original DataFrame. + exclude_columns: List of columns to exclude from the value comparison but not from row + matching (can be a list of string column names or column expressions). Only simple + column expressions are supported, e.g. F.col("col_name"). This parameter does not alter + the list of columns used to determine row matches; it only controls which columns are + skipped during the column value comparison. + null_safe_row_matching: If True, treats nulls as equal when matching rows. + null_safe_column_value_matching: If True, treats nulls as equal when matching column values. + If enabled, (NULL, NULL) column values are equal and matching. + row_filter: Optional SQL expression to filter rows in the input DataFrame. Auto-injected + from the check filter. + + Returns: + Tuple[Column, Callable]: - A Spark Column representing the condition for comparison violations. - A closure that applies the comparison validation. """ @@ -1249,17 +1352,20 @@ def is_data_fresh_per_time_window( If `lookback_windows` is provided, only data within that lookback period will be validated. If omitted, the entire dataset will be checked. - :param column: Column name (str) or Column expression containing timestamps to check. - :param window_minutes: Time window in minutes to check for data arrival. - :param min_records_per_window: Minimum number of records expected per time window. - :param lookback_windows: Optional number of time windows to look back from `curr_timestamp`. - This filters records to include only those within the specified number of time windows from `curr_timestamp`. - If no lookback is provided, the check is applied to the entire dataset. - :param row_filter: Optional SQL expression to filter rows before checking. - :param curr_timestamp: Optional current timestamp column. If not provided, current_timestamp() function is used. - :return: A tuple of: - - A Spark Column representing the condition for missing data within a time window. - - A closure that applies the completeness check and adds the necessary condition columns. + Args: + column: Column name (str) or Column expression containing timestamps to check. + window_minutes: Time window in minutes to check for data arrival. + min_records_per_window: Minimum number of records expected per time window. + lookback_windows: Optional number of time windows to look back from `curr_timestamp`. + This filters records to include only those within the specified number of time windows from `curr_timestamp`. + If no lookback is provided, the check is applied to the entire dataset. + row_filter: Optional SQL expression to filter rows before checking. + curr_timestamp: Optional current timestamp column. If not provided, current_timestamp() function is used. + + Returns: + A tuple of: + - A Spark Column representing the condition for missing data within a time window. + - A closure that applies the completeness check and adds the necessary condition columns. """ col_str_norm, _, col_expr = _get_normalized_column_and_expr(column) @@ -1284,8 +1390,11 @@ def apply(df: DataFrame) -> DataFrame: Creates time windows and checks if each window has the minimum required records. - :param df: The input DataFrame to validate. - :return: The DataFrame with additional condition columns for completeness validation. + Args: + df: The input DataFrame to validate. + + Returns: + The DataFrame with additional condition columns for completeness validation. """ # Build filter condition filter_condition = F.lit(True) @@ -1353,13 +1462,16 @@ def _match_rows( NULL <=> NULL matches; 1 <=> NULL does not match; - :param df: The input DataFrame to join. - :param ref_df: The reference DataFrame to join against. - :param pk_column_names: List of primary key column names in the input DataFrame. - :param ref_pk_column_names: List of primary key column names in the reference DataFrame. - :param check_missing_records: If True, perform a full outer join to find missing records in both DataFrames. - :param null_safe_row_matching: If True, treats nulls as equal when matching rows. - :return: A DataFrame with the results of the join. + Args: + df: The input DataFrame to join. + ref_df: The reference DataFrame to join against. + pk_column_names: List of primary key column names in the input DataFrame. + ref_pk_column_names: List of primary key column names in the reference DataFrame. + check_missing_records: If True, perform a full outer join to find missing records in both DataFrames. + null_safe_row_matching: If True, treats nulls as equal when matching rows. + + Returns: + A DataFrame with the results of the join. """ join_condition = F.lit(True) for column, ref_column in zip(pk_column_names, ref_pk_column_names): @@ -1415,13 +1527,16 @@ def _add_column_diffs( For each column in `compare_columns`, it checks if the values in `df` and `ref_df` are equal. If a difference is found, it adds the column name and the differing values to a map stored in `columns_changed_col`. - :param df: The input DataFrame containing columns to compare. - :param compare_columns: List of column names to compare between `df` and `ref_df`. - :param columns_changed_col: Name of the column to store the map of changed columns and their differences. - :param null_safe_column_value_matching: If True, treats nulls as equal when matching column values. - If enabled (NULL, NULL) column values are equal and matching. - If False, uses a standard inequality comparison (`!=`), where (NULL, NULL) values are not considered equal. - :return: A DataFrame with the added `columns_changed_col` containing the map of changed columns and differences. + Args: + df: The input DataFrame containing columns to compare. + compare_columns: List of column names to compare between `df` and `ref_df`. + columns_changed_col: Name of the column to store the map of changed columns and their differences. + null_safe_column_value_matching: If True, treats nulls as equal when matching column values. + If enabled (NULL, NULL) column values are equal and matching. + If False, uses a standard inequality comparison (`!=`), where (NULL, NULL) values are not considered equal. + + Returns: + A DataFrame with the added `columns_changed_col` containing the map of changed columns and differences. """ if compare_columns: columns_changed = [ @@ -1473,13 +1588,16 @@ def _add_compare_condition( about mismatched records. The mismatches are determined based on the presence of missing rows, extra rows, and differences in specified columns. - :param df: The input DataFrame containing the comparison results. - :param condition_col: The name of the column to add, which will store mismatch information. - :param row_missing_col: The name of the column indicating missing rows. - :param row_extra_col: The name of the column indicating extra rows. - :param columns_changed_col: The name of the column containing differences in compared columns. - :param filter_col: The name of the column used to filter records for comparison. - :return: The input DataFrame with the added `condition_col` containing mismatch information. + Args: + df: The input DataFrame containing the comparison results. + condition_col: The name of the column to add, which will store mismatch information. + row_missing_col: The name of the column indicating missing rows. + row_extra_col: The name of the column indicating extra rows. + columns_changed_col: The name of the column containing differences in compared columns. + filter_col: The name of the column used to filter records for comparison. + + Returns: + The input DataFrame with the added `condition_col` containing mismatch information. """ all_is_ok = ~F.col(row_missing_col) & ~F.col(row_extra_col) & (F.size(F.col(columns_changed_col)) == 0) return df.withColumn( @@ -1512,17 +1630,20 @@ def _is_aggr_compare( Constructs a condition and closure that verify whether an aggregation on a column (or groups of columns) satisfies a comparison against a limit (e.g., greater than). - :param column: Column name (str) or Column expression to aggregate. - :param limit: Numeric value, column name, or SQL expression for the limit. - :param aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max'. - :param group_by: Optional list of columns or Column expressions to group by. - :param row_filter: Optional SQL expression to filter rows before aggregation. - :param compare_op: Comparison operator (e.g., operator.gt, operator.lt). - :param compare_op_label: Human-readable label for the comparison (e.g., 'greater than'). - :param compare_op_name: Name identifier for the comparison (e.g., 'greater_than'). - :return: A tuple of: - - A Spark Column representing the condition for the aggregation check. - - A closure that applies the aggregation check logic. + Args: + column: Column name (str) or Column expression to aggregate. + limit: Numeric value, column name, or SQL expression for the limit. + aggr_type: Aggregation type: 'count', 'sum', 'avg', 'min', or 'max'. + group_by: Optional list of columns or Column expressions to group by. + row_filter: Optional SQL expression to filter rows before aggregation. + compare_op: Comparison operator (e.g., operator.gt, operator.lt). + compare_op_label: Human-readable label for the comparison (e.g., 'greater than'). + compare_op_name: Name identifier for the comparison (e.g., 'greater_than'). + + Returns: + A tuple of: + - A Spark Column representing the condition for the aggregation check. + - A closure that applies the aggregation check logic. """ supported_aggr_types = {"count", "sum", "avg", "min", "max"} if aggr_type not in supported_aggr_types: @@ -1561,8 +1682,11 @@ def apply(df: DataFrame) -> DataFrame: and compares the result against the limit. Adds condition and metric columns used for check evaluation. - :param df: The input DataFrame to validate. - :return: The DataFrame with additional condition and metric columns for aggregation validation. + Args: + df: The input DataFrame to validate. + + Returns: + The DataFrame with additional condition and metric columns for aggregation validation. """ filter_col = F.expr(row_filter) if row_filter else F.lit(True) filtered_expr = F.when(filter_col, aggr_col_expr) if row_filter else aggr_col_expr @@ -1612,13 +1736,18 @@ def _get_ref_df( (using `ref_df_name` as the key) or by reading a table from the Unity Catalog (using `ref_table`). It raises an error if the necessary reference source is not properly specified or cannot be found. - :param ref_df_name: The key name of the reference DataFrame in the provided dictionary (optional). - :param ref_table: The name of the reference table to read from the Spark catalog (optional). - :param ref_dfs: A dictionary mapping reference DataFrame names to DataFrame objects. - :param spark: The active SparkSession used to read the reference table if needed. - :return: A Spark DataFrame representing the reference dataset. - :raises ValueError: If neither or both of `ref_df_name` and `ref_table` are provided, - or if the specified reference DataFrame is not found. + Args: + ref_df_name: The key name of the reference DataFrame in the provided dictionary (optional). + ref_table: The name of the reference table to read from the Spark catalog (optional). + ref_dfs: A dictionary mapping reference DataFrame names to DataFrame objects. + spark: The active SparkSession used to read the reference table if needed. + + Returns: + A Spark DataFrame representing the reference dataset. + + Raises: + ValueError: If neither or both of `ref_df_name` and `ref_table` are provided, + or if the specified reference DataFrame is not found. """ if ref_df_name: if not ref_dfs: @@ -1648,8 +1777,11 @@ def _cleanup_alias_name(column: str) -> str: This helper avoids issues when using struct field names as aliases, since dots in column names can cause ambiguity in Spark SQL. - :param column: The column name as a string. - :return: A sanitized column name with dots replaced by underscores. + Args: + column: The column name as a string. + + Returns: + A sanitized column name with dots replaced by underscores. """ # avoid issues with structs return column.replace(".", "_") @@ -1664,10 +1796,15 @@ def _get_limit_expr( This helper converts the provided limit (literal, string expression, or Column) into a Spark Column expression suitable for use in conditions. - :param limit: The limit to use in the condition. Can be a literal (int, float, date, datetime), - a string SQL expression, or a Spark Column. - :return: A Spark Column expression representing the limit. - :raises ValueError: If the limit is not provided (None). + Args: + limit: The limit to use in the condition. Can be a literal (int, float, date, datetime), + a string SQL expression, or a Spark Column. + + Returns: + A Spark Column expression representing the limit. + + Raises: + ValueError: If the limit is not provided (None). """ if limit is None: raise ValueError("Limit is not provided.") @@ -1687,11 +1824,14 @@ def _get_normalized_column_and_expr(column: str | Column) -> tuple[str, str, Col of the column are available, along with the corresponding Spark Column expression. Useful for generating aliases, conditions, and consistent messaging. - :param column: The input column, provided as either a string column name or a Spark Column expression. - :return: A tuple containing: - - Normalized column name as a string (suitable for use in aliases or metadata). - - Original column name as a string. - - Spark Column expression corresponding to the input. + Args: + column: The input column, provided as either a string column name or a Spark Column expression. + + Returns: + A tuple containing: + - Normalized column name as a string (suitable for use in aliases or metadata). + - Original column name as a string. + - Spark Column expression corresponding to the input. """ col_expr = F.expr(column) if isinstance(column, str) else column column_str = get_column_name_or_alias(col_expr) @@ -1708,13 +1848,16 @@ def _handle_fk_composite_keys(columns: list[str | Column], ref_columns: list[str in both the main and reference datasets. It also updates the not-null condition to skip any rows where any of the composite key columns are NULL, in line with SQL ANSI foreign key semantics. - :param columns: List of columns (names or expressions) from the input DataFrame forming the composite key. - :param ref_columns: List of columns (names or expressions) from the reference DataFrame forming the composite key. - :param not_null_condition: Existing condition Column to be combined with not-null checks for the composite key. - :return: A tuple containing: - - Column expression representing the composite key in the input DataFrame. - - Column expression representing the composite key in the reference DataFrame. - - Updated not-null condition Column ensuring no NULLs in any composite key field. + Args: + columns: List of columns (names or expressions) from the input DataFrame forming the composite key. + ref_columns: List of columns (names or expressions) from the reference DataFrame forming the composite key. + not_null_condition: Existing condition Column to be combined with not-null checks for the composite key. + + Returns: + A tuple containing: + - Column expression representing the composite key in the input DataFrame. + - Column expression representing the composite key in the reference DataFrame. + - Updated not-null condition Column ensuring no NULLs in any composite key field. """ # Extract column names from columns for consistent aliasing columns_names = [get_column_name_or_alias(col) if not isinstance(col, str) else col for col in columns] @@ -1738,9 +1881,12 @@ def _build_fk_composite_key_struct(columns: list[str | Column], columns_names: l ensuring each field in the struct has a consistent alias based on the provided column names. This is used for comparing composite foreign keys as a single struct value. - :param columns: List of columns (names as str or Spark Column expressions) to include in the struct. - :param columns_names: List of normalized column names (str) to use as aliases for the struct fields. - :return: A Spark Column representing a struct with the specified columns and aliases. + Args: + columns: List of columns (names as str or Spark Column expressions) to include in the struct. + columns_names: List of normalized column names (str) to use as aliases for the struct fields. + + Returns: + A Spark Column representing a struct with the specified columns and aliases. """ struct_fields = [] for alias, col in zip(columns_names, columns): @@ -1761,12 +1907,15 @@ def _validate_ref_params( - Exactly one of `ref_df_name` or `ref_table` is provided (not both, not neither). - The number of columns in the input DataFrame matches the number of reference columns. - :param columns: List of columns from the input DataFrame. - :param ref_columns: List of columns from the reference DataFrame or table. - :param ref_df_name: Optional name of the reference DataFrame. - :param ref_table: Optional name of the reference table. - :raises ValueError: If both or neither of `ref_df_name` and `ref_table` are provided, - or if the lengths of `columns` and `ref_columns` do not match. + Args: + columns: List of columns from the input DataFrame. + ref_columns: List of columns from the reference DataFrame or table. + ref_df_name: Optional name of the reference DataFrame. + ref_table: Optional name of the reference table. + + Raises: + ValueError: If both or neither of `ref_df_name` and `ref_table` are provided, + or if the lengths of `columns` and `ref_columns` do not match. """ if ref_df_name and ref_table: raise ValueError( @@ -1805,8 +1954,11 @@ def _get_network_address(ip_bits: Column, prefix_length: Column) -> Column: """ Returns the network address from IP bits using the CIDR prefix length. - :param ip_bits: 32-bit binary string representation of the IPv4 address - :param prefix_length: Prefix length for CIDR notation - :return: Network address as a 32-bit binary string + Args: + ip_bits: 32-bit binary string representation of the IPv4 address + prefix_length: Prefix length for CIDR notation + + Returns: + Network address as a 32-bit binary string """ return F.rpad(F.substring(ip_bits, 1, prefix_length), 32, "0") diff --git a/src/databricks/labs/dqx/checks_resolver.py b/src/databricks/labs/dqx/checks_resolver.py index 7d6286b8f..fc96b3780 100644 --- a/src/databricks/labs/dqx/checks_resolver.py +++ b/src/databricks/labs/dqx/checks_resolver.py @@ -14,10 +14,13 @@ def resolve_check_function( """ Resolves a function by name from the predefined functions and custom checks. - :param function_name: name of the function to resolve. - :param custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). - :param fail_on_missing: if True, raise an AttributeError if the function is not found. - :return: function or None if not found. + Args: + function_name: name of the function to resolve. + custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). + fail_on_missing: if True, raise an AttributeError if the function is not found. + + Returns: + function or None if not found. """ logger.debug(f"Resolving function: {function_name}") func = getattr(check_funcs, function_name, None) # resolve using predefined checks first diff --git a/src/databricks/labs/dqx/checks_serializer.py b/src/databricks/labs/dqx/checks_serializer.py index c7da89399..292d3cc7f 100644 --- a/src/databricks/labs/dqx/checks_serializer.py +++ b/src/databricks/labs/dqx/checks_serializer.py @@ -46,16 +46,19 @@ def serialize_checks_from_dataframe(df: DataFrame, run_config_name: str = "defau Converts a list of quality checks defined in a DataFrame to a list of quality checks defined as Python dictionaries. - :param df: DataFrame with data quality check rules. Each row should define a check. Rows should - have the following columns: - * `name` - Name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` (data is going into both dataframes) - * `check` - DQX check function used in the check; A `StructType` column defining the data quality check - * `filter` - Expression for filtering data quality checks - * `run_config_name` (optional) - Run configuration name for storing checks across runs - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - :param run_config_name: Run configuration name for filtering quality rules - :return: List of data quality check specifications as a Python dictionary + Args: + df: DataFrame with data quality check rules. Each row should define a check. Rows should + have the following columns: + * `name` - Name that will be given to a resulting column. Autogenerated if not provided + * `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` (data is going into both dataframes) + * `check` - DQX check function used in the check; A `StructType` column defining the data quality check + * `filter` - Expression for filtering data quality checks + * `run_config_name` (optional) - Run configuration name for storing checks across runs + * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + run_config_name: Run configuration name for filtering quality rules + + Returns: + List of data quality check specifications as a Python dictionary """ check_rows = df.where(f"run_config_name = '{run_config_name}'").collect() collect_limit = 500 @@ -98,8 +101,9 @@ def deserialize_checks_to_dataframe( """ Converts a list of quality checks defined as Python dictionaries to a DataFrame. - :param spark: Spark session. - :param checks: list of check specifications as Python dictionaries. Each check consists of the following fields: + Args: + spark: Spark session. + checks: list of check specifications as Python dictionaries. Each check consists of the following fields: * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true (it will be used as an error/warning message) or `null` if it's evaluated to `false` * `name` - Name that will be given to a resulting column. Autogenerated if not provided @@ -107,8 +111,10 @@ def deserialize_checks_to_dataframe( (data is going into both dataframes) * `filter` (optional) - Expression for filtering data quality checks * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - :param run_config_name: Run configuration name for storing quality checks across runs - :return: DataFrame with data quality check rules + run_config_name: Run configuration name for storing quality checks across runs + + Returns: + DataFrame with data quality check rules """ dq_rule_checks: list[DQRule] = deserialize_checks(checks) @@ -142,18 +148,21 @@ def deserialize_checks(checks: list[dict], custom_checks: dict[str, Any] | None """ Converts a list of quality checks defined as Python dictionaries to a list of `DQRule` objects. - :param checks: list of dictionaries describing checks. Each check is a dictionary - consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) - * `filter` (optional) - Expression for filtering data quality checks - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - :param custom_checks: dictionary with custom check functions (eg. ``globals()`` of the calling module). - If not specified, then only built-in functions are used for the checks. - :return: list of data quality check rules + Args: + checks: list of dictionaries describing checks. Each check is a dictionary + consisting of following fields: + * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true + - it will be used as an error/warning message, or `null` if it's evaluated to `false` + * `name` - name that will be given to a resulting column. Autogenerated if not provided + * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), + and `warn` (data is going into both dataframes) + * `filter` (optional) - Expression for filtering data quality checks + * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + custom_checks: dictionary with custom check functions (eg. ``globals()`` of the calling module). + If not specified, then only built-in functions are used for the checks. + + Returns: + list of data quality check rules """ status = ChecksValidator.validate_checks(checks, custom_checks) if status.has_errors: @@ -230,8 +239,10 @@ def serialize_checks(checks: list[DQRule]) -> list[dict]: Converts a list of quality checks defined as `DQRule` objects to a list of quality checks defined as Python dictionaries. - :param checks: List of DQRule instances to convert. - :return: List of dictionaries representing the DQRule instances. + Args: + checks: List of DQRule instances to convert. + Returns: + List of dictionaries representing the DQRule instances. """ dq_rules = [] for check in checks: @@ -245,9 +256,11 @@ def serialize_checks_to_bytes(checks: list[dict], file_path: Path) -> bytes: """ Serializes a list of checks to bytes in json or yaml (default) format. - :param checks: List of checks to serialize. - :param file_path: Path to the file where the checks will be serialized. - :return: Serialized checks as bytes. + Args: + checks: List of checks to serialize. + file_path: Path to the file where the checks will be serialized. + Returns: + Serialized checks as bytes. """ serializer = FILE_SERIALIZERS.get(file_path.suffix.lower(), yaml.safe_dump) # default to yaml return serializer(checks).encode("utf-8") @@ -257,8 +270,10 @@ def get_file_deserializer(filepath: str) -> Callable: """ Get the deserializer function based on file. - :param filepath: Path to the file. - :return: Deserializer function. + Args: + filepath: Path to the file. + Returns: + Deserializer function. """ ext = Path(filepath).suffix.lower() return FILE_DESERIALIZERS.get(ext.lower(), yaml.safe_load) # default to yaml diff --git a/src/databricks/labs/dqx/checks_storage.py b/src/databricks/labs/dqx/checks_storage.py index e5ee0197a..c761ccaa2 100644 --- a/src/databricks/labs/dqx/checks_storage.py +++ b/src/databricks/labs/dqx/checks_storage.py @@ -48,8 +48,11 @@ def load(self, config: T) -> list[dict]: The returned checks can be used as input for `apply_checks_by_metadata` or `apply_checks_by_metadata_and_split` functions. - :param config: configuration for loading checks, including the table location and run configuration name. - :return: list of dq rules or raise an error if checks file is missing or is invalid. + Args: + config: configuration for loading checks, including the table location and run configuration name. + + Returns: + list of dq rules or raise an error if checks file is missing or is invalid. """ @abstractmethod @@ -70,8 +73,11 @@ def load(self, config: TableChecksStorageConfig) -> list[dict]: """ Load checks (dq rules) from a Delta table in the workspace. - :param config: configuration for loading checks, including the table location and run configuration name. - :return: list of dq rules or raise an error if checks table is missing or is invalid. + Args: + config: configuration for loading checks, including the table location and run configuration name. + + Returns: + list of dq rules or raise an error if checks table is missing or is invalid. """ logger.info(f"Loading quality rules (checks) from table '{config.location}'") if not self.ws.tables.exists(config.location).table_exists: @@ -83,9 +89,12 @@ def save(self, checks: list[dict], config: TableChecksStorageConfig) -> None: """ Save checks to a Delta table in the workspace. - :param checks: list of dq rules to save - :param config: configuration for saving checks, including the table location and run configuration name. - :raises ValueError: if the table name is not provided + Args: + checks: list of dq rules to save + config: configuration for saving checks, including the table location and run configuration name. + + Raises: + ValueError: if the table name is not provided """ logger.info(f"Saving quality rules (checks) to table '{config.location}'") rules_df = deserialize_checks_to_dataframe(self.spark, checks, run_config_name=config.run_config_name) @@ -106,8 +115,11 @@ def load(self, config: WorkspaceFileChecksStorageConfig) -> list[dict]: """Load checks (dq rules) from a file (json or yaml) in the workspace. This does not require installation of DQX in the workspace. - :param config: configuration for loading checks, including the file location and storage type. - :return: list of dq rules or raise an error if checks file is missing or is invalid. + Args: + config: configuration for loading checks, including the file location and storage type. + + Returns: + list of dq rules or raise an error if checks file is missing or is invalid. """ file_path = config.location logger.info(f"Loading quality rules (checks) from '{file_path}' in the workspace.") @@ -129,8 +141,9 @@ def save(self, checks: list[dict], config: WorkspaceFileChecksStorageConfig) -> """Save checks (dq rules) to yaml file in the workspace. This does not require installation of DQX in the workspace. - :param checks: list of dq rules to save - :param config: configuration for saving checks, including the file location and storage type. + Args: + checks: list of dq rules to save + config: configuration for saving checks, including the file location and storage type. """ logger.info(f"Saving quality rules (checks) to '{config.location}' in the workspace.") file_path = Path(config.location) @@ -150,10 +163,15 @@ def load(self, config: FileChecksStorageConfig) -> list[dict]: """ Load checks (dq rules) from a file (json or yaml) in the local filesystem. - :param config: configuration for loading checks, including the file location. - :return: list of dq rules or raise an error if checks file is missing or is invalid. - :raises ValueError: if the file path is not provided - :raises FileNotFoundError: if the file path does not exist + Args: + config: configuration for loading checks, including the file location. + + Returns: + list of dq rules or raise an error if checks file is missing or is invalid. + + Raises: + ValueError: if the file path is not provided + FileNotFoundError: if the file path does not exist """ file_path = config.location logger.info(f"Loading quality rules (checks) from '{file_path}'.") @@ -172,10 +190,13 @@ def save(self, checks: list[dict], config: FileChecksStorageConfig) -> None: """ Save checks (dq rules) to a file (json or yaml) in the local filesystem. - :param checks: list of dq rules to save - :param config: configuration for saving checks, including the file location. - :raises ValueError: if the file path is not provided - :raises FileNotFoundError: if the file path does not exist + Args: + checks: list of dq rules to save + config: configuration for saving checks, including the file location. + + Raises: + ValueError: if the file path is not provided + FileNotFoundError: if the file path does not exist """ logger.info(f"Saving quality rules (checks) to '{config.location}'.") file_path = Path(config.location) @@ -205,9 +226,14 @@ def load(self, config: InstallationChecksStorageConfig) -> list[dict]: """ Load checks (dq rules) from the installation configuration. - :param config: configuration for loading checks, including the run configuration name and method. - :return: list of dq rules or raise an error if checks file is missing or is invalid. - :raises NotFound: if the checks file or table is not found in the installation. + Args: + config: configuration for loading checks, including the run configuration name and method. + + Returns: + list of dq rules or raise an error if checks file is missing or is invalid. + + Raises: + NotFound: if the checks file or table is not found in the installation. """ handler, config = self._get_storage_handler_and_config(config) return handler.load(config) @@ -217,8 +243,9 @@ def save(self, checks: list[dict], config: InstallationChecksStorageConfig) -> N Save checks (dq rules) to yaml file or table in the installation folder. This will overwrite existing checks file or table. - :param checks: list of dq rules to save - :param config: configuration for saving checks, including the run configuration name, method, and table location. + Args: + checks: list of dq rules to save + config: configuration for saving checks, including the run configuration name, method, and table location. """ handler, config = self._get_storage_handler_and_config(config) return handler.save(checks, config) @@ -256,8 +283,11 @@ def __init__(self, ws: WorkspaceClient): def load(self, config: VolumeFileChecksStorageConfig) -> list[dict]: """Load checks (dq rules) from a file (json or yaml) in a Unity Catalog volume. - :param config: configuration for loading checks, including the file location and storage type. - :return: list of dq rules or raise an error if checks file is missing or is invalid. + Args: + config: configuration for loading checks, including the file location and storage type. + + Returns: + list of dq rules or raise an error if checks file is missing or is invalid. """ file_path = config.location logger.info(f"Loading quality rules (checks) from '{file_path}' in a volume.") @@ -285,8 +315,9 @@ def save(self, checks: list[dict], config: VolumeFileChecksStorageConfig) -> Non """Save checks (dq rules) to yaml file in a Unity Catalog volume. This does not require installation of DQX in a Unity Catalog volume. - :param checks: list of dq rules to save - :param config: configuration for saving checks, including the file location and storage type. + Args: + checks: list of dq rules to save + config: configuration for saving checks, including the file location and storage type. """ logger.info(f"Saving quality rules (checks) to '{config.location}' in a Unity Catalog volume.") file_path = Path(config.location) @@ -308,8 +339,11 @@ def create(self, config: BaseChecksStorageConfig) -> ChecksStorageHandler: """ Abstract method to create a handler based on the type of the provided configuration object. - :param config: Configuration object for loading or saving checks. - :return: An instance of the corresponding BaseChecksStorageHandler. + Args: + config: Configuration object for loading or saving checks. + + Returns: + An instance of the corresponding BaseChecksStorageHandler. """ @@ -322,9 +356,14 @@ def create(self, config: BaseChecksStorageConfig) -> ChecksStorageHandler: """ Factory method to create a handler based on the type of the provided configuration object. - :param config: Configuration object for loading or saving checks. - :return: An instance of the corresponding BaseChecksStorageHandler. - :raises ValueError: If the configuration type is unsupported. + Args: + config: Configuration object for loading or saving checks. + + Returns: + An instance of the corresponding BaseChecksStorageHandler. + + Raises: + ValueError: If the configuration type is unsupported. """ if isinstance(config, FileChecksStorageConfig): return FileChecksStorageHandler() diff --git a/src/databricks/labs/dqx/checks_validator.py b/src/databricks/labs/dqx/checks_validator.py index f3254377c..604b3df8e 100644 --- a/src/databricks/labs/dqx/checks_validator.py +++ b/src/databricks/labs/dqx/checks_validator.py @@ -80,10 +80,13 @@ def _validate_checks_dict( """ Validates the structure and content of a given check dictionary. - :param check: The check dictionary to validate. - :param custom_check_functions: A dictionary containing custom check functions. - :param validate_custom_check_functions: If True, validate custom check functions. - :return: A list of error messages if any validation fails, otherwise an empty list. + Args: + check: The check dictionary to validate. + custom_check_functions: A dictionary containing custom check functions. + validate_custom_check_functions: If True, validate custom check functions. + + Returns: + A list of error messages if any validation fails, otherwise an empty list. """ errors: list[str] = [] @@ -112,10 +115,13 @@ def _validate_check_block( """ Validates a check block within a configuration. - :param check: The check configuration to validate. - :param custom_check_functions: A dictionary containing custom check functions. - :param validate_custom_check_functions: If True, validate custom check functions. - :return: A list of error messages if any validation fails, otherwise an empty list. + Args: + check: The check configuration to validate. + custom_check_functions: A dictionary containing custom check functions. + validate_custom_check_functions: If True, validate custom check functions. + + Returns: + A list of error messages if any validation fails, otherwise an empty list. """ check_block = check["check"] @@ -148,11 +154,14 @@ def _validate_check_function_arguments( """ Validates the provided arguments for a given function and updates the errors list if any validation fails. - :param arguments: A dictionary of arguments to validate. - :param func: The function for which the arguments are being validated. - :param for_each_column: A list of columns to iterate over for the check. - :param check: A dictionary containing the check configuration. - :return: A list of error messages if any validation fails, otherwise an empty list. + Args: + arguments: A dictionary of arguments to validate. + func: The function for which the arguments are being validated. + for_each_column: A list of columns to iterate over for the check. + check: A dictionary containing the check configuration. + + Returns: + A list of error messages if any validation fails, otherwise an empty list. """ if not isinstance(arguments, dict): return [f"'arguments' should be a dictionary in the 'check' block: {check}"] @@ -180,11 +189,14 @@ def _validate_func_args(arguments: dict, func: Callable, check: dict, func_param """ Validates the arguments passed to a function against its signature. - :param arguments: A dictionary of argument names and their values to be validated. - :param func: The function whose arguments are being validated. - :param check: A dictionary containing additional context or information for error messages. - :param func_parameters: The parameters of the function as obtained from its signature. - :return: A list of error messages if any validation fails, otherwise an empty list. + Args: + arguments: A dictionary of argument names and their values to be validated. + func: The function whose arguments are being validated. + check: A dictionary containing additional context or information for error messages. + func_parameters: The parameters of the function as obtained from its signature. + + Returns: + A list of error messages if any validation fails, otherwise an empty list. """ errors: list[str] = [] if not arguments and func_parameters: @@ -275,12 +287,15 @@ def _validate_func_list_args( """ Validates the list arguments passed to a function against its signature. - :param arguments: A dictionary of argument names and their values to be validated. - :param func: The function whose arguments are being validated. - :param check: A dictionary containing additional context or information for error messages. - :param expected_type_args: Expected types for the list items. - :param value: The value of the argument to validate. - :return: A list of error messages if any validation fails, otherwise an empty list. + Args: + arguments: A dictionary of argument names and their values to be validated. + func: The function whose arguments are being validated. + check: A dictionary containing additional context or information for error messages. + expected_type_args: Expected types for the list items. + value: The value of the argument to validate. + + Returns: + A list of error messages if any validation fails, otherwise an empty list. """ if not isinstance(value, list): return [ diff --git a/src/databricks/labs/dqx/cli.py b/src/databricks/labs/dqx/cli.py index c229e5e0f..df53e749e 100644 --- a/src/databricks/labs/dqx/cli.py +++ b/src/databricks/labs/dqx/cli.py @@ -21,8 +21,9 @@ def open_remote_config(w: WorkspaceClient, *, ctx: WorkspaceContext | None = Non """ Opens remote configuration in the browser. - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param ctx: The WorkspaceContext instance to use for accessing the workspace. + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + ctx: The WorkspaceContext instance to use for accessing the workspace. """ ctx = ctx or WorkspaceContext(w) workspace_link = ctx.installation.workspace_link(WorkspaceConfig.__file__) @@ -34,8 +35,9 @@ def open_dashboards(w: WorkspaceClient, *, ctx: WorkspaceContext | None = None): """ Opens remote dashboard directory in the browser. - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param ctx: The WorkspaceContext instance to use for accessing the workspace. + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + ctx: The WorkspaceContext instance to use for accessing the workspace. """ ctx = ctx or WorkspaceContext(w) workspace_link = ctx.installation.workspace_link("") @@ -47,8 +49,9 @@ def installations(w: WorkspaceClient, *, product_name: str = "dqx") -> list[dict """ Show installations by different users on the same workspace. - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param product_name: The name of the product to search for in the installation folder. + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + product_name: The name of the product to search for in the installation folder. """ logger.info("Fetching installations...") all_users = [] @@ -81,10 +84,11 @@ def validate_checks( """ Validate checks stored in the installation directory as a file. - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param run_config: The name of the run configuration to use. - :param validate_custom_check_functions: Whether to validate custom check functions (default is True). - :param ctx: The WorkspaceContext instance to use for accessing the workspace. + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + run_config: The name of the run configuration to use. + validate_custom_check_functions: Whether to validate custom check functions (default is True). + ctx: The WorkspaceContext instance to use for accessing the workspace. """ ctx = ctx or WorkspaceContext(w) config = ctx.installation.load(WorkspaceConfig) @@ -109,9 +113,10 @@ def profile(w: WorkspaceClient, *, run_config: str = "default", ctx: WorkspaceCo """ Profile input data and generate quality rule (checks) candidates. - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param run_config: The name of the run configuration to use. - :param ctx: The WorkspaceContext instance to use for accessing the workspace. + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + run_config: The name of the run configuration to use. + ctx: The WorkspaceContext instance to use for accessing the workspace. """ ctx = ctx or WorkspaceContext(w) ctx.deployed_workflows.run_workflow("profiler", run_config) @@ -122,8 +127,9 @@ def workflows(w: WorkspaceClient, *, ctx: WorkspaceContext | None = None): """ Show deployed workflows and their state - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param ctx: The WorkspaceContext instance to use for accessing the workspace. + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + ctx: The WorkspaceContext instance to use for accessing the workspace. """ ctx = ctx or WorkspaceContext(w) logger.info("Fetching deployed jobs...") @@ -137,9 +143,10 @@ def logs(w: WorkspaceClient, *, workflow: str | None = None, ctx: WorkspaceConte """ Show logs of the latest job run. - :param w: The WorkspaceClient instance to use for accessing the workspace. - :param workflow: The name of the workflow to show logs for. - :param ctx: The WorkspaceContext instance to use for accessing the workspace + Args: + w: The WorkspaceClient instance to use for accessing the workspace. + workflow: The name of the workflow to show logs for. + ctx: The WorkspaceContext instance to use for accessing the workspace """ ctx = ctx or WorkspaceContext(w) ctx.deployed_workflows.relay_logs(workflow) diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index fb5207087..9ac4fe05b 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -80,10 +80,16 @@ class WorkspaceConfig: def get_run_config(self, run_config_name: str | None = "default") -> RunConfig: """Get the run configuration for a given run name, or the default configuration if no run name is provided. - :param run_config_name: The name of the run configuration to get. - :return: The run configuration. - :raises ValueError: If no run configurations are available or if the specified run configuration name is - not found. + + Args: + run_config_name: The name of the run configuration to get. + + Returns: + The run configuration. + + Raises: + ValueError: If no run configurations are available or if the specified run configuration name is + not found. """ if not self.run_configs: raise ValueError("No run configurations available") @@ -108,7 +114,8 @@ class FileChecksStorageConfig(BaseChecksStorageConfig): """ Configuration class for storing checks in a file. - :param location: The file path where the checks are stored. + Args: + location: The file path where the checks are stored. """ location: str @@ -123,7 +130,8 @@ class WorkspaceFileChecksStorageConfig(BaseChecksStorageConfig): """ Configuration class for storing checks in a workspace file. - :param location: The workspace file path where the checks are stored. + Args: + location: The workspace file path where the checks are stored. """ location: str @@ -138,10 +146,11 @@ class TableChecksStorageConfig(BaseChecksStorageConfig): """ Configuration class for storing checks in a table. - :param location: The table name where the checks are stored. - :param run_config_name: The name of the run configuration to use for checks (default is 'default'). - :param mode: The mode for writing checks to a table (e.g., 'append' or 'overwrite'). - The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. + Args: + location: The table name where the checks are stored. + run_config_name: The name of the run configuration to use for checks (default is 'default'). + mode: The mode for writing checks to a table (e.g., 'append' or 'overwrite'). + The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. """ location: str @@ -158,7 +167,8 @@ class VolumeFileChecksStorageConfig(BaseChecksStorageConfig): """ Configuration class for storing checks in a Unity Catalog volume file. - :param location: The Unity Catalog volume file path where the checks are stored. + Args: + location: The Unity Catalog volume file path where the checks are stored. """ location: str @@ -175,11 +185,12 @@ class InstallationChecksStorageConfig( """ Configuration class for storing checks in an installation. - :param location: The installation path where the checks are stored (e.g., table name, file path). - Not used when using installation method, as it is retrieved from the installation config. - :param run_config_name: The name of the run configuration to use for checks (default is 'default'). - :param product_name: The product name for retrieving checks from the installation (default is 'dqx'). - :param assume_user: Whether to assume the user is the owner of the checks (default is True). + Args: + location: The installation path where the checks are stored (e.g., table name, file path). + Not used when using installation method, as it is retrieved from the installation config. + run_config_name: The name of the run configuration to use for checks (default is 'default'). + product_name: The product name for retrieving checks from the installation (default is 'dqx'). + assume_user: Whether to assume the user is the owner of the checks (default is True). """ location: str = "installation" # retrieved from the installation config diff --git a/src/databricks/labs/dqx/config_loader.py b/src/databricks/labs/dqx/config_loader.py index c67ba1c2f..b42d7fd0f 100644 --- a/src/databricks/labs/dqx/config_loader.py +++ b/src/databricks/labs/dqx/config_loader.py @@ -18,9 +18,10 @@ def load_run_config( """ Load run configuration from the installation. - :param run_config_name: name of the run configuration to use - :param assume_user: if True, assume user installation - :param product_name: name of the product + Args: + run_config_name: name of the run configuration to use + assume_user: if True, assume user installation + product_name: name of the product """ installation = self.get_installation(assume_user, product_name) return self._load_run_config(installation, run_config_name) @@ -29,8 +30,9 @@ def get_installation(self, assume_user: bool, product_name: str) -> Installation """ Get the installation for the given product name. - :param assume_user: if True, assume user installation - :param product_name: name of the product + Args: + assume_user: if True, assume user installation + product_name: name of the product """ if assume_user: installation = Installation.assume_user_home(self.ws, product_name) @@ -46,8 +48,9 @@ def _load_run_config(installation: Installation, run_config_name: str | None) -> """ Load run configuration from the installation. - :param installation: the installation object - :param run_config_name: name of the run configuration to use + Args: + installation: the installation object + run_config_name: name of the run configuration to use """ config = installation.load(WorkspaceConfig) return config.get_run_config(run_config_name) diff --git a/src/databricks/labs/dqx/contexts/application.py b/src/databricks/labs/dqx/contexts/application.py index 255a349d4..78c98b2ef 100644 --- a/src/databricks/labs/dqx/contexts/application.py +++ b/src/databricks/labs/dqx/contexts/application.py @@ -18,7 +18,8 @@ class GlobalContext(abc.ABC): """ Returns the parent run ID. - :return: The parent run ID as an integer. + Returns: + The parent run ID as an integer. """ def __init__(self, named_parameters: dict[str, str] | None = None): @@ -30,8 +31,11 @@ def replace(self, **kwargs): """ Replace cached properties. - :param kwargs: Key-value pairs of properties to replace. - :return: The updated GlobalContext instance. + Args: + kwargs: Key-value pairs of properties to replace. + + Returns: + The updated GlobalContext instance. """ for key, value in kwargs.items(): self.__dict__[key] = value diff --git a/src/databricks/labs/dqx/contexts/workflows.py b/src/databricks/labs/dqx/contexts/workflows.py index 255846e88..6886941fc 100644 --- a/src/databricks/labs/dqx/contexts/workflows.py +++ b/src/databricks/labs/dqx/contexts/workflows.py @@ -38,8 +38,11 @@ def connect_config(self) -> core.Config: """ Returns the connection configuration. - :return: The core.Config instance. - :raises AssertionError: If the connect configuration is not provided. + Returns: + The core.Config instance. + + Raises: + AssertionError: If the connect configuration is not provided. """ connect = self.config.connect assert connect, "connect is required" diff --git a/src/databricks/labs/dqx/contexts/workspace.py b/src/databricks/labs/dqx/contexts/workspace.py index b1b834573..8d05506ef 100644 --- a/src/databricks/labs/dqx/contexts/workspace.py +++ b/src/databricks/labs/dqx/contexts/workspace.py @@ -9,8 +9,9 @@ class WorkspaceContext(CliContext): """ WorkspaceContext class that extends CliContext to provide workspace-specific functionality. - :param ws: The WorkspaceClient instance to use for accessing the workspace. - :param named_parameters: Optional dictionary of named parameters. + Args: + ws: The WorkspaceClient instance to use for accessing the workspace. + named_parameters: Optional dictionary of named parameters. """ def __init__(self, ws: WorkspaceClient, named_parameters: dict[str, str] | None = None): diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 0f11742e6..af8b80866 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -42,9 +42,10 @@ class DQEngineCore(DQEngineCoreBase): """Core engine to apply data quality checks to a DataFrame. - :param workspace_client: WorkspaceClient instance used to access the workspace. - :param spark: Optional SparkSession to use. If not provided, the active session is used. - :param extra_params: Optional extra parameters for the engine, such as result column names and run metadata. + Args: + workspace_client: WorkspaceClient instance used to access the workspace. + spark: Optional SparkSession to use. If not provided, the active session is used. + extra_params: Optional extra parameters for the engine, such as result column names and run metadata. """ def __init__( @@ -75,10 +76,13 @@ def apply_checks( ) -> DataFrame: """Apply data quality checks to the given DataFrame. - :param df: Input DataFrame to check. - :param checks: List of checks to apply. Each check must be a `DQRule` instance. - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame with errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of checks to apply. Each check must be a `DQRule` instance. + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame with errors and warnings result columns. """ if not checks: return self._append_empty_checks(df) @@ -107,11 +111,14 @@ def apply_checks_and_split( """Apply data quality checks to the given DataFrame and split the results into two DataFrames ("good" and "bad"). - :param df: Input DataFrame to check. - :param checks: List of checks to apply. Each check must be a `DQRule` instance. - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and - "bad" (rows with errors or warnings and the corresponding result columns). + Args: + df: Input DataFrame to check. + checks: List of checks to apply. Each check must be a `DQRule` instance. + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ if not checks: return df, self._append_empty_checks(df).limit(0) @@ -138,15 +145,18 @@ def apply_checks_by_metadata( ) -> DataFrame: """Apply data quality checks defined as metadata to the given DataFrame. - :param df: Input DataFrame to check. - :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame with errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame with errors and warnings result columns. """ dq_rule_checks = deserialize_checks(checks, custom_check_functions) @@ -162,15 +172,18 @@ def apply_checks_by_metadata_and_split( """Apply data quality checks defined as metadata to the given DataFrame and split the results into two DataFrames ("good" and "bad"). - :param df: Input DataFrame to check. - :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames). - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame that includes errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use. + - `name` - Optional name for the resulting column. Auto-generated if not provided. + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames). + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame that includes errors and warnings result columns. """ dq_rule_checks = deserialize_checks(checks, custom_check_functions) @@ -189,10 +202,13 @@ def validate_checks( This method validates the presence of required keys, the existence and callability of functions, and the types of arguments passed to those functions. - :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param validate_custom_check_functions: If True, validate custom check functions. - :return: ChecksValidationStatus indicating the validation result. + Args: + checks: List of checks to apply to the DataFrame. Each check should be a dictionary. + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + validate_custom_check_functions: If True, validate custom check functions. + + Returns: + ChecksValidationStatus indicating the validation result. """ return ChecksValidator.validate_checks(checks, custom_check_functions, validate_custom_check_functions) @@ -200,8 +216,11 @@ def get_invalid(self, df: DataFrame) -> DataFrame: """ Return records that violate data quality checks (rows with warnings or errors). - :param df: Input DataFrame. - :return: DataFrame with rows that have errors or warnings and the corresponding result columns. + Args: + df: Input DataFrame. + + Returns: + DataFrame with rows that have errors or warnings and the corresponding result columns. """ return df.where( F.col(self._result_column_names[ColumnArguments.ERRORS]).isNotNull() @@ -212,8 +231,11 @@ def get_valid(self, df: DataFrame) -> DataFrame: """ Return records that do not violate data quality checks (rows with warnings but no errors). - :param df: Input DataFrame. - :return: DataFrame with warning rows but without the results columns. + Args: + df: Input DataFrame. + + Returns: + DataFrame with warning rows but without the results columns. """ return df.where(F.col(self._result_column_names[ColumnArguments.ERRORS]).isNull()).drop( self._result_column_names[ColumnArguments.ERRORS], self._result_column_names[ColumnArguments.WARNINGS] @@ -226,8 +248,11 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: The returned checks can be used as input to `apply_checks_by_metadata`. - :param filepath: Path to a file containing checks definitions. - :return: List of DQ rules. + Args: + filepath: Path to a file containing checks definitions. + + Returns: + List of DQ rules. """ return FileChecksStorageHandler().load(FileChecksStorageConfig(location=filepath)) @@ -236,8 +261,9 @@ def save_checks_in_local_file(checks: list[dict], filepath: str): """ Save DQ rules (checks) to a local YAML or JSON file. - :param checks: List of DQ rules (checks) to save. - :param filepath: Path to a file where the checks definitions will be saved. + Args: + checks: List of DQ rules (checks) to save. + filepath: Path to a file where the checks definitions will be saved. """ return FileChecksStorageHandler().save(checks, FileChecksStorageConfig(location=filepath)) @@ -245,9 +271,12 @@ def save_checks_in_local_file(checks: list[dict], filepath: str): def _get_check_columns(checks: list[DQRule], criticality: str) -> list[DQRule]: """Get check columns based on criticality. - :param checks: list of checks to apply to the DataFrame - :param criticality: criticality - :return: list of check columns + Args: + checks: list of checks to apply to the DataFrame + criticality: criticality + + Returns: + list of check columns """ return [check for check in checks if check.criticality == criticality] @@ -259,8 +288,11 @@ def _all_are_dq_rules(checks: list[DQRule]) -> bool: def _append_empty_checks(self, df: DataFrame) -> DataFrame: """Append empty checks at the end of DataFrame. - :param df: DataFrame without checks - :return: DataFrame with checks + Args: + df: DataFrame without checks + + Returns: + DataFrame with checks """ return df.select( "*", @@ -279,11 +311,14 @@ def _create_results_array( - Collects the individual check conditions into an array, filtering out empty results. - Adds a new array column that contains only failing checks (if any), or null otherwise. - :param df: The input DataFrame to which checks are applied. - :param checks: List of DQRule instances representing the checks to apply. - :param dest_col: Name of the output column where the check results map will be stored. - :param ref_dfs: Optional dictionary of reference DataFrames, keyed by name, for use by dataset-level checks. - :return: DataFrame with an added array column (`dest_col`) containing the results of the applied checks. + Args: + df: The input DataFrame to which checks are applied. + checks: List of DQRule instances representing the checks to apply. + dest_col: Name of the output column where the check results map will be stored. + ref_dfs: Optional dictionary of reference DataFrames, keyed by name, for use by dataset-level checks. + + Returns: + DataFrame with an added array column (`dest_col`) containing the results of the applied checks. """ if not checks: # No checks then just append a null array result @@ -353,10 +388,13 @@ def apply_checks( ) -> DataFrame: """Apply data quality checks to the given DataFrame. - :param df: Input DataFrame to check. - :param checks: List of checks to apply. Each check must be a `DQRule` instance. - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame with errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of checks to apply. Each check must be a `DQRule` instance. + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame with errors and warnings result columns. """ return self._engine.apply_checks(df, checks, ref_dfs) @@ -366,11 +404,14 @@ def apply_checks_and_split( """Apply data quality checks to the given DataFrame and split the results into two DataFrames ("good" and "bad"). - :param df: Input DataFrame to check. - :param checks: List of checks to apply. Each check must be a `DQRule` instance. - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and - "bad" (rows with errors or warnings and the corresponding result columns). + Args: + df: Input DataFrame to check. + checks: List of checks to apply. Each check must be a `DQRule` instance. + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + A tuple of two DataFrames: "good" (may include rows with warnings but no result columns) and + "bad" (rows with errors or warnings and the corresponding result columns). """ return self._engine.apply_checks_and_split(df, checks, ref_dfs) @@ -383,15 +424,18 @@ def apply_checks_by_metadata( ) -> DataFrame: """Apply data quality checks defined as metadata to the given DataFrame. - :param df: Input DataFrame to check. - :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame with errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame with errors and warnings result columns. """ return self._engine.apply_checks_by_metadata(df, checks, custom_check_functions, ref_dfs) @@ -405,15 +449,18 @@ def apply_checks_by_metadata_and_split( """Apply data quality checks defined as metadata to the given DataFrame and split the results into two DataFrames ("good" and "bad"). - :param df: Input DataFrame to check. - :param checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param ref_dfs: Optional reference DataFrames to use in the checks. - :return: DataFrame that includes errors and warnings result columns. + Args: + df: Input DataFrame to check. + checks: List of dictionaries describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + ref_dfs: Optional reference DataFrames to use in the checks. + + Returns: + DataFrame that includes errors and warnings result columns. """ return self._engine.apply_checks_by_metadata_and_split(df, checks, custom_check_functions, ref_dfs) @@ -434,11 +481,12 @@ def apply_checks_and_save_in_table( If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. - :param checks: List of `DQRule` checks to apply. - :param input_config: Input configuration (e.g., table/view or file location and read options). - :param output_config: Output configuration (e.g., table name, mode, and write options). - :param quarantine_config: Optional configuration for writing invalid records. - :param ref_dfs: Optional reference DataFrames used by checks. + Args: + checks: List of `DQRule` checks to apply. + input_config: Input configuration (e.g., table/view or file location and read options). + output_config: Output configuration (e.g., table name, mode, and write options). + quarantine_config: Optional configuration for writing invalid records. + ref_dfs: Optional reference DataFrames used by checks. """ # Read data from the specified table df = read_input_data(self.spark, input_config) @@ -471,15 +519,18 @@ def apply_checks_by_metadata_and_save_in_table( If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. - :param checks: List of dicts describing checks. Each item must include 'check' (function name plus - arguments) and may include 'name' (result column; auto-generated if omitted) and 'criticality' - ('error' or 'warn'). - :param input_config: Input configuration (e.g., table/view or file location and read options). - :param output_config: Output configuration (e.g., table name, mode, and write options). - :param quarantine_config: Optional configuration for writing invalid records. - :param custom_check_functions: Optional mapping of custom check function names - to callables/modules (e.g., globals()). - :param ref_dfs: Optional reference DataFrames used by checks. + Args: + checks: List of dicts describing checks. Each check dictionary must contain the following: + - `check` - A check definition including check function and arguments to use; + - `name` - Optional name for the resulting column. Auto-generated if not provided; + - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + (rows appear in both DataFrames); + input_config: Input configuration (e.g., table/view or file location and read options). + output_config: Output configuration (e.g., table name, mode, and write options). + quarantine_config: Optional configuration for writing invalid records. + custom_check_functions: Optional mapping of custom check function names + to callables/modules (e.g., globals()). + ref_dfs: Optional reference DataFrames used by checks. """ # Read data from the specified table df = read_input_data(self.spark, input_config) @@ -506,10 +557,13 @@ def validate_checks( This method validates the presence of required keys, the existence and callability of functions, and the types of arguments passed to those functions. - :param checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - :param custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). - :param validate_custom_check_functions: If True, validate custom check functions. - :return: ChecksValidationStatus indicating the validation result. + Args: + checks: List of checks to apply to the DataFrame. Each check should be a dictionary. + custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + validate_custom_check_functions: If True, validate custom check functions. + + Returns: + ChecksValidationStatus indicating the validation result. """ return DQEngineCore.validate_checks(checks, custom_check_functions, validate_custom_check_functions) @@ -517,8 +571,11 @@ def get_invalid(self, df: DataFrame) -> DataFrame: """ Return records that violate data quality checks (rows with warnings or errors). - :param df: Input DataFrame. - :return: DataFrame with rows that have errors or warnings and the corresponding result columns. + Args: + df: Input DataFrame. + + Returns: + DataFrame with rows that have errors or warnings and the corresponding result columns. """ return self._engine.get_invalid(df) @@ -526,8 +583,11 @@ def get_valid(self, df: DataFrame) -> DataFrame: """ Return records that do not violate data quality checks (rows with warnings but no errors). - :param df: Input DataFrame. - :return: DataFrame with warning rows but without the results columns. + Args: + df: Input DataFrame. + + Returns: + DataFrame with warning rows but without the results columns. """ return self._engine.get_valid(df) @@ -548,14 +608,17 @@ def save_results_in_table( - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`. - A write occurs only when both a DataFrame and its corresponding config are available. - :param output_df: DataFrame with valid rows to be saved (optional). - :param quarantine_df: DataFrame with invalid rows to be saved (optional). - :param output_config: Configuration describing where/how to write the valid rows. If omitted, falls back to the run config. - :param quarantine_config: Configuration describing where/how to write the invalid rows. If omitted, falls back to the run config. - :param run_config_name: Name of the run configuration to load when a config parameter is omitted. - :param product_name: Product/installation identifier used to resolve installation paths for config loading. - :param assume_user: Whether to assume a per-user installation when loading the run configuration. - :return: None + Args: + output_df: DataFrame with valid rows to be saved (optional). + quarantine_df: DataFrame with invalid rows to be saved (optional). + output_config: Configuration describing where/how to write the valid rows. If omitted, falls back to the run config. + quarantine_config: Configuration describing where/how to write the invalid rows. If omitted, falls back to the run config. + run_config_name: Name of the run configuration to load when a config parameter is omitted. + product_name: Product/installation identifier used to resolve installation paths for config loading. + assume_user: Whether to assume a per-user installation when loading the run configuration. + + Returns: + None """ if output_df is not None and output_config is None: run_config = self._run_config_loader.load_run_config(run_config_name, assume_user, product_name) @@ -585,9 +648,14 @@ def load_checks(self, config: BaseChecksStorageConfig) -> list[dict]: - `InstallationChecksStorageConfig` (installation directory); - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); - :param config: Configuration object describing the storage backend. - :return: List of DQ rules (checks) represented as dictionaries. - :raises ValueError: If the configuration type is unsupported. + Args: + config: Configuration object describing the storage backend. + + Returns: + List of DQ rules (checks) represented as dictionaries. + + Raises: + ValueError: If the configuration type is unsupported. """ handler = self._checks_handler_factory.create(config) return handler.load(config) @@ -607,10 +675,15 @@ def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> No - `InstallationChecksStorageConfig` (installation directory); - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); - :param checks: List of DQ rules (checks) to save (as dictionaries). - :param config: Configuration object describing the storage backend and write options. - :return: None - :raises ValueError: If the configuration type is unsupported. + Args: + checks: List of DQ rules (checks) to save (as dictionaries). + config: Configuration object describing the storage backend and write options. + + Returns: + None + + Raises: + ValueError: If the configuration type is unsupported. """ handler = self._checks_handler_factory.create(config) handler.save(checks, config) @@ -624,8 +697,11 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: Load DQ rules (checks) from a local JSON or YAML file. - :param filepath: Path to a file containing checks definitions. - :return: List of DQ rules (checks) represented as dictionaries. + Args: + filepath: Path to a file containing checks definitions. + + Returns: + List of DQ rules (checks) represented as dictionaries. """ warnings.warn( "Use `load_checks` method with `FileChecksStorageConfig` instead. " @@ -641,9 +717,12 @@ def save_checks_in_local_file(checks: list[dict], path: str): Save DQ rules (checks) to a local YAML or JSON file. - :param checks: List of DQ rules (checks) to save. - :param path: File path where the checks definitions will be saved. - :return: None + Args: + checks: List of DQ rules (checks) to save. + path: File path where the checks definitions will be saved. + + Returns: + None """ warnings.warn( "Use `save_checks` method with `FileChecksStorageConfig` instead. " @@ -658,8 +737,11 @@ def load_checks_from_workspace_file(self, workspace_path: str) -> list[dict]: Load checks stored in a Databricks workspace file. - :param workspace_path: Path to the workspace file containing checks definitions. - :return: List of DQ rules (checks) represented as dictionaries. + Args: + workspace_path: Path to the workspace file containing checks definitions. + + Returns: + List of DQ rules (checks) represented as dictionaries. """ warnings.warn( "Use `load_checks` method with `WorkspaceFileChecksStorageConfig` instead. " @@ -674,9 +756,12 @@ def save_checks_in_workspace_file(self, checks: list[dict], workspace_path: str) Save checks to a Databricks workspace file. - :param checks: List of DQ rules (checks) to save. - :param workspace_path: Path to the workspace file where checks will be saved. - :return: None + Args: + checks: List of DQ rules (checks) to save. + workspace_path: Path to the workspace file where checks will be saved. + + Returns: + None """ warnings.warn( "Use `save_checks` method with `WorkspaceFileChecksStorageConfig` instead. " @@ -691,9 +776,12 @@ def load_checks_from_table(self, table_name: str, run_config_name: str = "defaul Load checks from a table. - :param table_name: Fully qualified table name where checks are stored. - :param run_config_name: Name of the run configuration (used by the storage handler if needed). - :return: List of DQ rules (checks) represented as dictionaries. + Args: + table_name: Fully qualified table name where checks are stored. + run_config_name: Name of the run configuration (used by the storage handler if needed). + + Returns: + List of DQ rules (checks) represented as dictionaries. """ warnings.warn( "Use `load_checks` method with `TableChecksStorageConfig` instead. " @@ -710,11 +798,14 @@ def save_checks_in_table( Save checks to a table. - :param checks: List of DQ rules (checks) to save. - :param table_name: Fully qualified table name where checks will be written. - :param run_config_name: Name of the run configuration (used by the storage handler if needed). - :param mode: Write mode, e.g., "append" or "overwrite". - :return: None + Args: + checks: List of DQ rules (checks) to save. + table_name: Fully qualified table name where checks will be written. + run_config_name: Name of the run configuration (used by the storage handler if needed). + mode: Write mode, e.g., "append" or "overwrite". + + Returns: + None """ warnings.warn( "Use `save_checks` method with `TableChecksStorageConfig` instead. " @@ -737,11 +828,14 @@ def load_checks_from_installation( Load checks from the installation directory. - :param run_config_name: Named run configuration to resolve installation paths and defaults. - :param method: Deprecated parameter; ignored. - :param product_name: Product/installation identifier (e.g., "dqx"). - :param assume_user: Whether to assume a per-user installation layout. - :return: List of DQ rules (checks) represented as dictionaries. + Args: + run_config_name: Named run configuration to resolve installation paths and defaults. + method: Deprecated parameter; ignored. + product_name: Product/installation identifier (e.g., "dqx"). + assume_user: Whether to assume a per-user installation layout. + + Returns: + List of DQ rules (checks) represented as dictionaries. """ warnings.warn( "Use `load_checks` method with `InstallationChecksStorageConfig` instead. " @@ -768,12 +862,15 @@ def save_checks_in_installation( Save checks to the installation directory. - :param checks: List of DQ rules (checks) to save. - :param run_config_name: Named run configuration to resolve installation paths and defaults. - :param method: Deprecated parameter; ignored. - :param product_name: Product/installation identifier (e.g., "dqx"). - :param assume_user: Whether to assume a per-user installation layout. - :return: None + Args: + checks: List of DQ rules (checks) to save. + run_config_name: Named run configuration to resolve installation paths and defaults. + method: Deprecated parameter; ignored. + product_name: Product/installation identifier (e.g., "dqx"). + assume_user: Whether to assume a per-user installation layout. + + Returns: + None """ warnings.warn( "Use `save_checks` method with `InstallationChecksStorageConfig` instead. " @@ -796,10 +893,13 @@ def load_run_config( Load a run configuration by name. This wrapper will be removed in a future version. - :param run_config_name: Name of the run configuration to load. - :param assume_user: Whether to assume a per-user installation when resolving paths. - :param product_name: Product/installation identifier (e.g., "dqx"). - :return: Loaded `RunConfig` instance. + Args: + run_config_name: Name of the run configuration to load. + assume_user: Whether to assume a per-user installation when resolving paths. + product_name: Product/installation identifier (e.g., "dqx"). + + Returns: + Loaded `RunConfig` instance. """ warnings.warn( "Use `RunConfigLoader.load_run_config` method instead. This method will be removed in future versions.", diff --git a/src/databricks/labs/dqx/executor.py b/src/databricks/labs/dqx/executor.py index b10630c50..1bb5a8c5d 100644 --- a/src/databricks/labs/dqx/executor.py +++ b/src/databricks/labs/dqx/executor.py @@ -74,12 +74,15 @@ def apply(self, df: DataFrame, spark: SparkSession, ref_dfs: dict[str, DataFrame or violates the check. The condition contains a message when the check fails, or null when it passes. - :param df: The input DataFrame to which the rule is applied. - :param spark: The SparkSession used for executing the rule (unused for row rules). - :param ref_dfs: Optional dictionary of reference DataFrames (unused for row rules). - :return: DQCheckResult containing: - - condition: Spark Column representing the check condition. - - check_df: The input (main) DataFrame (used for downstream processing). + Args: + df: The input DataFrame to which the rule is applied. + spark: The SparkSession used for executing the rule (unused for row rules). + ref_dfs: Optional dictionary of reference DataFrames (unused for row rules). + + Returns: + DQCheckResult containing: + - condition: Spark Column representing the check condition. + - check_df: The input (main) DataFrame (used for downstream processing). """ condition = self.rule.check return DQCheckResult(condition=condition, check_df=df) @@ -116,12 +119,15 @@ def apply(self, df: DataFrame, spark: SparkSession, ref_dfs: dict[str, DataFrame - A condition (Spark Column) that represents the overall dataset check status. - A DataFrame with the results of the dataset-level evaluation. - :param df: The input DataFrame to which the rule is applied. - :param spark: The SparkSession used for executing the rule. - :param ref_dfs: Optional dictionary of reference DataFrames for dataset-level checks. - :return: DQCheckResult containing: - - condition: Spark Column representing the check condition. - - check_df: DataFrame produced by the check, containing evaluation results. + Args: + df: The input DataFrame to which the rule is applied. + spark: The SparkSession used for executing the rule. + ref_dfs: Optional dictionary of reference DataFrames for dataset-level checks. + + Returns: + DQCheckResult containing: + - condition: Spark Column representing the check condition. + - check_df: DataFrame produced by the check, containing evaluation results. """ condition, closure_func = self.rule.check diff --git a/src/databricks/labs/dqx/installer/install.py b/src/databricks/labs/dqx/installer/install.py index dd4a8caff..5c505a63f 100644 --- a/src/databricks/labs/dqx/installer/install.py +++ b/src/databricks/labs/dqx/installer/install.py @@ -59,8 +59,9 @@ class WorkspaceInstaller(WorkspaceContext): """ Installer for DQX workspace. - :param ws: The WorkspaceClient instance. - :param environ: Optional dictionary of environment variables. + Args: + environ: Optional dictionary of environment variables. + ws: The WorkspaceClient instance. """ def __init__(self, ws: WorkspaceClient, environ: dict[str, str] | None = None): @@ -79,7 +80,8 @@ def upgrades(self): """ Returns the Upgrades instance for the product. - :return: An Upgrades instance. + Returns: + An Upgrades instance. """ return Upgrades(self.product_info, self.installation) @@ -88,8 +90,11 @@ def installation(self): """ Returns the current installation for the product. - :return: An Installation instance. - :raises NotFound: If the installation is not found. + Returns: + An Installation instance. + + Raises: + NotFound: If the installation is not found. """ try: return self.product_info.current_installation(self.workspace_client) @@ -105,10 +110,15 @@ def run( """ Runs the installation process. - :param default_config: Optional default configuration. - :return: The final WorkspaceConfig used for the installation. - :raises ManyError: If multiple errors occur during installation. - :raises TimeoutError: If a timeout occurs during installation. + Args: + default_config: Optional default configuration. + + Returns: + The final WorkspaceConfig used for the installation. + + Raises: + ManyError: If multiple errors occur during installation. + TimeoutError: If a timeout occurs during installation. """ logger.info(f"Installing DQX v{self.product_info.version()}") try: @@ -154,8 +164,11 @@ def extract_major_minor(version_string: str): """ Extracts the major and minor version from a version string. - :param version_string: The version string to extract from. - :return: The major.minor version as a string, or None if not found. + Args: + version_string: The version string to extract from. + + Returns: + The major.minor version as a string, or None if not found. """ match = re.search(r"(\d+\.\d+)", version_string) if match: @@ -417,12 +430,17 @@ def configure(self, default_config: WorkspaceConfig | None = None) -> WorkspaceC Configures the workspace. Notes: - 1. Connection errors are not handled within this configure method. + * Connection errors are not handled within this configure method. - :param default_config: Optional default configuration. - :return: The final WorkspaceConfig used for the installation. - :raises NotFound: If the previous installation is not found. - :raises RuntimeWarning: If the existing installation is corrupted. + Args: + default_config: Optional default configuration. + + Returns: + The final WorkspaceConfig used for the installation. + + Raises: + NotFound: If the previous installation is not found. + RuntimeWarning: If the existing installation is corrupted. """ try: config = self.installation.load(WorkspaceConfig) @@ -519,8 +537,11 @@ def current(cls, ws: WorkspaceClient): """ Creates a current WorkspaceInstallation instance based on the current workspace client. - :param ws: The WorkspaceClient instance. - :return: A WorkspaceInstallation instance. + Args: + ws: The WorkspaceClient instance. + + Returns: + A WorkspaceInstallation instance. """ product_info = ProductInfo.from_class(WorkspaceConfig) installation = product_info.current_installation(ws) @@ -549,7 +570,8 @@ def config(self): """ Returns the configuration of the workspace installation. - :return: The WorkspaceConfig instance. + Returns: + The WorkspaceConfig instance. """ return self._config @@ -558,7 +580,8 @@ def folder(self): """ Returns the installation folder path. - :return: The installation folder path as a string. + Returns: + The installation folder path as a string. """ return self._installation.install_folder() @@ -574,7 +597,8 @@ def run(self) -> bool: """ Runs the workflow installation. - :return: True if the installation finished successfully, False otherwise. + Returns: + True if the installation finished successfully, False otherwise. """ logger.info(f"Installing DQX v{self._product_info.version()}") install_tasks = [self._workflows_installer.create_jobs, self._create_dq_dashboard] diff --git a/src/databricks/labs/dqx/installer/workflow_task.py b/src/databricks/labs/dqx/installer/workflow_task.py index b831b12bf..af8df1515 100644 --- a/src/databricks/labs/dqx/installer/workflow_task.py +++ b/src/databricks/labs/dqx/installer/workflow_task.py @@ -103,7 +103,11 @@ def remove_extra_indentation(doc: str) -> str: """ Remove extra indentation from docstring. - :param doc: Docstring + Args: + doc (str): Docstring to process. + + Returns: + str: Processed docstring with extra indentation removed. """ lines = doc.splitlines() stripped = [] diff --git a/src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py b/src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py index f7b96bdbc..b7f0c78ae 100644 --- a/src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py +++ b/src/databricks/labs/dqx/llm/extract_yaml_checks_examples.py @@ -16,11 +16,15 @@ def extract_yaml_checks_from_content(content: str, source_name: str = "content") -> list[dict[str, Any]]: - """Extract all YAML examples from MDX content string. + """ + Extract all YAML examples from MDX content string. + + Args: + content: The MDX content string to extract YAML from + source_name: Name of the source for logging purposes (default: "content") - :param content: The MDX content string to extract YAML from - :param source_name: Name of the source for logging purposes (default: "content") - :return: List of parsed YAML objects from all valid blocks + Returns: + List of parsed YAML objects from all valid blocks """ # Extract YAML from code blocks @@ -62,11 +66,17 @@ def extract_yaml_checks_from_content(content: str, source_name: str = "content") def extract_yaml_checks_from_mdx(mdx_file_path: str) -> list[dict[str, Any]]: - """Extract all YAML examples from a given MDX file. + """ + Extract all YAML examples from a given MDX file. + + Args: + mdx_file_path: Path to the MDX file to extract YAML from - :param mdx_file_path: Path to the MDX file to extract YAML from - :raises FileNotFoundError: If the MDX file does not exist - :return: List of parsed YAML objects from all valid blocks + Returns: + List of parsed YAML objects from all valid blocks + + Raises: + FileNotFoundError: If the MDX file does not exist """ mdx_file = Path(mdx_file_path) @@ -82,12 +92,17 @@ def extract_yaml_checks_from_mdx(mdx_file_path: str) -> list[dict[str, Any]]: def extract_yaml_checks_examples(output_file_path: Path | None = None) -> bool: - """Extract all YAML examples from both quality_rules.mdx and quality_checks.mdx. + """ + Extract all YAML examples from both quality_rules.mdx and quality_checks.mdx. Creates a combined YAML file with all examples from the documentation files in the LLM resources directory for use in language model processing. - :return: True if extraction was successful, False otherwise + Args: + output_file_path: Path to the output file to write the combined YAML content + + Returns: + True if extraction was successful, False otherwise """ all_combined_content = [] success_count = 0 diff --git a/src/databricks/labs/dqx/llm/utils.py b/src/databricks/labs/dqx/llm/utils.py index 3c1d4e92f..33ab98b3d 100644 --- a/src/databricks/labs/dqx/llm/utils.py +++ b/src/databricks/labs/dqx/llm/utils.py @@ -13,15 +13,18 @@ def get_check_function_definition(custom_check_functions: dict[str, Any] | None = None) -> list[dict[str, str]]: - """A utility function to get the definition of all check functions. + """ + A utility function to get the definition of all check functions. This function is primarily used to generate a prompt for the LLM to generate check functions. - :param custom_check_functions: A dictionary of custom check functions. - If provided, the function will use the custom check functions to resolve the check function. - If not provided, the function will use only the built-in check functions. + If provided, the function will use the custom check functions to resolve the check function. + If not provided, the function will use only the built-in check functions. + + Args: + custom_check_functions: A dictionary of custom check functions. Returns: - list[dict]: A list of dictionaries, each containing the definition of a check function. + list[dict]: A list of dictionaries, each containing the definition of a check function. """ function_docs: list[dict[str, str]] = [] for name, func_type in CHECK_FUNC_REGISTRY.items(): @@ -45,9 +48,11 @@ def get_check_function_definition(custom_check_functions: dict[str, Any] | None def load_yaml_checks_examples() -> str: - """Load yaml_checks_examples.yml file from the llm/resources folder. + """ + Load yaml_checks_examples.yml file from the llm/resources folder. - :return: checks examples as yaml string. + Returns: + checks examples as yaml string. """ resource = Path(str(files("databricks.labs.dqx.llm.resources") / "yaml_checks_examples.yml")) diff --git a/src/databricks/labs/dqx/pii/pii_detection_funcs.py b/src/databricks/labs/dqx/pii/pii_detection_funcs.py index 997fb4fac..e3fd7dccc 100644 --- a/src/databricks/labs/dqx/pii/pii_detection_funcs.py +++ b/src/databricks/labs/dqx/pii/pii_detection_funcs.py @@ -35,14 +35,17 @@ def does_not_contain_pii( entities (e.g. PERSON, ADDRESS, EMAIL_ADDRESS). If PII is detected, the message includes a JSON string with the entity types, location within the string, and confidence score from the model. - :param column: Column to check; can be a string column name or a column expression - :param language: Optional language of the text (default: 'en') - :param threshold: Confidence threshold for PII detection (0.0 to 1.0, default: 0.7) - Higher values = less sensitive, fewer false positives - Lower values = more sensitive, more potential false positives - :param entities: Optional list of entities to detect - :param nlp_engine_config: Optional NLP engine configuration used for PII detection; Can be `dict` or `NLPEngineConfiguration` - :return: Column object for condition that fails when PII is detected + Args: + column: Column to check; can be a string column name or a column expression + language: Optional language of the text (default: 'en') + threshold: Confidence threshold for PII detection (0.0 to 1.0, default: 0.7) + Higher values = less sensitive, fewer false positives + Lower values = more sensitive, more potential false positives + entities: Optional list of entities to detect + nlp_engine_config: Optional NLP engine configuration used for PII detection; Can be `dict` or `NLPEngineConfiguration` + + Returns: + Column object for condition that fails when PII is detected """ warnings.warn( "PII detection uses pandas user-defined functions which may degrade performance. " @@ -95,11 +98,14 @@ def _build_detection_udf( """ Builds a UDF with the provided threshold, entities, language, and analyzer. - :param nlp_engine_config: Dictionary configuring the NLP engine used for PII detection - :param language: Language of the text - :param threshold: Confidence threshold for named entity detection (0.0 to 1.0) - :param entities: List of entities to detect - :return: PySpark UDF which can be called to detect PII with the given configuration + Args: + nlp_engine_config: Dictionary configuring the NLP engine used for PII detection + language: Language of the text + threshold: Confidence threshold for named entity detection (0.0 to 1.0) + entities: List of entities to detect + + Returns: + PySpark UDF which can be called to detect PII with the given configuration """ @pandas_udf("string") # type: ignore[call-overload] @@ -108,7 +114,8 @@ def _get_analyzer() -> AnalyzerEngine: """ Gets an `AnalyzerEngine` for use with PII detection checks. - :return: Presidio `AnalyzerEngine` + Returns: + Presidio `AnalyzerEngine` """ provider = NlpEngineProvider(nlp_configuration=nlp_engine_config) nlp_engine = provider.create_engine() @@ -120,8 +127,11 @@ def _detect_named_entities(text: str) -> str | None: """ Detects named entities in the input text using a Presidio analyzer. - :param text: Input text to analyze for named entities - :return: JSON string with detected entities, or `None` if no entities are found + Args: + text: Input text to analyze for named entities + + Returns: + JSON string with detected entities, or `None` if no entities are found """ if not text: return None diff --git a/src/databricks/labs/dqx/profiler/common.py b/src/databricks/labs/dqx/profiler/common.py index 77d65f28f..07f72442c 100644 --- a/src/databricks/labs/dqx/profiler/common.py +++ b/src/databricks/labs/dqx/profiler/common.py @@ -8,9 +8,12 @@ def val_to_str(value: Any, include_sql_quotes: bool = True): """ Converts a value to a string. - :param value: The value to convert. Can be a datetime, date, int, float, or other type. - :param include_sql_quotes: Whether to include quotes around the value. Default is True. - :return: The string representation of the value + Args: + value: The value to convert. Can be a datetime, date, int, float, or other type. + include_sql_quotes: Whether to include quotes around the value. Default is True. + + Returns: + The string representation of the value """ quote = "'" if include_sql_quotes else "" if isinstance(value, datetime.datetime): @@ -29,9 +32,12 @@ def val_maybe_to_str(value: Any, include_sql_quotes: bool = True): """ Converts a value to a string if it is a datetime or date. - :param value: The value to convert. Can be a datetime, date, or other type. - :param include_sql_quotes: Whether to include quotes around the value. Default is True. - :return: The string representation of the value. + Args: + value: The value to convert. Can be a datetime, date, or other type. + include_sql_quotes: Whether to include quotes around the value. Default is True. + + Returns: + The string representation of the value. """ quote = "'" if include_sql_quotes else "" if isinstance(value, datetime.datetime): diff --git a/src/databricks/labs/dqx/profiler/dlt_generator.py b/src/databricks/labs/dqx/profiler/dlt_generator.py index 85984428c..cc8dc49d4 100644 --- a/src/databricks/labs/dqx/profiler/dlt_generator.py +++ b/src/databricks/labs/dqx/profiler/dlt_generator.py @@ -17,12 +17,17 @@ def generate_dlt_rules( """ Generates Lakeflow Pipelines (formerly Delta Live Table (DLT)) rules in the specified language. - :param rules: A list of data quality profiles to generate rules for. - :param action: The action to take on rule violation (e.g., "drop", "fail"). - :param language: The language to generate the rules in ("SQL", "Python" or "Python_Dict"). - :return: A list of strings representing the Lakeflow Pipelines rules in SQL, a string representing - the Lakeflow Pipelines rules in Python, or dictionary with expressions. - :raises ValueError: If the specified language is not supported. + Args: + rules: A list of data quality profiles to generate rules for. + action: The action to take on rule violation (e.g., "drop", "fail"). + language: The language to generate the rules in ("SQL", "Python" or "Python_Dict"). + + Returns: + A list of strings representing the Lakeflow Pipelines rules in SQL, a string representing + the Lakeflow Pipelines rules in Python, or dictionary with expressions. + + Raises: + ValueError: If the specified language is not supported. """ lang = language.lower() @@ -44,9 +49,12 @@ def _dlt_generate_is_in(column: str, **params: dict): Generates a Lakeflow Pipelines (formerly Delta Live Table (DLT)) rule to check if a column's value is in a specified list. - :param column: The name of the column to check. - :param params: Additional parameters, including the list of values to check against. - :return: A string representing the Lakeflow Pipelines rule. + Args: + column: The name of the column to check. + params: Additional parameters, including the list of values to check against. + + Returns: + A string representing the Lakeflow Pipelines rule. """ in_str = ", ".join([val_to_str(v) for v in params["in"]]) return f"{column} in ({in_str})" @@ -57,9 +65,12 @@ def _dlt_generate_min_max(column: str, **params: dict): Generates a Lakeflow Pipelines (formerly Delta Live Table (DLT)) rule to check if a column's value is within a specified range. - :param column: The name of the column to check. - :param params: Additional parameters, including the minimum and maximum values. - :return: A string representing the Lakeflow Pipelines rule. + Args: + column: The name of the column to check. + params: Additional parameters, including the minimum and maximum values. + + Returns: + A string representing the Lakeflow Pipelines rule. """ min_limit = params.get("min") max_limit = params.get("max") @@ -82,9 +93,12 @@ def _dlt_generate_is_not_null_or_empty(column: str, **params: dict): Generates a Lakeflow Pipelines (formerly Delta Live Table (DLT)) rule to check if a column's value is not null or empty. - :param column: The name of the column to check. - :param params: Additional parameters, including whether to trim strings. - :return: A string representing the Lakeflow Pipelines rule. + Args: + column: The name of the column to check. + params: Additional parameters, including whether to trim strings. + + Returns: + A string representing the Lakeflow Pipelines rule. """ trim_strings = params.get("trim_strings", True) msg = f"{column} is not null and " @@ -107,8 +121,11 @@ def _generate_dlt_rules_python_dict(self, rules: list[DQProfile]) -> dict: """ Generates a Lakeflow Pipeline (formerly Delta Live Table (DLT)) rules as Python dictionary. - :param rules: A list of data quality profiles to generate rules for. - :return: A dict representing the Lakeflow Pipelines rules in Python. + Args: + rules: A list of data quality profiles to generate rules for. + + Returns: + A dict representing the Lakeflow Pipelines rules in Python. """ expectations = {} for rule in rules or []: @@ -132,9 +149,12 @@ def _generate_dlt_rules_python(self, rules: list[DQProfile], action: str | None """ Generates a Lakeflow Pipeline (formerly Delta Live Table (DLT)) rules in Python. - :param rules: A list of data quality profiles to generate rules for. - :param action: The action to take on rule violation (e.g., "drop", "fail"). - :return: A string representing the Lakeflow Pipelines rules in Python. + Args: + rules: A list of data quality profiles to generate rules for. + action: The action to take on rule violation (e.g., "drop", "fail"). + + Returns: + A string representing the Lakeflow Pipelines rules in Python. """ expectations = self._generate_dlt_rules_python_dict(rules) @@ -157,10 +177,15 @@ def _generate_dlt_rules_sql(self, rules: list[DQProfile], action: str | None = N """ Generates a Lakeflow Pipeline (formerly Delta Live Table (DLT)) rules in sql. - :param rules: A list of data quality profiles to generate rules for. - :param action: The action to take on rule violation (e.g., "drop", "fail"). - :return: A list of Lakeflow Pipelines rules. - :raises ValueError: If the specified language is not supported. + Args: + rules: A list of data quality profiles to generate rules for. + action: The action to take on rule violation (e.g., "drop", "fail"). + + Returns: + A list of Lakeflow Pipelines rules. + + Raises: + ValueError: If the specified language is not supported. """ dlt_rules = [] act_str = "" diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index 22b445135..0e0e0a4ca 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -13,9 +13,12 @@ def generate_dq_rules(self, profiles: list[DQProfile] | None = None, level: str """ Generates a list of data quality rules based on the provided dq profiles. - :param profiles: A list of data quality profiles to generate rules for. - :param level: The criticality level of the rules (default is "error"). - :return: A list of dictionaries representing the data quality rules. + Args: + profiles: A list of data quality profiles to generate rules for. + level: The criticality level of the rules (default is "error"). + + Returns: + A list of dictionaries representing the data quality rules. """ if profiles is None: profiles = [] @@ -41,10 +44,13 @@ def dq_generate_is_in(column: str, level: str = "error", **params: dict): """ Generates a data quality rule to check if a column's value is in a specified list. - :param column: The name of the column to check. - :param level: The criticality level of the rule (default is "error"). - :param params: Additional parameters, including the list of values to check against. - :return: A dictionary representing the data quality rule. + Args: + column: The name of the column to check. + level: The criticality level of the rule (default is "error"). + params: Additional parameters, including the list of values to check against. + + Returns: + A dictionary representing the data quality rule. """ return { "check": {"function": "is_in_list", "arguments": {"column": column, "allowed": params["in"]}}, @@ -57,10 +63,13 @@ def dq_generate_min_max(column: str, level: str = "error", **params: dict): """ Generates a data quality rule to check if a column's value is within a specified range. - :param column: The name of the column to check. - :param level: The criticality level of the rule (default is "error"). - :param params: Additional parameters, including the minimum and maximum values. - :return: A dictionary representing the data quality rule, or None if no limits are provided. + Args: + column: The name of the column to check. + level: The criticality level of the rule (default is "error"). + params: Additional parameters, including the minimum and maximum values. + + Returns: + A dictionary representing the data quality rule, or None if no limits are provided. """ min_limit = params.get("min") max_limit = params.get("max") @@ -115,10 +124,13 @@ def dq_generate_is_not_null(column: str, level: str = "error", **params: dict): """ Generates a data quality rule to check if a column's value is not null. - :param column: The name of the column to check. - :param level: The criticality level of the rule (default is "error"). - :param params: Additional parameters. - :return: A dictionary representing the data quality rule. + Args: + column: The name of the column to check. + level: The criticality level of the rule (default is "error"). + params: Additional parameters. + + Returns: + A dictionary representing the data quality rule. """ params = params or {} return { @@ -132,10 +144,13 @@ def dq_generate_is_not_null_or_empty(column: str, level: str = "error", **params """ Generates a data quality rule to check if a column's value is not null or empty. - :param column: The name of the column to check. - :param level: The criticality level of the rule (default is "error"). - :param params: Additional parameters, including whether to trim strings. - :return: A dictionary representing the data quality rule. + Args: + column: The name of the column to check. + level: The criticality level of the rule (default is "error"). + params: Additional parameters, including whether to trim strings. + + Returns: + A dictionary representing the data quality rule. """ return { "check": { diff --git a/src/databricks/labs/dqx/profiler/profiler.py b/src/databricks/labs/dqx/profiler/profiler.py index 1cd604610..a1d54f0fc 100644 --- a/src/databricks/labs/dqx/profiler/profiler.py +++ b/src/databricks/labs/dqx/profiler/profiler.py @@ -54,8 +54,11 @@ def get_columns_or_fields(columns: list[T.StructField]) -> list[T.StructField]: """ Extracts all fields from a list of StructField objects, including nested fields from StructType columns. - :param columns: A list of StructField objects to process. - :return: A list of StructField objects, including nested fields with prefixed names. + Args: + columns: A list of StructField objects to process. + + Returns: + A list of StructField objects, including nested fields with prefixed names. """ out_columns = [] for column in columns: @@ -74,10 +77,13 @@ def profile( """ Profiles a DataFrame to generate summary statistics and data quality rules. - :param df: The DataFrame to profile. - :param columns: An optional list of column names to include in the profile. If None, all columns are included. - :param options: An optional dictionary of options for profiling. - :return: A tuple containing a dictionary of summary statistics and a list of data quality profiles. + Args: + df: The DataFrame to profile. + columns: An optional list of column names to include in the profile. If None, all columns are included. + options: An optional dictionary of options for profiling. + + Returns: + A tuple containing a dictionary of summary statistics and a list of data quality profiles. """ columns = columns or df.columns df_columns = [f for f in df.schema.fields if f.name in columns] @@ -108,10 +114,13 @@ def profile_table( """ Profiles a table to generate summary statistics and data quality rules. - :param table: The fully-qualified table name (`catalog.schema.table`) to be profiled - :param columns: An optional list of column names to include in the profile. If None, all columns are included. - :param options: An optional dictionary of options for profiling. - :return: A tuple containing a dictionary of summary statistics and a list of data quality profiles. + Args: + table: The fully-qualified table name (`catalog.schema.table`) to be profiled + columns: An optional list of column names to include in the profile. If None, all columns are included. + options: An optional dictionary of options for profiling. + + Returns: + A tuple containing a dictionary of summary statistics and a list of data quality profiles. """ logger.info(f"Profiling {table} with options: {options}") df = read_input_data(spark=self.spark, input_config=InputConfig(location=table)) @@ -128,16 +137,19 @@ def profile_tables( """ Profiles Delta tables in Unity Catalog to generate summary statistics and data quality rules. - :param tables: An optional list of table names to include. - :param patterns: An optional list of table names or filesystem-style wildcards (e.g. 'schema.*') to include. - If None, all tables are included. By default, tables matching the pattern are included. - :param exclude_matched: Specifies whether to include tables matched by the pattern. If True, matched tables - are excluded. If False, matched tables are included. - :param columns: A dictionary with column names to include in the profile. Keys should be fully-qualified table - names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. - :param options: A dictionary with options for profiling each table. Keys should be fully-qualified table names - (e.g. `catalog.schema.table`) and values should be options for profiling. - :return: A dictionary mapping table names to tuples containing summary statistics and data quality profiles. + Args: + tables: An optional list of table names to include. + patterns: An optional list of table names or filesystem-style wildcards (e.g. 'schema.*') to include. + If None, all tables are included. By default, tables matching the pattern are included. + exclude_matched: Specifies whether to include tables matched by the pattern. If True, matched tables + are excluded. If False, matched tables are included. + columns: A dictionary with column names to include in the profile. Keys should be fully-qualified table + names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. + options: A dictionary with options for profiling each table. Keys should be fully-qualified table names + (e.g. `catalog.schema.table`) and values should be options for profiling. + + Returns: + A dictionary mapping table names to tuples containing summary statistics and data quality profiles. """ if not tables: if not patterns: @@ -150,10 +162,13 @@ def _get_tables(self, patterns: list[str] | None, exclude_matched: bool = False) """ Gets a list table names from Unity Catalog given a list of wildcard patterns. - :param patterns: A list of wildcard patterns to match against the table name. - :param exclude_matched: Specifies whether to include tables matched by the pattern. If True, matched tables - are excluded. If False, matched tables are included. - :return: A list of table names. + Args: + patterns: A list of wildcard patterns to match against the table name. + exclude_matched: Specifies whether to include tables matched by the pattern. If True, matched tables + are excluded. If False, matched tables are included. + + Returns: + A list of table names. """ tables = [] for catalog in self.ws.catalogs.list(): @@ -178,9 +193,12 @@ def _match_table_patterns(table: str, patterns: list[str]) -> bool: """ Checks if a table name matches any of the provided wildcard patterns. - :param table: The table name to check. - :param patterns: A list of wildcard patterns (e.g. 'catalog.schema.*') to match against the table name. - :return: True if the table name matches any of the patterns, False otherwise. + Args: + table: The table name to check. + patterns: A list of wildcard patterns (e.g. 'catalog.schema.*') to match against the table name. + + Returns: + True if the table name matches any of the patterns, False otherwise. """ return any(fnmatch(table, pattern) for pattern in patterns) @@ -194,13 +212,16 @@ def _profile_tables( """ Profiles a list of tables to generate summary statistics and data quality rules. - :param tables: A list of fully-qualified table names (`catalog.schema.table`) to be profiled - :param columns: A dictionary with column names to include in the profile. Keys should be fully-qualified table - names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. - :param options: A dictionary with options for profiling each table. Keys should be fully-qualified table names - (e.g. `catalog.schema.table`) and values should be options for profiling. - :param max_workers: An optional concurrency limit for profiling concurrently - :return: A dictionary mapping table names to tuples containing summary statistics and data quality profiles. + Args: + tables: A list of fully-qualified table names (`catalog.schema.table`) to be profiled + columns: A dictionary with column names to include in the profile. Keys should be fully-qualified table + names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. + options: A dictionary with options for profiling each table. Keys should be fully-qualified table names + (e.g. `catalog.schema.table`) and values should be options for profiling. + max_workers: An optional concurrency limit for profiling concurrently + + Returns: + A dictionary mapping table names to tuples containing summary statistics and data quality profiles. """ tables = tables or [] columns = columns or {} @@ -227,11 +248,14 @@ def _build_options_from_list(table: str, options: list[dict[str, Any]]) -> dict[ Builds the options dictionary from a list of matched options. Options with the highest similarity are prioritized in the result. - :param table: Table name - :param options: List of options dictionaries with the following fields: - * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options - * `options` - Dictionary of profiler options - :return: Dictionary of profiler options matching the provided table name + Args: + table: Table name + options: List of options dictionaries with the following fields: + * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options + * `options` - Dictionary of profiler options + + Returns: + Dictionary of profiler options matching the provided table name """ matched_options = DQProfiler._match_options_list(table, options) sorted_options = DQProfiler._sort_options_list(table, matched_options) @@ -246,11 +270,14 @@ def _match_options_list(table: str, options: list[dict[str, Any]]) -> list[dict[ """ Returns a subset of options whose 'table' pattern matches the provided table name. - :param table: Table name - :param options: List of options dictionaries with the following fields: - * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options - * `options` - Dictionary of profiler options - :return: List of options dictionaries matching the provided table name + Args: + table: Table name + options: List of options dictionaries with the following fields: + * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options + * `options` - Dictionary of profiler options + + Returns: + List of options dictionaries matching the provided table name """ return [opts for opts in options if fnmatch(table, opts.get("table", ""))] @@ -259,11 +286,14 @@ def _sort_options_list(table: str, options: list[dict[str, Any]]) -> list[dict[s """ Sorts the options list by sequence similarity with the provided table name. - :param table: Table name - :param options: List of options dictionaries with the following fields: - * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options - * `options` - Dictionary of profiler options - :return: List of options dictionaries sorted by similarity to the provided table name + Args: + table: Table name + options: List of options dictionaries with the following fields: + * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options + * `options` - Dictionary of profiler options + + Returns: + List of options dictionaries sorted by similarity to the provided table name """ return sorted( [opts | {"score": SequenceMatcher(None, table, opts.get("table", "")).quick_ratio()} for opts in options], @@ -315,13 +345,14 @@ def _calculate_metrics( """ Calculates various metrics for a given DataFrame column and updates the data quality rules. - :param df: The DataFrame containing the data. - :param dq_rules: A list to store the generated data quality rules. - :param field_name: The name of the column to calculate metrics for. - :param metrics: A dictionary to store the calculated metrics. - :param opts: A dictionary of options for metric calculation. - :param total_count: The total number of rows in the DataFrame. - :param typ: The data type of the column. + Args: + df: The DataFrame containing the data. + dq_rules: A list to store the generated data quality rules. + field_name: The name of the column to calculate metrics for. + metrics: A dictionary to store the calculated metrics. + opts: A dictionary of options for metric calculation. + total_count: The total number of rows in the DataFrame. + typ: The data type of the column. """ max_nulls = opts.get("max_null_ratio", 0) trim_strings = opts.get("trim_strings", True) @@ -376,8 +407,11 @@ def _get_df_summary_as_dict(self, df: DataFrame) -> dict[str, Any]: """ Generate summary for DataFrame and return it as a dictionary with column name as a key, and dict of metric/value. - :param df: The DataFrame to profile. - :return: A dictionary with metrics per column. + Args: + df: The DataFrame to profile. + + Returns: + A dictionary with metrics per column. """ sm_dict: dict[str, dict] = {} field_types = {f.name: f.dataType for f in df.schema.fields} @@ -391,10 +425,11 @@ def _process_row(self, row_dict: dict, metric: str, sm_dict: dict, field_types: """ Processes a row from the DataFrame summary and updates the summary dictionary. - :param row_dict: A dictionary representing a row from the DataFrame summary. - :param metric: The metric name (e.g., "mean", "stddev") for the current row. - :param sm_dict: The summary dictionary to update with the processed metrics. - :param field_types: A dictionary mapping column names to their data types. + Args: + row_dict: A dictionary representing a row from the DataFrame summary. + metric: The metric name (e.g., "mean", "stddev") for the current row. + sm_dict: The summary dictionary to update with the processed metrics. + field_types: A dictionary mapping column names to their data types. """ for metric_name, metric_value in row_dict.items(): if metric_name == "summary": @@ -407,11 +442,12 @@ def _process_metric(self, metric_name: str, metric_value: Any, metric: str, sm_d """ Processes a metric value and updates the summary dictionary with the casted value. - :param metric_name: The name of the metric (e.g., column name). - :param metric_value: The value of the metric to process. - :param metric: The type of metric (e.g., "stddev", "mean"). - :param sm_dict: The summary dictionary to update with the processed metric. - :param field_types: A dictionary mapping column names to their data types. + Args: + metric_name: The name of the metric (e.g., column name). + metric_value: The value of the metric to process. + metric: The type of metric (e.g., "stddev", "mean"). + sm_dict: The summary dictionary to update with the processed metric. + field_types: A dictionary mapping column names to their data types. """ typ = field_types[metric_name] if metric_value is not None: @@ -426,10 +462,13 @@ def _round_value(self, value: Any, direction: str, opts: dict[str, Any]) -> Any: """ Rounds a value based on the specified direction and options. - :param value: The value to round. - :param direction: The direction to round the value ("up" or "down"). - :param opts: A dictionary of options, including whether to round the value. - :return: The rounded value, or the original value if rounding is not enabled. + Args: + value: The value to round. + direction: The direction to round the value ("up" or "down"). + opts: A dictionary of options, including whether to round the value. + + Returns: + The rounded value, or the original value if rounding is not enabled. """ if not value or not opts.get("round", False): return value @@ -459,12 +498,15 @@ def _extract_min_max( """ Generates a data quality profile rule for column value ranges. - :param dst: A single-column DataFrame containing the data to analyze. - :param col_name: The name of the column to generate the rule for. - :param typ: The data type of the column. - :param metrics: A dictionary to store the calculated metrics. - :param opts: Optional dictionary of options for rule generation. - :return: A DQProfile object representing the min/max rule, or None if no rule is generated. + Args: + dst: A single-column DataFrame containing the data to analyze. + col_name: The name of the column to generate the rule for. + typ: The data type of the column. + metrics: A dictionary to store the calculated metrics. + opts: Optional dictionary of options for rule generation. + + Returns: + A DQProfile object representing the min/max rule, or None if no rule is generated. """ descr = None min_limit = None @@ -527,15 +569,18 @@ def _get_min_max( """ Calculates the minimum and maximum limits for a column based on the provided metrics and options. - :param col_name: The name of the column. - :param descr: The description of the min/max calculation. - :param max_limit: The maximum limit for the column. - :param metrics: A dictionary to store the calculated metrics. - :param min_limit: The minimum limit for the column. - :param mn_mx: A list containing the min, max, mean, and stddev values for the column. - :param opts: A dictionary of options for the min/max calculation. - :param typ: The data type of the column. - :return: A tuple containing the description, maximum limit, and minimum limit. + Args: + col_name: The name of the column. + descr: The description of the min/max calculation. + max_limit: The maximum limit for the column. + metrics: A dictionary to store the calculated metrics. + min_limit: The minimum limit for the column. + mn_mx: A list containing the min, max, mean, and stddev values for the column. + opts: A dictionary of options for the min/max calculation. + typ: The data type of the column. + + Returns: + A tuple containing the description, maximum limit, and minimum limit. """ if mn_mx and len(mn_mx) > 0: metrics["min"] = mn_mx[0][0] @@ -588,13 +633,16 @@ def _adjust_min_max_limits( """ Adjusts the minimum and maximum limits based on the data type of the column. - :param min_limit: The minimum limit to adjust. - :param max_limit: The maximum limit to adjust. - :param avg: The average value of the column. - :param typ: The PySpark data type of the column. - :param metrics: A dictionary containing the calculated metrics. - :param opts: A dictionary of options for min/max limit adjustment. - :return: A tuple containing the adjusted minimum and maximum limits. + Args: + min_limit: The minimum limit to adjust. + max_limit: The maximum limit to adjust. + avg: The average value of the column. + typ: The PySpark data type of the column. + metrics: A dictionary containing the calculated metrics. + opts: A dictionary of options for min/max limit adjustment. + + Returns: + A tuple containing the adjusted minimum and maximum limits. """ if isinstance(typ, T.IntegralType): min_limit = int(self._round_value(min_limit, "down", opts)) @@ -620,9 +668,12 @@ def _get_fields(col_name: str, schema: T.StructType) -> list[T.StructField]: """ Recursively extracts all fields from a nested StructType schema and prefixes them with the given column name. - :param col_name: The prefix to add to each field name. - :param schema: The StructType schema to extract fields from. - :return: A list of StructField objects with prefixed names. + Args: + col_name: The prefix to add to each field name. + schema: The StructType schema to extract fields from. + + Returns: + A list of StructField objects with prefixed names. """ fields = [] for f in schema.fields: @@ -638,9 +689,12 @@ def _do_cast(value: str | None, typ: T.DataType) -> Any | None: """ Casts a string value to a specified PySpark data type. - :param value: The string value to cast. Can be None. - :param typ: The PySpark data type to cast the value to. - :return: The casted value, or None if the input value is None. + Args: + value: The string value to cast. Can be None. + typ: The PySpark data type to cast the value to. + + Returns: + The casted value, or None if the input value is None. """ if not value: return None @@ -661,8 +715,11 @@ def _type_supports_distinct(typ: T.DataType) -> bool: """ Checks if the given PySpark data type supports distinct operations. - :param typ: The PySpark data type to check. - :return: True if the data type supports distinct operations, False otherwise. + Args: + typ: The PySpark data type to check. + + Returns: + True if the data type supports distinct operations, False otherwise. """ return typ == T.StringType() or typ == T.IntegerType() or typ == T.LongType() @@ -671,8 +728,11 @@ def _type_supports_min_max(typ: T.DataType) -> bool: """ Checks if the given PySpark data type supports min and max operations. - :param typ: The PySpark data type to check. - :return: True if the data type supports min and max operations, False otherwise. + Args: + typ: The PySpark data type to check. + + Returns: + True if the data type supports min and max operations, False otherwise. """ return isinstance(typ, T.NumericType) or typ == T.DateType() or typ == T.TimestampType() @@ -685,10 +745,15 @@ def _round_datetime(value: datetime.datetime, direction: str) -> datetime.dateti - "up" β†’ return the next midnight unless value is already midnight. - Raises ValueError for invalid direction. - :param value: The datetime value to round. - :param direction: The rounding direction ("up" or "down"). - :return: The rounded datetime value. - :raises ValueError: If direction is not 'up' or 'down'. + Args: + value: The datetime value to round. + direction: The rounding direction ("up" or "down"). + + Returns: + The rounded datetime value. + + Raises: + ValueError: If direction is not 'up' or 'down'. """ midnight = value.replace(hour=0, minute=0, second=0, microsecond=0) diff --git a/src/databricks/labs/dqx/profiler/runner.py b/src/databricks/labs/dqx/profiler/runner.py index 2554f1e9c..1228b352c 100644 --- a/src/databricks/labs/dqx/profiler/runner.py +++ b/src/databricks/labs/dqx/profiler/runner.py @@ -39,9 +39,12 @@ def run( """ Run the DQX profiler on the input data and return the generated checks and profile summary stats. - :param input_config: Input data configuration (e.g. table name or file location, read options). - :param profiler_config: Profiler configuration. - :return: A tuple containing the generated checks and profile summary statistics. + Args: + input_config: Input data configuration (e.g. table name or file location, read options). + profiler_config: Profiler configuration. + + Returns: + A tuple containing the generated checks and profile summary statistics. """ df = read_input_data(self.spark, input_config) summary_stats, profiles = self.profiler.profile( @@ -67,10 +70,11 @@ def save( """ Save the generated checks and profile summary statistics to the specified files. - :param checks: The generated checks. - :param summary_stats: The profile summary statistics. - :param checks_location: The file to save the checks to. - :param profile_summary_stats_file: The file to save the profile summary statistics to. + Args: + checks: The generated checks. + summary_stats: The profile summary statistics. + checks_location: The file to save the checks to. + profile_summary_stats_file: The file to save the profile summary statistics to. """ if not checks_location: raise ValueError("Check file not configured") diff --git a/src/databricks/labs/dqx/profiler/workflow.py b/src/databricks/labs/dqx/profiler/workflow.py index c1c0b683e..f11623c68 100644 --- a/src/databricks/labs/dqx/profiler/workflow.py +++ b/src/databricks/labs/dqx/profiler/workflow.py @@ -16,7 +16,8 @@ def profile(self, ctx: RuntimeContext): """ Profile the input data and save the generated checks and profile summary stats. - :param ctx: Runtime context. + Args: + ctx: Runtime context. """ run_config = ctx.run_config if not run_config.input_config: diff --git a/src/databricks/labs/dqx/rule.py b/src/databricks/labs/dqx/rule.py index 07ff57e31..299c26302 100644 --- a/src/databricks/labs/dqx/rule.py +++ b/src/databricks/labs/dqx/rule.py @@ -64,7 +64,9 @@ class SingleColumnMixin: def _get_column_as_string_expr(self, column: str | Column) -> Column: """Spark Column expression representing the column(s) as a string (not normalized). - :return: A Spark Column object representing the column(s) as a string (not normalized). + + Returns: + A Spark Column object representing the column(s) as a string (not normalized). """ return F.array(F.lit(get_column_name_or_alias(column))) @@ -86,7 +88,9 @@ class MultipleColumnsMixin: def _get_columns_as_string_expr(self, columns: list[str | Column]) -> Column: """Spark Column expression representing the column(s) as a string (not normalized). - :return: A Spark Column object representing the column(s) as a string (not normalized). + + Returns: + A Spark Column object representing the column(s) as a string (not normalized). """ return F.array(*[F.lit(get_column_name_or_alias(column)) for column in columns]) @@ -181,13 +185,16 @@ def get_check_condition(self) -> Column: """ Compute the check condition for the rule. - :return: The Spark Column representing the check condition. + Returns: + The Spark Column representing the check condition. """ @ft.cached_property def columns_as_string_expr(self) -> Column: """Spark Column expression representing the column(s) as a string (not normalized). - :return: A Spark Column object representing the column(s) as a string (not normalized). + + Returns: + A Spark Column object representing the column(s) as a string (not normalized). """ if self.column is not None: return self._get_column_as_string_expr(self.column) @@ -326,7 +333,8 @@ def get_check_condition(self) -> Column: """ Compute the check condition for this rule. - :return: The Spark Column representing the check condition. + Returns: + The Spark Column representing the check condition. """ check_condition = self.check # lazy evaluation of check function parameters return check_condition @@ -354,7 +362,8 @@ def get_check_condition(self) -> Column: """ Compute the check condition for this rule. - :return: The Spark Column representing the check condition. + Returns: + The Spark Column representing the check condition. """ check_condition, _ = self.check # lazy evaluation of check function parameters return check_condition @@ -395,7 +404,8 @@ class DQForEachColRule(DQRuleTypeMixin): def get_rules(self) -> list[DQRule]: """Build a list of rules for a set of columns. - :return: list of dq rules + Returns: + list of dq rules """ rules: list[DQRule] = [] for column in self.columns: diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index cd72817ae..1c1f5be6a 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -33,15 +33,20 @@ def get_column_name_or_alias( - Ensures the extracted expression is truncated to 255 characters. - Provides an optional normalization step for consistent naming. - :param column: Column, ConnectColumn or string representing a column. - :param normalize: If True, normalizes the column name (removes special characters, converts to lowercase). - :param allow_simple_expressions_only: If True, raises an error if the column expression is not a simple expression. - Complex PySpark expressions (e.g., conditionals, arithmetic, or nested transformations), cannot be fully - reconstructed correctly when converting to string (e.g. F.col("a") + F.lit(1)). - However, in certain situations this is acceptable, e.g. when using the output for reporting purposes. - :return: The extracted column alias or name. - :raises ValueError: If the column expression is invalid. - :raises TypeError: If the column type is unsupported. + Args: + column: Column, ConnectColumn or string representing a column. + normalize: If True, normalizes the column name (removes special characters, converts to lowercase). + allow_simple_expressions_only: If True, raises an error if the column expression is not a simple expression. + Complex PySpark expressions (e.g., conditionals, arithmetic, or nested transformations), cannot be fully + reconstructed correctly when converting to string (e.g. F.col("a") + F.lit(1)). + However, in certain situations this is acceptable, e.g. when using the output for reporting purposes. + + Returns: + The extracted column alias or name. + + Raises: + ValueError: If the column expression is invalid. + TypeError: If the column type is unsupported. """ if isinstance(column, str): col_str = column @@ -71,9 +76,12 @@ def get_columns_as_strings(columns: list[str | Column], allow_simple_expressions This function processes each column, ensuring that only valid column names are returned. - :param columns: List of columns, ConnectColumns or strings representing columns. - :param allow_simple_expressions_only: If True, raises an error if the column expression is not a simple expression. - :return: List of column names as strings. + Args: + columns: List of columns, ConnectColumns or strings representing columns. + allow_simple_expressions_only: If True, raises an error if the column expression is not a simple expression. + + Returns: + List of column names as strings. """ columns_as_strings = [] for col in columns: @@ -91,8 +99,11 @@ def is_simple_column_expression(col_name: str) -> bool: Returns True if the column name does not contain any disallowed characters: space, comma, semicolon, curly braces, parentheses, newline, tab, or equals sign. - :param col_name: Column name to validate. - :return: True if the column name is valid, False otherwise. + Args: + col_name: Column name to validate. + + Returns: + True if the column name is valid, False otherwise. """ return not bool(INVALID_COLUMN_NAME_PATTERN.search(col_name)) @@ -104,10 +115,15 @@ def normalize_bound_args(val: Any) -> Any: Handles primitives, dates, and column-like objects. Lists, tuples, and sets are recursively normalized with type preserved. - :param val: Value or collection of values to normalize. - :return: Normalized value or collection. - :raises ValueError: If a column resolves to an invalid name. - :raises TypeError: If a column type is unsupported. + Args: + val: Value or collection of values to normalize. + + Returns: + Normalized value or collection. + + Raises: + ValueError: If a column resolves to an invalid name. + TypeError: If a column type is unsupported. """ if isinstance(val, (list, tuple, set)): normalized = [normalize_bound_args(v) for v in val] @@ -132,8 +148,11 @@ def normalize_col_str(col_str: str) -> str: * convert to lowercase * limit the length to 255 characters to be compatible with metastore column names - :param col_str: Column or string representing a column. - :return: Normalized column name. + Args: + col_str: Column or string representing a column. + + Returns: + Normalized column name. """ max_chars = 255 return re.sub(COLUMN_NORMALIZE_EXPRESSION, "_", col_str[:max_chars].lower()).rstrip("_") @@ -146,9 +165,12 @@ def read_input_data( """ Reads input data from the specified location and format. - :param spark: SparkSession - :param input_config: InputConfig with source location/table name, format, and options - :return: DataFrame with values read from the input data + Args: + spark: SparkSession + input_config: InputConfig with source location/table name, format, and options + + Returns: + DataFrame with values read from the input data """ if not input_config.location: raise ValueError("Input location not configured") @@ -167,9 +189,12 @@ def read_input_data( def _read_file_data(spark: SparkSession, input_config: InputConfig) -> DataFrame: """ Reads input data from files (e.g. JSON). Streaming reads must use auto loader with a 'cloudFiles' format. - :param spark: SparkSession - :param input_config: InputConfig with source location, format, and options - :return: DataFrame with values read from the file data + Args: + spark: SparkSession + input_config: InputConfig with source location, format, and options + + Returns: + DataFrame with values read from the file data """ if not input_config.is_streaming: return spark.read.options(**input_config.options).load( @@ -187,9 +212,12 @@ def _read_file_data(spark: SparkSession, input_config: InputConfig) -> DataFrame def _read_table_data(spark: SparkSession, input_config: InputConfig) -> DataFrame: """ Reads input data from a table registered in Unity Catalog. - :param spark: SparkSession - :param input_config: InputConfig with source location, format, and options - :return: DataFrame with values read from the table data + Args: + spark: SparkSession + input_config: InputConfig with source location, format, and options + + Returns: + DataFrame with values read from the table data """ if not input_config.is_streaming: return spark.read.options(**input_config.options).table(input_config.location) @@ -199,8 +227,9 @@ def _read_table_data(spark: SparkSession, input_config: InputConfig) -> DataFram def save_dataframe_as_table(df: DataFrame, output_config: OutputConfig): """ Helper method to save a DataFrame to a Delta table. - :param df: The DataFrame to save - :param output_config: Output table name, write mode, and options + Args: + df: The DataFrame to save + output_config: Output table name, write mode, and options """ logger.info(f"Saving data to {output_config.location} table") @@ -262,7 +291,8 @@ def safe_json_load(value: str): Safely load a JSON string, returning the original value if it fails to parse. This allows to specify string value without a need to escape the quotes. - :param value: The value to parse as JSON. + Args: + value: The value to parse as JSON. """ try: return json.loads(value) # load as json if possible diff --git a/tests/e2e/test_pii_detection_checks.py b/tests/e2e/test_pii_detection_checks.py index bb02dce44..93a2f84a8 100644 --- a/tests/e2e/test_pii_detection_checks.py +++ b/tests/e2e/test_pii_detection_checks.py @@ -38,8 +38,10 @@ def test_run_pii_detection_notebook(make_notebook, make_job, library_ref): def validate_run_status(run: Run, client: WorkspaceClient) -> None: """ Validates that a job task run completed successfully. - :param run: `Run` object returned from a `WorkspaceClient.jobs.submit(...)` command - :param client: `WorkspaceClient` object for getting task output + + Args: + run: `Run` object returned from a `WorkspaceClient.jobs.submit(...)` command. + client: `WorkspaceClient` object for getting task output. """ task = run.tasks[0] termination_details = run.status.termination_details diff --git a/tests/e2e/test_run_demos.py b/tests/e2e/test_run_demos.py index cdac759cb..4bab8bb00 100644 --- a/tests/e2e/test_run_demos.py +++ b/tests/e2e/test_run_demos.py @@ -232,8 +232,10 @@ def test_run_dqx_streaming_demo_diy(make_notebook, make_job, tmp_path, library_r def validate_run_status(run: Run, client: WorkspaceClient) -> None: """ Validates that a job task run completed successfully. - :param run: `Run` object returned from a `WorkspaceClient.jobs.submit(...)` command - :param client: `WorkspaceClient` object for getting task output + + Args: + run: `Run` object returned from a `WorkspaceClient.jobs.submit(...)` command + client: `WorkspaceClient` object for getting task output """ task = run.tasks[0] termination_details = run.status.termination_details diff --git a/tests/unit/test_build_rules.py b/tests/unit/test_build_rules.py index 40ea187a6..e5ba93552 100644 --- a/tests/unit/test_build_rules.py +++ b/tests/unit/test_build_rules.py @@ -467,8 +467,11 @@ def _build_rules_foreach_col(*rules_col_set: DQForEachColRule) -> list[DQRule]: """ Build rules for each column from DQForEachColRule sets. - :param rules_col_set: list of dq rules which define multiple columns for the same check function - :return: list of dq rules + Args: + rules_col_set: list of dq rules which define multiple columns for the same check function + + Returns: + list of dq rules """ rules_nested = [rule_set.get_rules() for rule_set in rules_col_set] flat_rules = list(itertools.chain(*rules_nested)) From 70d1f29846aea3f6674c1bad64c1e271673ecf0f Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sat, 16 Aug 2025 11:10:58 +0200 Subject: [PATCH 29/37] updated docs --- docs/dqx/docs/dev/contributing.mdx | 7 +++++-- pyproject.toml | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 649d6badf..f4a18108a 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -319,7 +319,7 @@ def method(self, arg1: str, arg2: int) -> str: """ Short method description in Markdown format. - Very long method description in Markdown format. + Very long method description in Markdown format. Referring to *arg1* and *arg2* in the narrative text. Args: arg1 (str): Argument 1 description. @@ -331,8 +331,11 @@ def method(self, arg1: str, arg2: int) -> str: return "Hello, world!" ``` -Please note that double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. + +* Avoid using backticks around argument names in the narrative text of a docstring (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize argument names in descriptions. +* Double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. If you need a code example, use triple backticks, e.g.: ```python print("Hello, world!") ``` + diff --git a/pyproject.toml b/pyproject.toml index 7d89e2eea..b5f2ef8bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,7 +128,6 @@ hooks = { post-render = [ ] } [tool.pydoc-markdown.renderer] - type = "docusaurus" docs_base_path = "docs/dqx/docs" relative_output_path = "reference/api" From baf5dca00923792c8361c66f1a4ced1e6404bbd5 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 11:00:55 +0200 Subject: [PATCH 30/37] updated docs to replace backtics with * --- docs/dqx/docs/dev/contributing.mdx | 2 +- src/databricks/labs/dqx/base.py | 32 ++--- src/databricks/labs/dqx/check_funcs.py | 46 +++--- src/databricks/labs/dqx/checks_resolver.py | 2 +- src/databricks/labs/dqx/checks_serializer.py | 44 +++--- src/databricks/labs/dqx/checks_storage.py | 4 +- src/databricks/labs/dqx/config.py | 2 +- src/databricks/labs/dqx/engine.py | 136 +++++++++--------- src/databricks/labs/dqx/executor.py | 2 +- .../labs/dqx/pii/pii_detection_funcs.py | 12 +- src/databricks/labs/dqx/profiler/profiler.py | 24 ++-- src/databricks/labs/dqx/rule.py | 42 +++--- 12 files changed, 174 insertions(+), 174 deletions(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index f4a18108a..af159af00 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -332,7 +332,7 @@ def method(self, arg1: str, arg2: int) -> str: ``` -* Avoid using backticks around argument names in the narrative text of a docstring (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize argument names in descriptions. +* Avoid using backticks around object names in the docstring (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize object names. * Double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. If you need a code example, use triple backticks, e.g.: ```python diff --git a/src/databricks/labs/dqx/base.py b/src/databricks/labs/dqx/base.py index ceba036e3..aacce3d46 100644 --- a/src/databricks/labs/dqx/base.py +++ b/src/databricks/labs/dqx/base.py @@ -16,19 +16,19 @@ def __init__(self, workspace_client: WorkspaceClient): @cached_property def ws(self) -> WorkspaceClient: - """Return a verified `WorkspaceClient` configured for DQX. + """Return a verified *WorkspaceClient* configured for DQX. Ensures workspace connectivity and sets the product info used for - telemetry so that requests are attributed to `dqx`. + telemetry so that requests are attributed to *dqx*. """ return self._verify_workspace_client(self._workspace_client) @cached_property def spark(self) -> SparkSession: - """Return the `SparkSession` associated with this engine. + """Return the *SparkSession* associated with this engine. The session is created during initialization using - `SparkSession.builder.getOrCreate()`. + *SparkSession.builder.getOrCreate()*. """ return self._spark @@ -57,7 +57,7 @@ def apply_checks( Args: df: Input DataFrame to check. - checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. + checks: List of checks to apply to the DataFrame. Each check must be a *DQRule* instance. ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -73,7 +73,7 @@ def apply_checks_and_split( Args: df: Input DataFrame to check. - checks: List of checks to apply to the DataFrame. Each check must be a `DQRule` instance. + checks: List of checks to apply to the DataFrame. Each check must be a *DQRule* instance. ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -95,11 +95,11 @@ def apply_checks_by_metadata( Args: df: Input DataFrame to check. checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* (rows appear in both DataFrames). - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -120,11 +120,11 @@ def apply_checks_by_metadata_and_split( Args: df: Input DataFrame to check. checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* (rows appear in both DataFrames). - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -147,7 +147,7 @@ def validate_checks( Args: checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). validate_custom_check_functions: If True, validate custom check functions. Returns: @@ -184,7 +184,7 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: """ Load DQ rules (checks) from a local JSON or YAML file. - The returned checks can be used as input to `apply_checks_by_metadata`. + The returned checks can be used as input to *apply_checks_by_metadata*. Args: filepath: Path to a file containing checks definitions. diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 03ac0fc33..294d6b901 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -35,12 +35,12 @@ def make_condition(condition: Column, message: Column | str, alias: str) -> Colu condition: condition expression. - Pass the check if the condition evaluates to False - Fail the check if condition evaluates to True - message: message to output - it could be either `Column` object, or string constant + message: message to output - it could be either *Column* object, or string constant alias: name for the resulting column Returns: - an instance of `Column` type, that either returns string if condition is evaluated to `true`, - or `null` if condition is evaluated to `false` + an instance of *Column* type, that either returns string if condition is evaluated to *true*, + or *null* if condition is evaluated to *false* """ if isinstance(message, str): msg_col = F.lit(message) @@ -197,7 +197,7 @@ def sql_expression( Args: expression: SQL expression. Fail if expression evaluates to True, pass if it evaluates to False. - msg: optional message of the `Column` type, automatically generated if None + msg: optional message of the *Column* type, automatically generated if None name: optional name of the resulting column, automatically generated if None negate: if the condition should be negated (true) or not. For example, "col is not null" will mark null values as "bad". Although sometimes it's easier to specify it other way around "col is null" + negate set to False @@ -735,7 +735,7 @@ def is_unique( Build a uniqueness check condition and closure for dataset-level validation. This function checks whether the specified columns contain unique values within the dataset - and reports rows with duplicate combinations. When `nulls_distinct` + and reports rows with duplicate combinations. When *nulls_distinct* is True (default), rows with NULLs are treated as distinct (SQL ANSI behavior); otherwise, NULLs are treated as equal when checking for duplicates. @@ -835,7 +835,7 @@ def foreign_key( """ Build a foreign key check condition and closure for dataset-level validation. - This function verifies that values in the specified foreign key columns exist (or don't exist, if `negate=True`) in + This function verifies that values in the specified foreign key columns exist (or don't exist, if *negate=True*) in the corresponding reference columns of another DataFrame or table. Rows where foreign key values do not match the reference are reported as violations. @@ -951,7 +951,7 @@ def sql_query( msg: Optional custom message or Column expression. name: Optional name for the result. negate: If True, the condition is negated (i.e., the check fails when the condition is False). - input_placeholder: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the + input_placeholder: Name to be used in the sql query as *{{ input_placeholder }}* to refer to the input DataFrame on which the checks are applied. row_filter: Optional SQL expression for filtering rows before checking the foreign key. Auto-injected from the check filter. @@ -1235,7 +1235,7 @@ def compare_datasets( expressions are supported, e.g. F.col("col_name"). ref_columns: List of columns in the reference DataFrame or Table to row match against the source DataFrame (can be a list of string column names or column expressions). - The `columns` parameter is matched with `ref_columns` by position, so the order of + The *columns* parameter is matched with *ref_columns* by position, so the order of the provided columns in both lists must be exactly aligned. Only simple column expressions are supported, e.g. F.col("col_name"). ref_df_name: Name of the reference DataFrame (used when passing DataFrames directly). @@ -1349,15 +1349,15 @@ def is_data_fresh_per_time_window( Build a completeness freshness check that validates records arrive at least every X minutes with a threshold for the expected number of rows per time window. - If `lookback_windows` is provided, only data within that lookback period will be validated. + If *lookback_windows* is provided, only data within that lookback period will be validated. If omitted, the entire dataset will be checked. Args: column: Column name (str) or Column expression containing timestamps to check. window_minutes: Time window in minutes to check for data arrival. min_records_per_window: Minimum number of records expected per time window. - lookback_windows: Optional number of time windows to look back from `curr_timestamp`. - This filters records to include only those within the specified number of time windows from `curr_timestamp`. + lookback_windows: Optional number of time windows to look back from *curr_timestamp*. + This filters records to include only those within the specified number of time windows from *curr_timestamp*. If no lookback is provided, the check is applied to the entire dataset. row_filter: Optional SQL expression to filter rows before checking. curr_timestamp: Optional current timestamp column. If not provided, current_timestamp() function is used. @@ -1523,20 +1523,20 @@ def _add_column_diffs( """ Adds a column to the DataFrame that contains a map of changed columns and their differences. - This function compares specified columns between two datasets (`df` and `ref_df`) and identifies differences. - For each column in `compare_columns`, it checks if the values in `df` and `ref_df` are equal. - If a difference is found, it adds the column name and the differing values to a map stored in `columns_changed_col`. + This function compares specified columns between two datasets (*df* and *ref_df*) and identifies differences. + For each column in *compare_columns*, it checks if the values in *df* and *ref_df* are equal. + If a difference is found, it adds the column name and the differing values to a map stored in *columns_changed_col*. Args: df: The input DataFrame containing columns to compare. - compare_columns: List of column names to compare between `df` and `ref_df`. + compare_columns: List of column names to compare between *df* and *ref_df*. columns_changed_col: Name of the column to store the map of changed columns and their differences. null_safe_column_value_matching: If True, treats nulls as equal when matching column values. If enabled (NULL, NULL) column values are equal and matching. If False, uses a standard inequality comparison (`!=`), where (NULL, NULL) values are not considered equal. Returns: - A DataFrame with the added `columns_changed_col` containing the map of changed columns and differences. + A DataFrame with the added *columns_changed_col* containing the map of changed columns and differences. """ if compare_columns: columns_changed = [ @@ -1584,7 +1584,7 @@ def _add_compare_condition( ) -> DataFrame: """ Add the condition column only for mismatched records based on filter and differences. - This function adds a new column (`condition_col`) to the DataFrame, which contains structured information + This function adds a new column (*condition_col*) to the DataFrame, which contains structured information about mismatched records. The mismatches are determined based on the presence of missing rows, extra rows, and differences in specified columns. @@ -1597,7 +1597,7 @@ def _add_compare_condition( filter_col: The name of the column used to filter records for comparison. Returns: - The input DataFrame with the added `condition_col` containing mismatch information. + The input DataFrame with the added *condition_col* containing mismatch information. """ all_is_ok = ~F.col(row_missing_col) & ~F.col(row_extra_col) & (F.size(F.col(columns_changed_col)) == 0) return df.withColumn( @@ -1733,7 +1733,7 @@ def _get_ref_df( Retrieve the reference DataFrame based on the provided parameters. This helper fetches the reference DataFrame either from the supplied dictionary of DataFrames - (using `ref_df_name` as the key) or by reading a table from the Unity Catalog (using `ref_table`). + (using *ref_df_name* as the key) or by reading a table from the Unity Catalog (using *ref_table*). It raises an error if the necessary reference source is not properly specified or cannot be found. Args: @@ -1746,7 +1746,7 @@ def _get_ref_df( A Spark DataFrame representing the reference dataset. Raises: - ValueError: If neither or both of `ref_df_name` and `ref_table` are provided, + ValueError: If neither or both of *ref_df_name* and *ref_table* are provided, or if the specified reference DataFrame is not found. """ if ref_df_name: @@ -1904,7 +1904,7 @@ def _validate_ref_params( Validate reference parameters to ensure correctness and prevent ambiguity. This helper verifies that: - - Exactly one of `ref_df_name` or `ref_table` is provided (not both, not neither). + - Exactly one of *ref_df_name* or *ref_table* is provided (not both, not neither). - The number of columns in the input DataFrame matches the number of reference columns. Args: @@ -1914,8 +1914,8 @@ def _validate_ref_params( ref_table: Optional name of the reference table. Raises: - ValueError: If both or neither of `ref_df_name` and `ref_table` are provided, - or if the lengths of `columns` and `ref_columns` do not match. + ValueError: If both or neither of *ref_df_name* and *ref_table* are provided, + or if the lengths of *columns* and *ref_columns* do not match. """ if ref_df_name and ref_table: raise ValueError( diff --git a/src/databricks/labs/dqx/checks_resolver.py b/src/databricks/labs/dqx/checks_resolver.py index fc96b3780..4f13f1ae1 100644 --- a/src/databricks/labs/dqx/checks_resolver.py +++ b/src/databricks/labs/dqx/checks_resolver.py @@ -16,7 +16,7 @@ def resolve_check_function( Args: function_name: name of the function to resolve. - custom_check_functions: dictionary with custom check functions (eg. ``globals()`` of the calling module). + custom_check_functions: dictionary with custom check functions (eg. *globals()* of the calling module). fail_on_missing: if True, raise an AttributeError if the function is not found. Returns: diff --git a/src/databricks/labs/dqx/checks_serializer.py b/src/databricks/labs/dqx/checks_serializer.py index 292d3cc7f..4f2fc117a 100644 --- a/src/databricks/labs/dqx/checks_serializer.py +++ b/src/databricks/labs/dqx/checks_serializer.py @@ -49,12 +49,12 @@ def serialize_checks_from_dataframe(df: DataFrame, run_config_name: str = "defau Args: df: DataFrame with data quality check rules. Each row should define a check. Rows should have the following columns: - * `name` - Name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` (data is going into both dataframes) - * `check` - DQX check function used in the check; A `StructType` column defining the data quality check - * `filter` - Expression for filtering data quality checks - * `run_config_name` (optional) - Run configuration name for storing checks across runs - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + - *name* - Name that will be given to a resulting column. Autogenerated if not provided + - *criticality* (optional) - Possible values are *error* (data going only into "bad" dataframe) and *warn* (data is going into both dataframes) + - *check* - DQX check function used in the check; A *StructType* column defining the data quality check + - *filter* - Expression for filtering data quality checks + - *run_config_name* (optional) - Run configuration name for storing checks across runs + - *user_metadata* (optional) - User-defined key-value pairs added to metadata generated by the check. run_config_name: Run configuration name for filtering quality rules Returns: @@ -104,13 +104,13 @@ def deserialize_checks_to_dataframe( Args: spark: Spark session. checks: list of check specifications as Python dictionaries. Each check consists of the following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to - true (it will be used as an error/warning message) or `null` if it's evaluated to `false` - * `name` - Name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - Possible values are `error` (data going only into "bad" dataframe) and `warn` - (data is going into both dataframes) - * `filter` (optional) - Expression for filtering data quality checks - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + - *check* - Column expression to evaluate. This expression should return string value if it's evaluated to + true (it will be used as an error/warning message) or *null* if it's evaluated to *false* + - *name* - Name that will be given to a resulting column. Autogenerated if not provided + - *criticality* (optional) - Possible values are *error* (data going only into "bad" dataframe) and *warn* + (data is going into both dataframes) + - *filter* (optional) - Expression for filtering data quality checks + - *user_metadata* (optional) - User-defined key-value pairs added to metadata generated by the check. run_config_name: Run configuration name for storing quality checks across runs Returns: @@ -151,14 +151,14 @@ def deserialize_checks(checks: list[dict], custom_checks: dict[str, Any] | None Args: checks: list of dictionaries describing checks. Each check is a dictionary consisting of following fields: - * `check` - Column expression to evaluate. This expression should return string value if it's evaluated to true - - it will be used as an error/warning message, or `null` if it's evaluated to `false` - * `name` - name that will be given to a resulting column. Autogenerated if not provided - * `criticality` (optional) - possible values are `error` (data going only into "bad" dataframe), - and `warn` (data is going into both dataframes) - * `filter` (optional) - Expression for filtering data quality checks - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. - custom_checks: dictionary with custom check functions (eg. ``globals()`` of the calling module). + - *check* - Column expression to evaluate. This expression should return string value if it's evaluated to true + or *null* if it's evaluated to *false* + - *name* - name that will be given to a resulting column. Autogenerated if not provided + - *criticality* (optional) - possible values are *error* (data going only into "bad" dataframe), + and *warn* (data is going into both dataframes) + - *filter* (optional) - Expression for filtering data quality checks + - *user_metadata* (optional) - User-defined key-value pairs added to metadata generated by the check. + custom_checks: dictionary with custom check functions (eg. *globals()* of the calling module). If not specified, then only built-in functions are used for the checks. Returns: @@ -236,7 +236,7 @@ def deserialize_checks(checks: list[dict], custom_checks: dict[str, Any] | None def serialize_checks(checks: list[DQRule]) -> list[dict]: """ - Converts a list of quality checks defined as `DQRule` objects to a list of quality checks + Converts a list of quality checks defined as *DQRule* objects to a list of quality checks defined as Python dictionaries. Args: diff --git a/src/databricks/labs/dqx/checks_storage.py b/src/databricks/labs/dqx/checks_storage.py index c761ccaa2..03f523be7 100644 --- a/src/databricks/labs/dqx/checks_storage.py +++ b/src/databricks/labs/dqx/checks_storage.py @@ -45,8 +45,8 @@ class ChecksStorageHandler(ABC, Generic[T]): def load(self, config: T) -> list[dict]: """ Load quality rules from the source. - The returned checks can be used as input for `apply_checks_by_metadata` or - `apply_checks_by_metadata_and_split` functions. + The returned checks can be used as input for *apply_checks_by_metadata* or + *apply_checks_by_metadata_and_split* functions. Args: config: configuration for loading checks, including the table location and run configuration name. diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 9ac4fe05b..9001fbf64 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -150,7 +150,7 @@ class TableChecksStorageConfig(BaseChecksStorageConfig): location: The table name where the checks are stored. run_config_name: The name of the run configuration to use for checks (default is 'default'). mode: The mode for writing checks to a table (e.g., 'append' or 'overwrite'). - The `overwrite` mode will only replace checks for the specific run config and not all checks in the table. + The *overwrite* mode will only replace checks for the specific run config and not all checks in the table. """ location: str diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index af8b80866..3075c4419 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -78,7 +78,7 @@ def apply_checks( Args: df: Input DataFrame to check. - checks: List of checks to apply. Each check must be a `DQRule` instance. + checks: List of checks to apply. Each check must be a *DQRule* instance. ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -113,7 +113,7 @@ def apply_checks_and_split( Args: df: Input DataFrame to check. - checks: List of checks to apply. Each check must be a `DQRule` instance. + checks: List of checks to apply. Each check must be a *DQRule* instance. ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -148,11 +148,11 @@ def apply_checks_by_metadata( Args: df: Input DataFrame to check. checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* (rows appear in both DataFrames). - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -175,11 +175,11 @@ def apply_checks_by_metadata_and_split( Args: df: Input DataFrame to check. checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use. - - `name` - Optional name for the resulting column. Auto-generated if not provided. - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* (rows appear in both DataFrames). - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -204,7 +204,7 @@ def validate_checks( Args: checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). validate_custom_check_functions: If True, validate custom check functions. Returns: @@ -246,7 +246,7 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: """ Load DQ rules (checks) from a local JSON or YAML file. - The returned checks can be used as input to `apply_checks_by_metadata`. + The returned checks can be used as input to *apply_checks_by_metadata*. Args: filepath: Path to a file containing checks definitions. @@ -318,7 +318,7 @@ def _create_results_array( ref_dfs: Optional dictionary of reference DataFrames, keyed by name, for use by dataset-level checks. Returns: - DataFrame with an added array column (`dest_col`) containing the results of the applied checks. + DataFrame with an added array column (*dest_col*) containing the results of the applied checks. """ if not checks: # No checks then just append a null array result @@ -361,7 +361,7 @@ def _create_results_array( class DQEngine(DQEngineBase): """High-level engine to apply data quality checks and manage IO. - This class delegates core checking logic to `DQEngineCore` while providing helpers to + This class delegates core checking logic to *DQEngineCore* while providing helpers to read inputs, persist results, and work with different storage backends for checks. """ @@ -390,7 +390,7 @@ def apply_checks( Args: df: Input DataFrame to check. - checks: List of checks to apply. Each check must be a `DQRule` instance. + checks: List of checks to apply. Each check must be a *DQRule* instance. ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -406,7 +406,7 @@ def apply_checks_and_split( Args: df: Input DataFrame to check. - checks: List of checks to apply. Each check must be a `DQRule` instance. + checks: List of checks to apply. Each check must be a *DQRule* instance. ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -427,11 +427,11 @@ def apply_checks_by_metadata( Args: df: Input DataFrame to check. checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* + (rows appear in both DataFrames). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -452,11 +452,11 @@ def apply_checks_by_metadata_and_split( Args: df: Input DataFrame to check. checks: List of dictionaries describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* + (rows appear in both DataFrames). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). ref_dfs: Optional reference DataFrames to use in the checks. Returns: @@ -475,14 +475,14 @@ def apply_checks_and_save_in_table( """ Apply data quality checks to input data and save results. - If `quarantine_config` is provided, split the data into valid and invalid records: - - valid records are written using `output_config`; - - invalid records are written using `quarantine_config`. + If *quarantine_config* is provided, split the data into valid and invalid records: + - valid records are written using *output_config*. + - invalid records are written using *quarantine_config*. - If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. + If *quarantine_config* is not provided, write all rows (including result columns) using *output_config*. Args: - checks: List of `DQRule` checks to apply. + checks: List of *DQRule* checks to apply. input_config: Input configuration (e.g., table/view or file location and read options). output_config: Output configuration (e.g., table name, mode, and write options). quarantine_config: Optional configuration for writing invalid records. @@ -513,18 +513,18 @@ def apply_checks_by_metadata_and_save_in_table( """ Apply metadata-defined data quality checks to input data and save results. - If `quarantine_config` is provided, split the data into valid and invalid records: - - valid records are written using `output_config`; - - invalid records are written using `quarantine_config`. + If *quarantine_config* is provided, split the data into valid and invalid records: + - valid records are written using *output_config*; + - invalid records are written using *quarantine_config*. - If `quarantine_config` is not provided, write all rows (including result columns) using `output_config`. + If *quarantine_config* is not provided, write all rows (including result columns) using *output_config*. Args: checks: List of dicts describing checks. Each check dictionary must contain the following: - - `check` - A check definition including check function and arguments to use; - - `name` - Optional name for the resulting column. Auto-generated if not provided; - - `criticality` - Optional; either `error` (rows go only to the "bad" DataFrame) or `warn` - (rows appear in both DataFrames); + - *check* - A check definition including check function and arguments to use. + - *name* - Optional name for the resulting column. Auto-generated if not provided. + - *criticality* - Optional; either *error* (rows go only to the "bad" DataFrame) or *warn* + (rows appear in both DataFrames). input_config: Input configuration (e.g., table/view or file location and read options). output_config: Output configuration (e.g., table name, mode, and write options). quarantine_config: Optional configuration for writing invalid records. @@ -559,7 +559,7 @@ def validate_checks( Args: checks: List of checks to apply to the DataFrame. Each check should be a dictionary. - custom_check_functions: Optional dictionary with custom check functions (e.g., ``globals()`` of the calling module). + custom_check_functions: Optional dictionary with custom check functions (e.g., *globals()* of the calling module). validate_custom_check_functions: If True, validate custom check functions. Returns: @@ -604,8 +604,8 @@ def save_results_in_table( """Persist result DataFrames using explicit configs or the named run configuration. Behavior: - - If `output_df` is provided and `output_config` is None, load the run config and use its `output_config`. - - If `quarantine_df` is provided and `quarantine_config` is None, load the run config and use its `quarantine_config`. + - If *output_df* is provided and *output_config* is None, load the run config and use its *output_config*. + - If *quarantine_df* is provided and *quarantine_config* is None, load the run config and use its *quarantine_config*. - A write occurs only when both a DataFrame and its corresponding config are available. Args: @@ -635,18 +635,18 @@ def save_results_in_table( save_dataframe_as_table(quarantine_df, quarantine_config) def load_checks(self, config: BaseChecksStorageConfig) -> list[dict]: - """Load DQ rules (checks) from the storage backend described by `config`. + """Load DQ rules (checks) from the storage backend described by *config*. This method delegates to a storage handler selected by the factory - based on the concrete type of `config` and returns the parsed list - of checks (as dictionaries) ready for `apply_checks_by_metadata`. + based on the concrete type of *config* and returns the parsed list + of checks (as dictionaries) ready for *apply_checks_by_metadata*. Supported storage configurations include, for example: - - `FileChecksStorageConfig` (local file); - - `WorkspaceFileChecksStorageConfig` (Databricks workspace file); - - `TableChecksStorageConfig` (table-backed storage); - - `InstallationChecksStorageConfig` (installation directory); - - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); + - *FileChecksStorageConfig* (local file); + - *WorkspaceFileChecksStorageConfig* (Databricks workspace file); + - *TableChecksStorageConfig* (table-backed storage); + - *InstallationChecksStorageConfig* (installation directory); + - *VolumeFileChecksStorageConfig* (Unity Catalog volume file); Args: config: Configuration object describing the storage backend. @@ -661,19 +661,19 @@ def load_checks(self, config: BaseChecksStorageConfig) -> list[dict]: return handler.load(config) def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> None: - """Persist DQ rules (checks) to the storage backend described by `config`. + """Persist DQ rules (checks) to the storage backend described by *config*. The appropriate storage handler is resolved from the configuration type and used to write the provided checks. Any write semantics - (e.g., append/overwrite) are controlled by fields on `config` - such as `mode` where applicable. + (e.g., append/overwrite) are controlled by fields on *config* + such as *mode* where applicable. Supported storage configurations include, for example: - - `FileChecksStorageConfig` (local file); - - `WorkspaceFileChecksStorageConfig` (Databricks workspace file); - - `TableChecksStorageConfig` (table-backed storage); - - `InstallationChecksStorageConfig` (installation directory); - - `VolumeFileChecksStorageConfig` (Unity Catalog volume file); + - *FileChecksStorageConfig* (local file); + - *WorkspaceFileChecksStorageConfig* (Databricks workspace file); + - *TableChecksStorageConfig* (table-backed storage); + - *InstallationChecksStorageConfig* (installation directory); + - *VolumeFileChecksStorageConfig* (Unity Catalog volume file); Args: checks: List of DQ rules (checks) to save (as dictionaries). @@ -693,7 +693,7 @@ def save_checks(self, checks: list[dict], config: BaseChecksStorageConfig) -> No # @staticmethod def load_checks_from_local_file(filepath: str) -> list[dict]: - """Deprecated: Use `load_checks` with `FileChecksStorageConfig` instead. + """Deprecated: Use *load_checks* with *FileChecksStorageConfig* instead. Load DQ rules (checks) from a local JSON or YAML file. @@ -713,7 +713,7 @@ def load_checks_from_local_file(filepath: str) -> list[dict]: @staticmethod def save_checks_in_local_file(checks: list[dict], path: str): - """Deprecated: Use `save_checks` with `FileChecksStorageConfig` instead. + """Deprecated: Use *save_checks* with *FileChecksStorageConfig* instead. Save DQ rules (checks) to a local YAML or JSON file. @@ -733,7 +733,7 @@ def save_checks_in_local_file(checks: list[dict], path: str): return DQEngineCore.save_checks_in_local_file(checks, path) def load_checks_from_workspace_file(self, workspace_path: str) -> list[dict]: - """Deprecated: Use `load_checks` with `WorkspaceFileChecksStorageConfig` instead. + """Deprecated: Use *load_checks* with *WorkspaceFileChecksStorageConfig* instead. Load checks stored in a Databricks workspace file. @@ -752,7 +752,7 @@ def load_checks_from_workspace_file(self, workspace_path: str) -> list[dict]: return self.load_checks(WorkspaceFileChecksStorageConfig(location=workspace_path)) def save_checks_in_workspace_file(self, checks: list[dict], workspace_path: str): - """Deprecated: Use `save_checks` with `WorkspaceFileChecksStorageConfig` instead. + """Deprecated: Use *save_checks* with *WorkspaceFileChecksStorageConfig* instead. Save checks to a Databricks workspace file. @@ -772,7 +772,7 @@ def save_checks_in_workspace_file(self, checks: list[dict], workspace_path: str) return self.save_checks(checks, WorkspaceFileChecksStorageConfig(location=workspace_path)) def load_checks_from_table(self, table_name: str, run_config_name: str = "default") -> list[dict]: - """Deprecated: Use `load_checks` with `TableChecksStorageConfig` instead. + """Deprecated: Use *load_checks* with *TableChecksStorageConfig* instead. Load checks from a table. @@ -794,7 +794,7 @@ def load_checks_from_table(self, table_name: str, run_config_name: str = "defaul def save_checks_in_table( self, checks: list[dict], table_name: str, run_config_name: str = "default", mode: str = "append" ): - """Deprecated: Use `save_checks` with `TableChecksStorageConfig` instead. + """Deprecated: Use *save_checks* with *TableChecksStorageConfig* instead. Save checks to a table. @@ -824,7 +824,7 @@ def load_checks_from_installation( product_name: str = "dqx", assume_user: bool = True, ) -> list[dict]: - """Deprecated: Use `load_checks` with `InstallationChecksStorageConfig` instead. + """Deprecated: Use *load_checks* with *InstallationChecksStorageConfig* instead. Load checks from the installation directory. @@ -858,7 +858,7 @@ def save_checks_in_installation( product_name: str = "dqx", assume_user: bool = True, ): - """Deprecated: Use `save_checks` with `InstallationChecksStorageConfig` instead. + """Deprecated: Use *save_checks* with *InstallationChecksStorageConfig* instead. Save checks to the installation directory. @@ -889,7 +889,7 @@ def save_checks_in_installation( def load_run_config( self, run_config_name: str = "default", assume_user: bool = True, product_name: str = "dqx" ) -> RunConfig: - """Deprecated: Use `RunConfigLoader.load_run_config` directly. + """Deprecated: Use *RunConfigLoader.load_run_config* directly. Load a run configuration by name. This wrapper will be removed in a future version. @@ -899,7 +899,7 @@ def load_run_config( product_name: Product/installation identifier (e.g., "dqx"). Returns: - Loaded `RunConfig` instance. + Loaded *RunConfig* instance. """ warnings.warn( "Use `RunConfigLoader.load_run_config` method instead. This method will be removed in future versions.", diff --git a/src/databricks/labs/dqx/executor.py b/src/databricks/labs/dqx/executor.py index 1bb5a8c5d..059e7ba33 100644 --- a/src/databricks/labs/dqx/executor.py +++ b/src/databricks/labs/dqx/executor.py @@ -32,7 +32,7 @@ class DQRuleExecutor(abc.ABC): - DQRowRuleExecutor: Handles row-level rules (rules that produce a condition per row). - DQDatasetRuleExecutor: Handles dataset-level rules (rules that may aggregate or join DataFrames). - Subclasses must implement the `apply` method, which performs the rule check and returns + Subclasses must implement the *apply* method, which performs the rule check and returns a DQCheckResult containing: - The raw condition produced by the rule. - The DataFrame used for reporting (original or transformed). diff --git a/src/databricks/labs/dqx/pii/pii_detection_funcs.py b/src/databricks/labs/dqx/pii/pii_detection_funcs.py index e3fd7dccc..db53575f5 100644 --- a/src/databricks/labs/dqx/pii/pii_detection_funcs.py +++ b/src/databricks/labs/dqx/pii/pii_detection_funcs.py @@ -42,7 +42,7 @@ def does_not_contain_pii( Higher values = less sensitive, fewer false positives Lower values = more sensitive, more potential false positives entities: Optional list of entities to detect - nlp_engine_config: Optional NLP engine configuration used for PII detection; Can be `dict` or `NLPEngineConfiguration` + nlp_engine_config: Optional NLP engine configuration used for PII detection; Can be *dict* or *NLPEngineConfiguration* Returns: Column object for condition that fails when PII is detected @@ -80,9 +80,9 @@ def _validate_environment() -> None: user-defined functions. UDFs will fail with out-of-memory errors if these limits are exceeded. Because of this limitation, we limit use of PII detection checks to local Spark or a Databricks - workspace. Databricks Connect uses a `pyspark.sql.connect.session.SparkSession` with an external + workspace. Databricks Connect uses a *pyspark.sql.connect.session.SparkSession* with an external host (e.g. 'https://hostname.cloud.databricks.com'). To raise a clear error message, we check - the session and intentionally fail if `does_not_contain_pii` is called using Databricks Connect. + the session and intentionally fail if *does_not_contain_pii* is called using Databricks Connect. """ connect_session_pattern = re.compile(r"127.0.0.1|.*grpc.sock") session = pyspark.sql.SparkSession.builder.getOrCreate() @@ -112,10 +112,10 @@ def _build_detection_udf( def handler(batch: pd.Series) -> pd.Series: def _get_analyzer() -> AnalyzerEngine: """ - Gets an `AnalyzerEngine` for use with PII detection checks. + Gets an *AnalyzerEngine* for use with PII detection checks. Returns: - Presidio `AnalyzerEngine` + Presidio *AnalyzerEngine* """ provider = NlpEngineProvider(nlp_configuration=nlp_engine_config) nlp_engine = provider.create_engine() @@ -131,7 +131,7 @@ def _detect_named_entities(text: str) -> str | None: text: Input text to analyze for named entities Returns: - JSON string with detected entities, or `None` if no entities are found + JSON string with detected entities, or *None* if no entities are found """ if not text: return None diff --git a/src/databricks/labs/dqx/profiler/profiler.py b/src/databricks/labs/dqx/profiler/profiler.py index a1d54f0fc..f61bbf146 100644 --- a/src/databricks/labs/dqx/profiler/profiler.py +++ b/src/databricks/labs/dqx/profiler/profiler.py @@ -115,7 +115,7 @@ def profile_table( Profiles a table to generate summary statistics and data quality rules. Args: - table: The fully-qualified table name (`catalog.schema.table`) to be profiled + table: The fully-qualified table name (*catalog.schema.table*) to be profiled columns: An optional list of column names to include in the profile. If None, all columns are included. options: An optional dictionary of options for profiling. @@ -144,9 +144,9 @@ def profile_tables( exclude_matched: Specifies whether to include tables matched by the pattern. If True, matched tables are excluded. If False, matched tables are included. columns: A dictionary with column names to include in the profile. Keys should be fully-qualified table - names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. + names (e.g. *catalog.schema.table*) and values should be lists of column names to include in profiling. options: A dictionary with options for profiling each table. Keys should be fully-qualified table names - (e.g. `catalog.schema.table`) and values should be options for profiling. + (e.g. *catalog.schema.table*) and values should be options for profiling. Returns: A dictionary mapping table names to tuples containing summary statistics and data quality profiles. @@ -213,11 +213,11 @@ def _profile_tables( Profiles a list of tables to generate summary statistics and data quality rules. Args: - tables: A list of fully-qualified table names (`catalog.schema.table`) to be profiled + tables: A list of fully-qualified table names (*catalog.schema.table*) to be profiled columns: A dictionary with column names to include in the profile. Keys should be fully-qualified table - names (e.g. `catalog.schema.table`) and values should be lists of column names to include in profiling. + names (e.g. *catalog.schema.table*) and values should be lists of column names to include in profiling. options: A dictionary with options for profiling each table. Keys should be fully-qualified table names - (e.g. `catalog.schema.table`) and values should be options for profiling. + (e.g. *catalog.schema.table*) and values should be options for profiling. max_workers: An optional concurrency limit for profiling concurrently Returns: @@ -251,8 +251,8 @@ def _build_options_from_list(table: str, options: list[dict[str, Any]]) -> dict[ Args: table: Table name options: List of options dictionaries with the following fields: - * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options - * `options` - Dictionary of profiler options + * *table* - Table name or wildcard pattern (e.g. *catalog.schema.*) for applying options + * *options* - Dictionary of profiler options Returns: Dictionary of profiler options matching the provided table name @@ -273,8 +273,8 @@ def _match_options_list(table: str, options: list[dict[str, Any]]) -> list[dict[ Args: table: Table name options: List of options dictionaries with the following fields: - * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options - * `options` - Dictionary of profiler options + * *table* - Table name or wildcard pattern (e.g. *catalog.schema.*) for applying options + * *options* - Dictionary of profiler options Returns: List of options dictionaries matching the provided table name @@ -289,8 +289,8 @@ def _sort_options_list(table: str, options: list[dict[str, Any]]) -> list[dict[s Args: table: Table name options: List of options dictionaries with the following fields: - * `table` - Table name or wildcard pattern (e.g. `catalog.schema.*`) for applying options - * `options` - Dictionary of profiler options + * *table* - Table name or wildcard pattern (e.g. *catalog.schema.*) for applying options + * *options* - Dictionary of profiler options Returns: List of options dictionaries sorted by similarity to the provided table name diff --git a/src/databricks/labs/dqx/rule.py b/src/databricks/labs/dqx/rule.py index 299c26302..d700b351e 100644 --- a/src/databricks/labs/dqx/rule.py +++ b/src/databricks/labs/dqx/rule.py @@ -149,17 +149,17 @@ def _determine_rule_type(self, check_func: Callable) -> str | None: class DQRule(abc.ABC, DQRuleTypeMixin, SingleColumnMixin, MultipleColumnsMixin): """Represents a data quality rule that applies a quality check function to column(s) or column expression(s). This class includes the following attributes: - * `check_func` - The function used to perform the quality check. - * `name` (optional) - A custom name for the check; autogenerated if not provided. - * `criticality` (optional) - Defines the severity level of the check: - - `error`: Critical issues. - - `warn`: Potential issues. - * `column` (optional) - A single column to which the check function is applied. - * `columns` (optional) - A list of columns to which the check function is applied. - * `filter` (optional) - A filter expression to apply the check only to rows meeting specific conditions. - * `check_func_args` (optional) - Positional arguments for the check function (excluding `column`). - * `check_func_kwargs` (optional) - Keyword arguments for the check function (excluding `column`). - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + * *check_func* - The function used to perform the quality check. + * *name* (optional) - A custom name for the check; autogenerated if not provided. + * *criticality* (optional) - Defines the severity level of the check: + - *error*: Critical issues. + - *warn*: Potential issues. + * *column* (optional) - A single column to which the check function is applied. + * *columns* (optional) - A list of columns to which the check function is applied. + * *filter* (optional) - A filter expression to apply the check only to rows meeting specific conditions. + * *check_func_args* (optional) - Positional arguments for the check function (excluding *column*). + * *check_func_kwargs* (optional) - Keyword arguments for the check function (excluding *column*). + * *user_metadata* (optional) - User-defined key-value pairs added to metadata generated by the check. """ check_func: Callable @@ -380,16 +380,16 @@ class DQForEachColRule(DQRuleTypeMixin): """Represents a data quality rule that applies to a quality check function repeatedly on each specified column of the provided list of columns. This class includes the following attributes: - * `columns` - A list of column names or expressions to which the check function should be applied. - * `check_func` - The function used to perform the quality check. - * `name` (optional) - A custom name for the check; autogenerated if not provided. - * `criticality` - The severity level of the check: - - `'warn'` for potential issues. - - `'error'` for critical issues. - * `filter` (optional) - A filter expression to apply the check only to rows meeting specific conditions. - * `check_func_args` (optional) - Positional arguments for the check function (excluding column names). - * `check_func_kwargs` (optional) - Keyword arguments for the check function (excluding column names). - * `user_metadata` (optional) - User-defined key-value pairs added to metadata generated by the check. + * *columns* - A list of column names or expressions to which the check function should be applied. + * *check_func* - The function used to perform the quality check. + * *name* (optional) - A custom name for the check; autogenerated if not provided. + * *criticality* - The severity level of the check: + - '*warn*' for potential issues. + - '*error*' for critical issues. + * *filter* (optional) - A filter expression to apply the check only to rows meeting specific conditions. + * *check_func_args* (optional) - Positional arguments for the check function (excluding column names). + * *check_func_kwargs* (optional) - Keyword arguments for the check function (excluding column names). + * *user_metadata* (optional) - User-defined key-value pairs added to metadata generated by the check. """ columns: list[str | Column | list[str | Column]] From 1a322fc52d7652681ec8f515d313649bc50d52f8 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 11:26:55 +0200 Subject: [PATCH 31/37] added header for sidebar --- docs/dqx/src/css/custom.css | 20 +++++++++++++++++++- src/databricks/labs/dqx/check_funcs.py | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/dqx/src/css/custom.css b/docs/dqx/src/css/custom.css index 866392255..c5611e0c0 100644 --- a/docs/dqx/src/css/custom.css +++ b/docs/dqx/src/css/custom.css @@ -58,4 +58,22 @@ button { .header-github-link:hover::before { background-color: var(--ifm-navbar-link-hover-color); -} \ No newline at end of file +} + +/* TOC sidebar header (desktop and mobile) */ +.theme-doc-toc-desktop .table-of-contents::before, +.theme-doc-toc-desktop .tableOfContents::before { + content: "πŸ“„ On this page:"; + display: block; + font-weight: 600; + margin-bottom: 0.5rem; + color: var(--ifm-color-content); +} + +.theme-doc-toc-mobile .menu__list::before { + content: "πŸ“„ On this page:"; + display: block; + font-weight: 600; + margin: 0.5rem 0 0.25rem 0.5rem; + color: var(--ifm-color-content); +} diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 294d6b901..c2c5f7339 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -951,7 +951,7 @@ def sql_query( msg: Optional custom message or Column expression. name: Optional name for the result. negate: If True, the condition is negated (i.e., the check fails when the condition is False). - input_placeholder: Name to be used in the sql query as *{{ input_placeholder }}* to refer to the + input_placeholder: Name to be used in the sql query as `{{ input_placeholder }}` to refer to the input DataFrame on which the checks are applied. row_filter: Optional SQL expression for filtering rows before checking the foreign key. Auto-injected from the check filter. @@ -975,7 +975,7 @@ def sql_query( def _replace_template(sql: str, replacements: dict[str, str]) -> str: """ - Replace `{{ template }}` placeholders in sql with actual names, allowing for whitespace between braces. + Replace *{{ template }}* placeholders in sql with actual names, allowing for whitespace between braces. """ for key, val in replacements.items(): pattern = r"\{\{\s*" + re.escape(key) + r"\s*\}\}" From 09c49f23e5464847df753208aace787df8b36fa5 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 11:39:52 +0200 Subject: [PATCH 32/37] refactor api docs instructions --- docs/dqx/docs/dev/contributing.mdx | 15 +++------------ docs/dqx/docs/dev/docs_authoring.mdx | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index af159af00..00f9b1eed 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -45,7 +45,7 @@ Here are the example steps to submit your first contribution: 3. `git checkout main` (or `gcm` if you're using [ohmyzsh](https://ohmyz.sh/)). 4. `git pull` (or `gl` if you're using [ohmyzsh](https://ohmyz.sh/)). 5. `git checkout -b FEATURENAME` (or `gcb FEATURENAME` if you're using [ohmyzsh](https://ohmyz.sh/)). -6. .. do the work. Please make sure to document all your public methods, functions and classes using [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). We generate API documentation automatically using the docstrings. +6. .. do the work. Please make sure to document all your public methods, functions and classes using [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). We generate API documentation automatically using the docstrings. See more on writing docstrings [here](#writing-docstrings). 7. `make fmt` (Note: If you have an issue with `make fmt`, ensure your IDE folder is ignored in .gitignore. Already added for .idea/ and .cursor/) 8. `make lint` 9. .. fix if any issues are reported @@ -299,16 +299,7 @@ def viz_type(self) -> str: return self.viz.get("type", "UNKNOWN") ``` -## Writing API docs - -API docs are generated using [pydoc-markdown](https://github.com/pypa/pydoc-markdown). - -To generate API docs, run the following command: -```shell -hatch run docs:pydoc-markdown -``` - -The API docs are generated in the `docs/dqx/docs/reference/api` directory. +## Writing Docstrings Use [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) format for the docstrings, so that they are rendered correctly in the API docs. @@ -332,7 +323,7 @@ def method(self, arg1: str, arg2: int) -> str: ``` -* Avoid using backticks around object names in the docstring (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize object names. +* Avoid using backticks around object names in docstrings (e.g., \`arg1\`), as this can cause issues when rendering API documentation. Instead, use italics (e.g., \*arg1\*) to emphasize object names. * Double curly braces are not allowed in the description. Mask them with backslashes, e.g.: `{{`. If you need a code example, use triple backticks, e.g.: ```python diff --git a/docs/dqx/docs/dev/docs_authoring.mdx b/docs/dqx/docs/dev/docs_authoring.mdx index fa7fc52f0..619781333 100644 --- a/docs/dqx/docs/dev/docs_authoring.mdx +++ b/docs/dqx/docs/dev/docs_authoring.mdx @@ -14,6 +14,8 @@ We also use [MDX](https://mdxjs.com/) to write markdown files that include JSX c For styling, we use [Tailwind CSS](https://tailwindcss.com/), a utility-first CSS framework for rapidly building custom designs. +API docs are generated using [pydoc-markdown](https://github.com/pypa/pydoc-markdown). + ## Writing Documentation Most of the documentation is written in markdown files with the `.mdx` extension. @@ -32,7 +34,6 @@ brew install node npm install --global yarn ``` - ## Setup To set up the documentation locally, follow these steps: @@ -115,7 +116,6 @@ make docs-build It will throw an error on any unresolved link. - ## Content alignment and structure of folders When writing documentation, make sure to align the content with the existing documentation. @@ -138,4 +138,16 @@ No need for: - Deep technical details - Implementation details -Or any other details that are not necessary for the **end-user**. \ No newline at end of file +Or any other details that are not necessary for the **end-user**. + +## API Documentation + +The API docs are generated in the `docs/dqx/docs/reference/api` directory. + +To generate API docs, run the following command: +```shell +hatch run docs:pydoc-markdown +``` + +This command is run also as part of make docs build and server. +The command will generate the API documentation from the Python codebase using pydoc-markdown. \ No newline at end of file From d676a52bf598574b0d457d6a084c40f70d54f812 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 11:44:36 +0200 Subject: [PATCH 33/37] updated docstring --- src/databricks/labs/dqx/rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/rule.py b/src/databricks/labs/dqx/rule.py index d700b351e..c54c4dac8 100644 --- a/src/databricks/labs/dqx/rule.py +++ b/src/databricks/labs/dqx/rule.py @@ -384,8 +384,8 @@ class DQForEachColRule(DQRuleTypeMixin): * *check_func* - The function used to perform the quality check. * *name* (optional) - A custom name for the check; autogenerated if not provided. * *criticality* - The severity level of the check: - - '*warn*' for potential issues. - - '*error*' for critical issues. + - *warn* for potential issues. + - *error* for critical issues. * *filter* (optional) - A filter expression to apply the check only to rows meeting specific conditions. * *check_func_args* (optional) - Positional arguments for the check function (excluding column names). * *check_func_kwargs* (optional) - Keyword arguments for the check function (excluding column names). From b68713fa3c05c28939d0f4334daed5d0447ff41c Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 12:10:48 +0200 Subject: [PATCH 34/37] updated docstrings --- src/databricks/labs/dqx/check_funcs.py | 8 ++++---- tests/e2e/test_pii_detection_checks.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index c2c5f7339..cac6fc71d 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -975,7 +975,7 @@ def sql_query( def _replace_template(sql: str, replacements: dict[str, str]) -> str: """ - Replace *{{ template }}* placeholders in sql with actual names, allowing for whitespace between braces. + Replace {{ template }} placeholders in sql with actual names, allowing for whitespace between braces. """ for key, val in replacements.items(): pattern = r"\{\{\s*" + re.escape(key) + r"\s*\}\}" @@ -1458,9 +1458,9 @@ def _match_rows( Perform a null-safe join between two DataFrames based on primary key columns. Ensure that corresponding pk columns are compared together, match by position in pk and ref pk cols. Use eq null safe join to ensure that: - 1 == 1 matches; - NULL <=> NULL matches; - 1 <=> NULL does not match; + - 1 == 1 matches + - NULL <=> NULL matches + - 1 <=> NULL does not match Args: df: The input DataFrame to join. diff --git a/tests/e2e/test_pii_detection_checks.py b/tests/e2e/test_pii_detection_checks.py index 93a2f84a8..036729459 100644 --- a/tests/e2e/test_pii_detection_checks.py +++ b/tests/e2e/test_pii_detection_checks.py @@ -40,8 +40,8 @@ def validate_run_status(run: Run, client: WorkspaceClient) -> None: Validates that a job task run completed successfully. Args: - run: `Run` object returned from a `WorkspaceClient.jobs.submit(...)` command. - client: `WorkspaceClient` object for getting task output. + run: *Run* object returned from a *WorkspaceClient.jobs.submit(...)* command. + client: *WorkspaceClient* object for getting task output. """ task = run.tasks[0] termination_details = run.status.termination_details From 2e9400734324c799f3ae768ef596055d1a2248d5 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 12:30:31 +0200 Subject: [PATCH 35/37] Added definition of done --- docs/dqx/docs/dev/contributing.mdx | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 00f9b1eed..0dc210ba0 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -45,7 +45,7 @@ Here are the example steps to submit your first contribution: 3. `git checkout main` (or `gcm` if you're using [ohmyzsh](https://ohmyz.sh/)). 4. `git pull` (or `gl` if you're using [ohmyzsh](https://ohmyz.sh/)). 5. `git checkout -b FEATURENAME` (or `gcb FEATURENAME` if you're using [ohmyzsh](https://ohmyz.sh/)). -6. .. do the work. Please make sure to document all your public methods, functions and classes using [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). We generate API documentation automatically using the docstrings. See more on writing docstrings [here](#writing-docstrings). +6. .. do the work and make sure [Definition of Done (DoD) items](#definition-of-done) are fulfilled. 7. `make fmt` (Note: If you have an issue with `make fmt`, ensure your IDE folder is ignored in .gitignore. Already added for .idea/ and .cursor/) 8. `make lint` 9. .. fix if any issues are reported @@ -72,6 +72,24 @@ Here are the example steps to submit your first contribution: request description to [automatically link it](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue) to an existing issue. +## Definition of Done + +Please ensure the following DoDs are met before submitting your pull request: +- [ ] **Code formatting** β€” Code is formatted consistently with the project style (`make fmt`). +- [ ] **Linting & checks** β€” Code passes linting and static analysis (`make lint`). +- [ ] **Testing** β€” Unit, integration and e2e tests cover changes and all tests pass (`make test`, `make integration`, `make e2e`). +- [ ] **Docstrings** β€” Public functions, classes, and modules include [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). Docstrings are clean and consistent ([guidance](#writing-docstrings)). +- [ ] **Documentation** β€” If user-facing behavior changes, ensure docs are updated. See [docs authoring](/docs/dev/docs_authoring). +- [ ] **Commit signing** β€” Commits are GPG-signed (e.g., `git commit -S -a -m "your message"`). +- [ ] **Pull request requirements** β€” PR is linked to an existing issue and includes a clear description of the changes. +- [ ] **Backward compatibility** β€” Check that changes don’t break existing APIs or document breaking changes clearly. +- [ ] **Security considerations** β€” Sensitive data (keys, passwords, etc.) are not hardcoded or exposed. +- [ ] **Performance considerations** β€” New code does not introduce obvious performance bottlenecks. Benchmarks added if performance is a concern. + +Alternatively, you can create a [Draft PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you are not ready to submit it yet. +This will allow you to get initial feedback without worrying about the Definition of Done (DoD) items. +However, please note that integration and e2e tests are not run automatically for Draft PRs. They will only be run when you mark the PR as ready for review. + ## Local Setup This section provides a step-by-step guide for setting up and starting work on the project. These steps will help you set up your project environment and dependencies for efficient development. From 4870bc010e22e6333f9aa8499c36a3a3fbe019b8 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 12:30:58 +0200 Subject: [PATCH 36/37] updated definition of done --- docs/dqx/docs/dev/contributing.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 0dc210ba0..d700b737f 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -82,7 +82,7 @@ Please ensure the following DoDs are met before submitting your pull request: - [ ] **Documentation** β€” If user-facing behavior changes, ensure docs are updated. See [docs authoring](/docs/dev/docs_authoring). - [ ] **Commit signing** β€” Commits are GPG-signed (e.g., `git commit -S -a -m "your message"`). - [ ] **Pull request requirements** β€” PR is linked to an existing issue and includes a clear description of the changes. -- [ ] **Backward compatibility** β€” Check that changes don’t break existing APIs or document breaking changes clearly. +- [ ] **Backward compatibility** β€” Check that changes don't break existing APIs or document breaking changes clearly in the PR. - [ ] **Security considerations** β€” Sensitive data (keys, passwords, etc.) are not hardcoded or exposed. - [ ] **Performance considerations** β€” New code does not introduce obvious performance bottlenecks. Benchmarks added if performance is a concern. From 62debffe953159083ba35fddb2c442f02459ab6e Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 17 Aug 2025 12:41:34 +0200 Subject: [PATCH 37/37] updated definition of done --- docs/dqx/docs/dev/contributing.mdx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index d700b737f..8cc803b3d 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -86,9 +86,19 @@ Please ensure the following DoDs are met before submitting your pull request: - [ ] **Security considerations** β€” Sensitive data (keys, passwords, etc.) are not hardcoded or exposed. - [ ] **Performance considerations** β€” New code does not introduce obvious performance bottlenecks. Benchmarks added if performance is a concern. -Alternatively, you can create a [Draft PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you are not ready to submit it yet. -This will allow you to get initial feedback without worrying about the Definition of Done (DoD) items. -However, please note that integration and e2e tests are not run automatically for Draft PRs. They will only be run when you mark the PR as ready for review. +Alternatively, you may open a [Draft PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if your work is not yet ready for submission. +This is a good way to gather early feedback without needing to meet all Definition of Done (DoD) requirements. + + +* Draft PRs β€” Integration and end-to-end (e2e) tests are not run automatically on Draft PRs. These tests will only execute once the PR is marked as "Ready for Review". +* Forked PRs β€” No checks are run automatically for PRs opened from forks. + Project maintainers must approve it. + Integration and end-to-end (e2e) tests cannot be run for PRs opened from forks. + In these cases, project maintainers will run the tests manually before merging the code. + The contributor should ensure that the tests are passing in their local environment before submitting the PR. + +In all cases, the full test suite is executed on code merged into `main` branch as part of the nightly CI/CD pipeline. + ## Local Setup