Adding support for running checks on multiple tables

demos/dqx_demo_tool.py

@@ -7,7 +7,7 @@
 # MAGIC
 # MAGIC Run in your terminal: `databricks labs install dqx`
 # MAGIC
-# MAGIC When prompt provide the following:
+# MAGIC When prompt provide the following and leave other options as default:
 # MAGIC * Input data location: `/databricks-datasets/delta-sharing/samples/nyctaxi_2019`
 # MAGIC * Input format: `delta`
 # MAGIC * Output table: valid fully qualified table name (catalog.schema.table when using Unity Catalog or schema.table). The output data will be saved there as part of the demo.
@@ -24,6 +24,8 @@
 # MAGIC log_level: INFO
 # MAGIC version: 1
 # MAGIC serverless_clusters: true # optional
+# MAGIC profiler_max_parallelism: 4
+# MAGIC quality_checker_max_parallelism: 4
 # MAGIC run_configs:
 # MAGIC - name: default
 # MAGIC   checks_location: checks.yml
@@ -68,10 +70,20 @@
 # MAGIC
 # MAGIC Run in the terminal:
 # MAGIC ```
+# MAGIC # run for all configured run configs (default)
+# MAGIC databricks labs dqx profile
+# MAGIC
+# MAGIC # or run for a specific run config
 # MAGIC databricks labs dqx profile --run-config "default"
+# MAGIC
+# MAGIC # You can also run for tables/views matching wildcard patterns. Conventions:
+# MAGIC # * Run config is used as a template for all relevant fields except location
+# MAGIC # * Input table location is derived from the patterns
+# MAGIC # * For table-based checks location, save checks to the delta table, otherwise to install_folder/checks/<input_table>.yml
+# MAGIC databricks labs dqx profile --run-config "default" --patterns "main.nytaxi.*;main.default.table"
 # MAGIC ```
 # MAGIC
-# MAGIC This will profile the data defined in the `input_config` field of the config. The generated quality rule candidates and summary statistics are saved in the installation folder as per the `checks_location`, `profiler_config` fields.
+# MAGIC This will profile the data defined in the `input_config` field of the run config. The generated quality rule candidates and summary statistics are saved in the installation folder as per the `checks_location`, `profiler_config` fields.
 
 # COMMAND ----------
 
@@ -80,35 +92,62 @@
 # MAGIC
 # MAGIC Run in the terminal:
 # MAGIC ```
+# MAGIC # run for all configured run configs (default)
+# MAGIC databricks labs dqx apply-checks
+# MAGIC
+# MAGIC # or run for a specific run config
 # MAGIC databricks labs dqx apply-checks --run-config "default"
+# MAGIC
+# MAGIC # You can also run for tables/views matching wildcard patterns. Conventions:
+# MAGIC # * Run config is used as a template for all relevant fields except location
+# MAGIC # * Input table location is derived from the patterns
+# MAGIC # * For table-based checks location, load checks from the delta table, otherwise from install_folder/checks/<input_table>.yml
+# MAGIC # * For output and quarantine tables location, use <input_table>_dq_output and <input_table>_dq_quarantine suffixes
+# MAGIC databricks labs dqx apply-checks --run-config "default" --patterns "main.nytaxi.*;main.default.table"
 # MAGIC ```
 # MAGIC
-# MAGIC This will apply quality checks defined in the `checks_location` field of the config to the data defined in the `input_config`. The results are written to the output as defined in the `output_config` and `quarantine_config` fields.
+# MAGIC This will apply quality checks defined in the `checks_location` field of the run config to the data defined in the `input_config`. The results are written to the output as defined in the `output_config` and `quarantine_config` fields.
 
 # COMMAND ----------
 
 # MAGIC %md
 # MAGIC ### Run end-to-end (e2e) workflow
 # MAGIC
-# MAGIC You can optionally, run both profiler and quality checker in a sequency using the e2e workflow.
+# MAGIC You can optionally, run both profiler and quality checker in a sequence using the e2e workflow.
 # MAGIC
 # MAGIC Run in the terminal:
 # MAGIC ```
+# MAGIC # run for all configured run configs (default)
+# MAGIC databricks labs dqx e2e
+# MAGIC
+# MAGIC # or run for a specific run config
 # MAGIC databricks labs dqx e2e --run-config "default"
+# MAGIC
+# MAGIC # You can also run for tables/views matching wildcard patterns. Conventions:
+# MAGIC # * Run config is used as a template for all relevant fields except location
+# MAGIC # * Input table location is derived from the patterns
+# MAGIC # * For table-based checks location, save/load checks from the delta table, otherwise from install_folder/checks/<input_table>.yml
+# MAGIC # * For output and quarantine tables location, use <input_table>_dq_output and <input_table>_dq_quarantine suffixes
+# MAGIC databricks labs dqx e2e --run-config "default" --patterns "main.nytaxi.*;main.default.table"
 # MAGIC ```
 # MAGIC
 # MAGIC This will use the settings from the profiler and quality checker as explained before.
 
 # COMMAND ----------
 
+# MAGIC %md
+# MAGIC You can also profile and run quality checking across multiple tables in a single method call (see multi table demo).
+
+# COMMAND ----------
+
 # MAGIC %md
 # MAGIC ## Programmatic Approach
 
 # COMMAND ----------
 
 # MAGIC %md
 # MAGIC ### Installation of DQX in the Databricks cluster
-# MAGIC Once DQX is installed in the workspace, we install it in the DQX library in the cluster.
+# MAGIC Once DQX is installed in the workspace as a tool, it must be installed in the cluster.
 
 # COMMAND ----------
 
@@ -171,7 +210,7 @@
 from databricks.labs.dqx.engine import DQEngine
 from databricks.labs.dqx.config import InstallationChecksStorageConfig, WorkspaceFileChecksStorageConfig
 from databricks.labs.dqx.config_loader import RunConfigLoader
-from databricks.labs.dqx.utils import read_input_data
+from databricks.labs.dqx.io import read_input_data
 from databricks.sdk import WorkspaceClient
 
 
@@ -285,7 +324,7 @@
 # COMMAND ----------
 
 from databricks.labs.dqx.engine import DQEngine
-from databricks.labs.dqx.utils import read_input_data
+from databricks.labs.dqx.io import read_input_data
 from databricks.sdk import WorkspaceClient
 from databricks.labs.dqx.config import InstallationChecksStorageConfig, WorkspaceFileChecksStorageConfig
 from databricks.labs.dqx.config_loader import RunConfigLoader
@@ -343,7 +382,7 @@
 # COMMAND ----------
 
 # MAGIC %md
-# MAGIC ### End-to-end programmatic approach 
+# MAGIC ### End-to-end programmatic approach
 # MAGIC
 # MAGIC You can use a single method call to apply checks and save the results.
 
@@ -364,6 +403,11 @@
 
 # COMMAND ----------
 
+# MAGIC %md
+# MAGIC You can also profile and run quality checking across multiple tables in a single method call (see multi table demo).
+
+# COMMAND ----------
+
 # MAGIC %md
 # MAGIC ### View data quality in DQX Dashboard