Document for private PYPI and Auto Upload dependencies for workspace installation

docs/dqx/docs/installation.mdx

@@ -358,6 +358,28 @@ resources:
             #    package: databricks-labs-dqx==0.8.0
 ```
 
+### Installing DQX with company-hosted PyPI mirror
+
+Some enterprises block the public PyPI index and host a company-controlled PyPI mirror. To install DQX while using a company-hosted PyPI mirror for finding its dependencies, add all DQX dependencies to the company-hosted PyPI mirror (see "dependencies" in [`pyproject.toml`](https://github.com/databrickslabs/dqx/blob/v0.10.0/pyproject.toml)) and set the environment variable `PIP_INDEX_URL` to the company-hosted PyPI mirror URL while installing DQX:
+
+```commandline
+PIP_INDEX_URL="https://url-to-company-hosted-pypi.internal" databricks labs install dqx
+```
+
+#### Installing DQX as a tool
+
+During DQX installation as a tool in the workspace installation reply *yes* to the question "Does the given workspace block Internet access"?
+
+If the host has no access to github, then dqx installation will not be able to download the files locally and will fail.
+In order to address that, follow the steps below
+ - install dqx on a host which has access to github using the instruction above
+ - zip the installation from ~/.databricks/labs/dqx
+ - copy the zip file to the target host and unzip
+Now the installation can be done in offline mode (ensure databricks cli is upgraded to version v0.244.0 or higher)
+```commandline
+PIP_INDEX_URL="https://url-to-company-hosted-pypi.internal" databricks labs install dqx --offline=true
+```
+
 ### Upgrade DQX in the Databricks workspace
 
 Verify that DQX is installed:

src/databricks/labs/dqx/config.py

@@ -141,6 +141,9 @@ class WorkspaceConfig:
 
     # whether to use serverless clusters for the jobs, only used during workspace installation
     serverless_clusters: bool = True
+    upload_dependencies: bool = (
+        False  # whether to upload dependencies to the workspace during installation to enable DQX in restricted (no-internet) environments
+    )
     extra_params: ExtraParams | None = None  # extra parameters to pass to the jobs, e.g. result_column_names
 
     # cluster configuration for the jobs (applicable for non-serverless clusters only)

src/databricks/labs/dqx/installer/config_provider.py

@@ -98,6 +98,9 @@ def prompt_new_installation(self, install_folder: str | None = None) -> Workspac
 
         warehouse_id = self._warehouse_configurator.create()
 
+        # Ask if the workspace blocks Internet access to determine if dependencies should be uploaded
+        upload_dependencies = self._prompts.confirm("Does the given workspace block Internet access?")
+
         return WorkspaceConfig(
             log_level=log_level,
             run_configs=[
@@ -114,6 +117,7 @@ def prompt_new_installation(self, install_folder: str | None = None) -> Workspac
                 )
             ],
             serverless_clusters=serverless_clusters,
+            upload_dependencies=upload_dependencies,
             profiler_spark_conf=profiler_spark_conf,
             profiler_override_clusters=profiler_override_clusters,
             quality_checker_spark_conf=quality_checker_spark_conf,

src/databricks/labs/dqx/installer/workflow_installer.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import importlib.metadata
 import logging
 import os.path
 import re
@@ -487,9 +488,79 @@ def _library_dep_order(library: str):
             case _:
                 return 2
 
+    @staticmethod
+    def _extract_dependency_prefix(requirement: str) -> str | None:
+        """
+        Extract package name prefix from a requirement string.
+
+        Args:
+            requirement: Requirement string (e.g., "databricks-sdk>=0.71")
+
+        Returns:
+            Package name prefix with underscores, or None if invalid.
+        """
+        match = re.match(r'^([a-zA-Z0-9-]+)', requirement)
+        if match:
+            pkg_name = match.group(1)
+            return pkg_name.replace('-', '_')
+        return None
+
+    @staticmethod
+    def _get_fallback_dependencies() -> list[str]:
+        """
+        Get fallback dependency prefixes when metadata is unavailable. The list should match the dependency list from the pyproject.toml
+
+        Returns:
+            List of core dependency prefixes.
+        """
+        return [
+            "databricks_labs_blueprint",
+            "databricks_sdk",
+            "databricks_labs_lsql",
+            "sqlalchemy",
+        ]
+
+    @staticmethod
+    def _get_dependency_prefixes() -> list[str]:
+        """
+        Dynamically retrieve dependency prefixes from package metadata.
+
+        Includes both core and optional (extras) dependencies to ensure workflows
+        have access to all required packages.
+
+        Returns:
+            List of dependency package name prefixes for wheel files.
+        """
+        try:
+            requires = importlib.metadata.requires('databricks-labs-dqx')
+        except importlib.metadata.PackageNotFoundError:
+            logger.warning("databricks-labs-dqx package metadata not found, using fallback dependencies")
+            return WorkflowDeployment._get_fallback_dependencies()
+
+        if not requires:
+            logger.warning("No dependencies found in package metadata")
+            return []
+
+        prefixes = []
+        for req in requires:
+            prefix = WorkflowDeployment._extract_dependency_prefix(req)
+            if prefix:
+                prefixes.append(prefix)
+
+        # Remove duplicates while preserving order
+        unique_prefixes = list(dict.fromkeys(prefixes))
+        logger.info(f"Discovered {len(unique_prefixes)} dependencies from package metadata (including extras)")
+        return unique_prefixes
+
     def _upload_wheel(self):
         wheel_paths = []
         with self._wheels:
+            # Upload dependencies if workspace blocks Internet access
+            if self._config.upload_dependencies:
+                logger.info("Uploading dependencies to workspace...")
+                dependency_prefixes = self._get_dependency_prefixes()
+                for whl in self._wheels.upload_wheel_dependencies(dependency_prefixes):
+                    wheel_paths.append(whl)
             wheel_paths.sort(key=WorkflowDeployment._library_dep_order)
             wheel_paths.append(self._wheels.upload_to_wsfs())
             wheel_paths = [f"/Workspace{wheel}" for wheel in wheel_paths]