Support passing of index URLs to piplite (CLI, runtime configuration, and requirements files)

jupyterlite_pyodide_kernel/tests/test_piplite.py

@@ -135,6 +135,69 @@ def a_lite_config_file(request, an_empty_lite_dir):
     return an_empty_lite_dir / request.param
 
 
+@pytest.mark.parametrize(
+    "index_urls",
+    [
+        "https://example.com/simple",
+        ["https://example.com/simple", "https://pypi.org/simple"],
+    ],
+    ids=["string", "list"],
+)
+def test_validate_piplite_install_default_options_valid(
+    script_runner, a_lite_config_file, index_urls
+):
+    """valid index_urls (string or list) in pipliteInstallDefaultOptions passes check"""
+    lite_dir = a_lite_config_file.parent
+    output = lite_dir / "_output"
+
+    build = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert build.success
+    shutil.copy2(output / a_lite_config_file.name, a_lite_config_file)
+
+    whole_file = config_data = json.loads(a_lite_config_file.read_text(**UTF8))
+    if a_lite_config_file.name == JUPYTERLITE_IPYNB:
+        config_data = whole_file["metadata"][JUPYTERLITE_METADATA]
+
+    config_data[JUPYTER_CONFIG_DATA].setdefault(LITE_PLUGIN_SETTINGS, {}).setdefault(
+        PYODIDE_KERNEL_PLUGIN_ID, {}
+    )["pipliteInstallDefaultOptions"] = {"index_urls": index_urls}
+
+    a_lite_config_file.write_text(json.dumps(whole_file, **JSON_FMT), **UTF8)
+    rebuild = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert rebuild.success
+
+    check = script_runner.run(["jupyter", "lite", "check"], cwd=str(lite_dir))
+    assert check.success, f"check failed for index_urls={index_urls!r}"
+
+
+def test_validate_piplite_install_default_options_invalid(
+    script_runner, a_lite_config_file
+):
+    """a non-string, non-list index_urls in pipliteInstallDefaultOptions fails check"""
+    lite_dir = a_lite_config_file.parent
+    output = lite_dir / "_output"
+
+    build = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert build.success
+    shutil.copy2(output / a_lite_config_file.name, a_lite_config_file)
+
+    whole_file = config_data = json.loads(a_lite_config_file.read_text(**UTF8))
+    if a_lite_config_file.name == JUPYTERLITE_IPYNB:
+        config_data = whole_file["metadata"][JUPYTERLITE_METADATA]
+
+    config_data[JUPYTER_CONFIG_DATA].setdefault(LITE_PLUGIN_SETTINGS, {}).setdefault(
+        PYODIDE_KERNEL_PLUGIN_ID, {}
+    )["pipliteInstallDefaultOptions"] = {"index_urls": 42}
+
+    invalid_config = json.dumps(whole_file, **JSON_FMT)
+    a_lite_config_file.write_text(invalid_config, **UTF8)
+    rebuild = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert rebuild.success
+
+    recheck = script_runner.run(["jupyter", "lite", "check"], cwd=str(lite_dir))
+    assert not recheck.success, invalid_config
+
+
 def test_validate_config(script_runner, a_lite_config_file):
     lite_dir = a_lite_config_file.parent
     output = lite_dir / "_output"

packages/pyodide-kernel-extension/schema/kernel.v0.schema.json

@@ -25,6 +25,29 @@
       "default": [],
       "format": "uri"
     },
+    "pipliteInstallDefaultOptions": {
+      "type": "object",
+      "description": "Default options to pass to piplite.install",
+      "default": {},
+      "properties": {
+        "index_urls": {
+          "description": "Base URL(s) of extra indices to use, passed as `index_urls` to micropip.install. Accepts a single URL string or a list of URL strings.",
+          "oneOf": [
+            {
+              "type": "string",
+              "format": "uri"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "format": "uri"
+              }
+            }
+          ]
+        }
+      }
+    },
     "loadPyodideOptions": {
       "type": "object",
       "description": "additional options to provide to `loadPyodide`, see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide",

packages/pyodide-kernel-extension/src/index.ts

@@ -61,6 +61,7 @@ const kernel: JupyterFrontEndPlugin<void> = {
     const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href);
     const disablePyPIFallback = !!config.disablePyPIFallback;
     const loadPyodideOptions = config.loadPyodideOptions || {};
+    const pipliteInstallDefaultOptions = config.pipliteInstallDefaultOptions || {};
 
     for (const [key, value] of Object.entries(loadPyodideOptions)) {
       if (key.endsWith('URL') && typeof value === 'string') {
@@ -116,6 +117,7 @@ const kernel: JupyterFrontEndPlugin<void> = {
           mountDrive,
           loadPyodideOptions,
           contentsManager,
+          pipliteInstallDefaultOptions,
           browsingContextId: serviceWorkerManager?.browsingContextId,
           logger,
         });

packages/pyodide-kernel/py/piplite/piplite/cli.py

@@ -13,7 +13,7 @@ async def install(
         deps: bool = True,                              # --no-deps
         credentials: str | None = None,                 # no CLI alias
         pre: bool = False,                              # --pre
-        index_urls: list[str] | str | None = None,      # no CLI alias
+        index_urls: list[str] | str | None = None,      # -i and --index-url
         *,
         constraints: list[str] | None = None,           # --constraints
         reinstall: bool = False,                        # no CLI alias
@@ -37,6 +37,11 @@ async def install(
 
 REQ_FILE_SPEC = r"^(?P<flag>-r|--requirements)\s*=?\s*(?P<path_ref>.+)$"
 
+# Matches --index-url or -i directives (with optional = separator and optional quotes)
+INDEX_URL_SPEC = (
+    r'^(--index-url|-i)\s*=?\s*(?:"([^"]*)"|\047([^\047]*)\047|([^\s]*))\s*$'
+)
+
 __all__ = ["get_transformed_code"]
 
 
@@ -95,6 +100,13 @@ def _get_parser() -> ArgumentParser:
         action="store_true",
         help="whether pre-release packages should be considered",
     )
+    parser.add_argument(
+        "--index-url",
+        "-i",
+        type=str,
+        default=None,
+        help="base URL of the package index to use for lookup",
+    )
     parser.add_argument(
         "--force-reinstall",
         action="store_true",
@@ -138,8 +150,7 @@ async def get_action_kwargs(argv: list[str]) -> tuple[str | None, dict[str, Any]
     except (Exception, SystemExit):
         return None, {}
 
-    kwargs = {}
-
+    kwargs: dict[str, Any] = {}
     action = args.action
 
     if action == "install":
@@ -154,33 +165,54 @@ async def get_action_kwargs(argv: list[str]) -> tuple[str | None, dict[str, Any]
         if args.verbose:
             kwargs["keep_going"] = True
 
+        if args.index_url:
+            kwargs["index_urls"] = args.index_url
+
         if args.force_reinstall:
             kwargs["reinstall"] = True
 
+        collected_index_urls: list[str] = []
         for req_file in args.requirements or []:
-            async for spec in _specs_from_requirements_file(Path(req_file)):
+            async for spec in _specs_from_requirements_file(
+                Path(req_file), collected_index_urls=collected_index_urls
+            ):
                 kwargs["requirements"] += [spec]
 
         for const_file in args.constraints or []:
             async for spec in _specs_from_requirements_file(Path(const_file)):
-                kwargs["constraints"] += [spec]
+                kwargs.setdefault("constraints", []).append(spec)
+
+        # Apply index URLs from requirements files only if --index-url was not
+        # already given on the command line.
+        if collected_index_urls and "index_urls" not in kwargs:
+            kwargs["index_urls"] = collected_index_urls
 
     return action, kwargs
 
 
-async def _specs_from_requirements_file(spec_path: Path) -> AsyncIterator[str]:
+async def _specs_from_requirements_file(
+    spec_path: Path,
+    *,
+    collected_index_urls: list[str] | None = None,
+) -> AsyncIterator[str]:
     """Extract package specs from a ``requirements.txt``-style file."""
     if not spec_path.exists():
         warn(f"piplite could not find requirements file {spec_path}")
         return
 
-    for line_no, line in enumerate(spec_path.read_text(encoding="utf").splitlines()):
-        async for spec in _specs_from_requirements_line(spec_path, line_no + 1, line):
+    for line_no, line in enumerate(spec_path.read_text(encoding="utf-8").splitlines()):
+        async for spec in _specs_from_requirements_line(
+            spec_path, line_no + 1, line, collected_index_urls=collected_index_urls
+        ):
             yield spec
 
 
 async def _specs_from_requirements_line(
-    spec_path: Path, line_no: int, line: str
+    spec_path: Path,
+    line_no: int,
+    line: str,
+    *,
+    collected_index_urls: list[str] | None = None,
 ) -> AsyncIterator[str]:
     """Get package specs from a line of a ``requirements.txt``-style file.
 
@@ -195,13 +227,23 @@ async def _specs_from_requirements_line(
     if file_match:
         ref = file_match.groupdict()["path_ref"]
         ref_path = Path(ref if ref.startswith("/") else spec_path.parent / ref)
-        async for sub_spec in _specs_from_requirements_file(ref_path):
+        async for sub_spec in _specs_from_requirements_file(
+            ref_path, collected_index_urls=collected_index_urls
+        ):
             yield sub_spec
+        return
+
+    index_url_match = re.match(INDEX_URL_SPEC, raw)
+    if index_url_match:
+        # Extract the URL from whichever capture group matched (quoted or bare)
+        # and collect it so the caller can forward it to piplite.install.
+        url = next((g for g in index_url_match.groups()[1:] if g is not None), None)
+        if url and collected_index_urls is not None:
+            collected_index_urls.append(url)
+        return
     elif raw.startswith("-"):
         warn(f"{spec_path}:{line_no}: unrecognized spec: {raw}")
         return
-    else:
-        spec = raw
 
-    if spec:
-        yield spec
+    if raw:
+        yield raw

packages/pyodide-kernel/py/piplite/piplite/piplite.py

@@ -20,18 +20,26 @@
 logger = logging.getLogger(__name__)
 
 
-#: a list of Warehouse-like API endpoints or derived multi-package all.json
-_PIPLITE_URLS: list[str] = []
-
 #: a cache of available packages
 _PIPLITE_INDICES: dict[str, dict[str, Any]] = {}
 
-#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS
-_PIPLITE_DISABLE_PYPI = False
-
 #: a well-known file name respected by the rest of the build chain
 ALL_JSON = "/all.json"
 
+#: This is the runtime configuration set from the JS side via worker.ts, which acts as
+#: a single source of truth for all site-level piplite settings.
+#:
+#: Keys written by worker.ts on every kernel start:
+#:   piplite_urls  – list of local all.json warehouse index URLs
+#:   disable_pypi  – whether to block fallback to pypi.org
+#:
+#: Keys that are optionally written by worker.ts from pipliteInstallDefaultOptions:
+#:   index_urls    – default index URL(s) that get forwarded to micropip.install
+_PIPLITE_DEFAULT_INSTALL_ARGS: dict[str, Any] = {
+    "piplite_urls": [],  # a list of Warehouse-like API endpoints or derived multi-package all.json
+    "disable_pypi": False,  # don't fall back to pypi.org if package not found in _PIPLITE_URLS
+}
+
 
 class PiplitePyPIDisabled(ValueError):
     """An error for when PyPI is disabled at the site level, but a download was
@@ -91,7 +99,7 @@ async def _query_package(
     fetch_kwargs: dict[str, Any] | None = None,
 ) -> ProjectInfo:
     """Fetch the warehouse API metadata for a specific ``pkgname``."""
-    for piplite_url in _PIPLITE_URLS:
+    for piplite_url in _PIPLITE_DEFAULT_INSTALL_ARGS.get("piplite_urls", []):
         if not piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON):
             logger.warning("Non-all.json piplite URL not supported %s", piplite_url)
             continue
@@ -105,7 +113,7 @@ async def _query_package(
         if pypi_json_from_index:
             return pypi_json_from_index
 
-    if _PIPLITE_DISABLE_PYPI:
+    if _PIPLITE_DEFAULT_INSTALL_ARGS.get("disable_pypi", False):
         raise PiplitePyPIDisabled(
             f"{name} could not be installed: PyPI fallback is disabled"
         )
@@ -130,7 +138,17 @@ async def _install(
     constraints: list[str] | None = None,
     reinstall: bool = False,
 ):
-    """Invoke micropip.install with a patch to get data from local indexes"""
+    """Invoke micropip.install with a patch to get data from local indexes.
+
+    If ``index_urls`` is not explicitly provided and a default has been
+    configured via ``_PIPLITE_DEFAULT_INSTALL_ARGS`` (e.g. from
+    ``pipliteInstallDefaultOptions`` in ``jupyter-lite.json``), the default
+    is used.
+    """
+    effective_index_urls = index_urls
+    if effective_index_urls is None:
+        effective_index_urls = _PIPLITE_DEFAULT_INSTALL_ARGS.get("index_urls")
+
     with patch("micropip.package_index.query_package", _query_package):
         return await micropip.install(
             requirements=requirements,
@@ -139,7 +157,7 @@ async def _install(
             credentials=credentials,
             pre=pre,
             constraints=constraints,
-            index_urls=index_urls,
+            index_urls=effective_index_urls,
             verbose=verbose,
             reinstall=reinstall,
         )

packages/pyodide-kernel/src/kernel.ts

@@ -157,6 +157,7 @@ export class PyodideKernel extends BaseKernel implements IKernel {
       location: this.location,
       mountDrive: options.mountDrive,
       loadPyodideOptions: options.loadPyodideOptions || {},
+      pipliteInstallDefaultOptions: options.pipliteInstallDefaultOptions,
       browsingContextId: options.browsingContextId,
       kernelId: this.id,
     };
@@ -432,6 +433,11 @@ export namespace PyodideKernel {
      */
     mountDrive: boolean;
 
+    /**
+     * Default options to pass to piplite.install
+     */
+    pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions;
+
     /**
      * additional options to provide to `loadPyodide`
      * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide

packages/pyodide-kernel/src/tokens.ts

@@ -83,9 +83,26 @@ export interface ICoincidentPyodideWorkerKernel extends IPyodideWorkerKernel {
 export type IRemotePyodideWorkerKernel = IPyodideWorkerKernel;
 
 /**
- * An namespace for Pyodide workers.
+ * A namespace for Pyodide workers.
  */
 export namespace IPyodideWorkerKernel {
+  /**
+   * Options for piplite installation.
+   */
+  export interface IPipliteInstallOptions {
+    /**
+     * Base URL(s) of extra indices to use, forwarded to micropip.install as
+     * ``index_urls``.  Accepts either a single URL string or a list of URL
+     * strings.
+     */
+    index_urls?: string | string[];
+
+    /**
+     * Any additional piplite install options.
+     */
+    [key: string]: any;
+  }
+
   /**
    * Initialization options for a worker.
    */
@@ -125,6 +142,11 @@ export namespace IPyodideWorkerKernel {
      */
     mountDrive: boolean;
 
+    /**
+     * Default options to pass to piplite.install (e.g. index_urls).
+     */
+    pipliteInstallDefaultOptions?: IPipliteInstallOptions;
+
     /**
      * A unique ID to identify the origin of this request.
      * This should be provided by `IServiceWorkerManager` and is used to