diff --git a/examples/jupyter-lite.json b/examples/jupyter-lite.json index fe309fc4..8b9c6e3c 100644 --- a/examples/jupyter-lite.json +++ b/examples/jupyter-lite.json @@ -4,8 +4,11 @@ "appName": "JupyterLite Pyodide Kernel", "litePluginSettings": { "@jupyterlite/pyodide-kernel-extension:kernel": { + "pipliteInstallDefaultOptions": { + "index_urls": "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" + }, "loadPyodideOptions": { - "packages": ["matplotlib", "micropip", "numpy", "sqlite3", "ssl"], + "packages": ["micropip", "sqlite3", "ssl"], "lockFileURL": "https://cdn.jsdelivr.net/pyodide/v0.27.2/full/pyodide-lock.json?from-lite-config=1" } } diff --git a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json index 8449920f..24264109 100644 --- a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json +++ b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json @@ -25,6 +25,21 @@ "default": [], "format": "uri" }, + "pipliteInstallDefaultOptions": { + "type": "object", + "description": "Default options to pass to piplite.install", + "default": {}, + "properties": { + "index_urls": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Base URLs of extra indices to use" + } + } + }, "loadPyodideOptions": { "type": "object", "description": "additional options to provide to `loadPyodide`, see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide", diff --git a/packages/pyodide-kernel-extension/src/index.ts b/packages/pyodide-kernel-extension/src/index.ts index 448143b2..2ff65f78 100644 --- a/packages/pyodide-kernel-extension/src/index.ts +++ b/packages/pyodide-kernel-extension/src/index.ts @@ -58,6 +58,10 @@ const kernel: JupyterLiteServerPlugin = { const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href); const disablePyPIFallback = !!config.disablePyPIFallback; const loadPyodideOptions = config.loadPyodideOptions || {}; + const pipliteInstallDefaultOptions = config.pipliteInstallDefaultOptions || {}; + + // Parse any configured index URLs + const index_urls = pipliteInstallDefaultOptions.index_urls || []; for (const [key, value] of Object.entries(loadPyodideOptions)) { if (key.endsWith('URL') && typeof value === 'string') { @@ -99,6 +103,10 @@ const kernel: JupyterLiteServerPlugin = { mountDrive, loadPyodideOptions, contentsManager, + pipliteInstallDefaultOptions: { + index_urls, + ...pipliteInstallDefaultOptions, + }, }); }, }); diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index f4da0311..349ea9a0 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -13,7 +13,7 @@ async def install( deps: bool = True, # --no-deps credentials: str | None = None, # no CLI alias pre: bool = False, # --pre - index_urls: list[str] | str | None = None, # no CLI alias + index_urls: list[str] | str | None = None, # -i and --index-url *, verbose: bool | int | None = None, ): @@ -26,11 +26,37 @@ async def install( import re import sys import typing +from typing import Optional, List, Tuple +from dataclasses import dataclass + from argparse import ArgumentParser from pathlib import Path + +@dataclass +class RequirementsContext: + """Track state while parsing requirements files.""" + + index_url: Optional[str] = None + requirements: List[str] = None + + def __post_init__(self): + if self.requirements is None: + self.requirements = [] + + def add_requirement(self, req: str): + """Add a requirement with the currently active index URL.""" + self.requirements.append((req, self.index_url)) + + REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" +# Matches a pip-style index URL, with support for quote enclosures +INDEX_URL_PREFIX = ( + r'^(--index-url|-i)\s*=?\s*(?:"([^"]*)"|\047([^\047]*)\047|([^\s]*))\s*$' +) + + __all__ = ["get_transformed_code"] @@ -76,6 +102,12 @@ def _get_parser() -> ArgumentParser: action="store_true", help="whether pre-release packages should be considered", ) + parser.add_argument( + "--index-url", + "-i", + type=str, + help="the index URL to use for package lookup", + ) parser.add_argument( "packages", nargs="*", @@ -102,7 +134,6 @@ async def get_transformed_code(argv: list[str]) -> typing.Optional[str]: async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict]: """Get the arguments to `piplite` subcommands from CLI-like tokens.""" - parser = _get_parser() try: @@ -111,66 +142,133 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict return None, {} kwargs = {} - action = args.action if action == "install": - kwargs["requirements"] = args.packages + # CLI index URL, if provided, is the only one we'll use + cli_index_url = args.index_url + all_requirements = [] + last_seen_file_index = None + if args.packages: + all_requirements.extend((pkg, cli_index_url) for pkg in args.packages) + + # Process requirements files + for req_file in args.requirements or []: + context = RequirementsContext() + + if not Path(req_file).exists(): + warn(f"piplite could not find requirements file {req_file}") + continue + + # Process the file and capture any index URL it contains + for line_no, line in enumerate( + Path(req_file).read_text(encoding="utf-8").splitlines() + ): + await _packages_from_requirements_line( + Path(req_file), line_no + 1, line, context + ) + + # Keep track of the last index URL we saw in any requirements file + if context.index_url is not None: + last_seen_file_index = context.index_url + + # Add requirements - if CLI provided an index URL, use that instead + if cli_index_url: + all_requirements.extend( + (req, cli_index_url) for req, _ in context.requirements + ) + else: + all_requirements.extend(context.requirements) + + if all_requirements: + kwargs["requirements"] = [] + + # Add all requirements + kwargs["requirements"].extend(req for req, _ in all_requirements) + + # Use index URL with proper precedence: + # 1. CLI index URL if provided + # 2. Otherwise, last seen index URL from any requirements file + effective_index = cli_index_url or last_seen_file_index + if effective_index: + kwargs["index_urls"] = effective_index + + # Other CLI flags remain unchanged if args.pre: kwargs["pre"] = True - if args.no_deps: kwargs["deps"] = False - if args.verbose: kwargs["keep_going"] = True - for req_file in args.requirements or []: - kwargs["requirements"] += await _packages_from_requirements_file( - Path(req_file) - ) - return action, kwargs -async def _packages_from_requirements_file(req_path: Path) -> list[str]: - """Extract (potentially nested) package requirements from a requirements file.""" +async def _packages_from_requirements_file( + req_path: Path, +) -> Tuple[List[Tuple[str, Optional[List[str]]]], List[str]]: + """Extract package requirements and index URLs from a requirements file. + + This function processes a requirements file to collect both package requirements + and any index URLs specified in it (with support for nested requirements). + + Returns: + A tuple of: + - List of (requirement, index_urls) pairs, where index_urls is a list of URLs + that should be used for this requirement + - List of index URLs found in the file + """ + if not req_path.exists(): warn(f"piplite could not find requirements file {req_path}") - return [] + return [], [] - requirements = [] + context = RequirementsContext() - for line_no, line in enumerate(req_path.read_text(encoding="utf").splitlines()): - requirements += await _packages_from_requirements_line( - req_path, line_no + 1, line - ) + for line_no, line in enumerate(req_path.read_text(encoding="utf-8").splitlines()): + await _packages_from_requirements_line(req_path, line_no + 1, line, context) - return requirements + return context.requirements, context.index_urls async def _packages_from_requirements_line( - req_path: Path, line_no: int, line: str -) -> list[str]: + req_path: Path, line_no: int, line: str, context: RequirementsContext +) -> None: """Extract (potentially nested) package requirements from line of a requirements file. `micropip` has a sufficient pep508 implementation to handle most cases """ req = line.strip().split("#")[0].strip() - # is it another requirement file? + if not req: + return + + # Handle nested requirements file req_file_match = re.match(REQ_FILE_PREFIX, req) if req_file_match: - if req_file_match[2].startswith("/"): - sub_req = Path(req) + sub_path = req_file_match[2] + if sub_path.startswith("/"): + sub_req = Path(sub_path) else: - sub_req = req_path.parent / req_file_match[2] - return await _packages_from_requirements_file(sub_req) + sub_req = req_path.parent / sub_path + nested_context = RequirementsContext() + await _packages_from_requirements_file(sub_req, nested_context) + # Use the last index URL from nested file, if one was found + if nested_context.index_url: + context.index_url = nested_context.index_url + context.requirements.extend(nested_context.requirements) + return + + # Check for index URL - this becomes the new active index URL. + index_match = re.match(INDEX_URL_PREFIX, req) + if index_match: + url = next(group for group in index_match.groups()[1:] if group is not None) + context.index_url = url.strip() + return if req.startswith("-"): warn(f"{req_path}:{line_no}: unrecognized requirement: {req}") - req = None - if not req: - return [] - return [req] + return + + context.add_requirement(req) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index e19b7e24..fc68195b 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -27,12 +27,25 @@ #: a cache of available packages _PIPLITE_INDICES = {} -#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS -_PIPLITE_DISABLE_PYPI = False - #: a well-known file name respected by the rest of the build chain ALL_JSON = "/all.json" +#: default arguments for piplite.install +_PIPLITE_DEFAULT_INSTALL_ARGS = { + "requirements": None, + "keep_going": False, + "deps": True, + "credentials": None, + "pre": False, + "index_urls": None, + "verbose": False, +} + +# Internal flags that affect package lookup behavior +_PIPLITE_INTERNAL_FLAGS = { + "disable_pypi": False, # don't fall back to pypi.org if package not found in _PIPLITE_URLS +} + class PiplitePyPIDisabled(ValueError): """An error for when PyPI is disabled at the site level, but a download was @@ -94,7 +107,7 @@ async def _query_package( if pypi_json_from_index: return pypi_json_from_index - if _PIPLITE_DISABLE_PYPI: + if _PIPLITE_INTERNAL_FLAGS["disable_pypi"]: raise PiplitePyPIDisabled( f"{name} could not be installed: PyPI fallback is disabled" ) @@ -116,17 +129,46 @@ async def _install( *, verbose: bool | int = False, ): - """Invoke micropip.install with a patch to get data from local indexes""" - with patch("micropip.package_index.query_package", _query_package): - return await micropip.install( - requirements=requirements, - keep_going=keep_going, - deps=deps, - credentials=credentials, - pre=pre, - index_urls=index_urls, - verbose=verbose, - ) + """Invoke micropip.install with a patch to get data from local indexes. + + This function handles the installation of Python packages, respecting index URLs + from various sources (CLI, requirements files, or defaults) while maintaining + precedence order provided for indices. + + Arguments maintain the same semantics as micropip.install, but with additional + handling of index URLs and installation defaults. + """ + try: + install_args = _PIPLITE_DEFAULT_INSTALL_ARGS.copy() + + provided_args = { + "requirements": requirements, + "keep_going": keep_going, + "deps": deps, + "credentials": credentials, + "pre": pre, + "verbose": verbose, + } + + if index_urls is not None: + provided_args["index_urls"] = index_urls + + install_args.update({k: v for k, v in provided_args.items() if v is not None}) + + if verbose: + logger.info(f"Installing with arguments: {install_args}") + if install_args.get("index_urls"): + logger.info(f"Using index URL: {install_args['index_urls']}") + + with patch("micropip.package_index.query_package", _query_package): + return await micropip.install(**install_args) + + except Exception as e: + if install_args.get("index_urls"): + logger.error( + f"Failed to install using index URLs {install_args['index_urls']}: {e}" + ) + raise def install( diff --git a/packages/pyodide-kernel/src/kernel.ts b/packages/pyodide-kernel/src/kernel.ts index 16036257..fc30c320 100644 --- a/packages/pyodide-kernel/src/kernel.ts +++ b/packages/pyodide-kernel/src/kernel.ts @@ -381,6 +381,11 @@ export namespace PyodideKernel { */ mountDrive: boolean; + /** + * Default options to pass to piplite.install + */ + pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions; + /** * additional options to provide to `loadPyodide` * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide diff --git a/packages/pyodide-kernel/src/tokens.ts b/packages/pyodide-kernel/src/tokens.ts index eaaed75f..bd5dfd09 100644 --- a/packages/pyodide-kernel/src/tokens.ts +++ b/packages/pyodide-kernel/src/tokens.ts @@ -48,9 +48,24 @@ export interface IPyodideWorkerKernel extends IWorkerKernel { export type IRemotePyodideWorkerKernel = IPyodideWorkerKernel; /** - * An namespace for Pyodide workers. + * A namespace for Pyodide workers. */ export namespace IPyodideWorkerKernel { + /** + * Options for piplite installation. + */ + export interface IPipliteInstallOptions { + /** + * Base URLs of extra indices to use + */ + index_urls?: string[]; + + /** + * Any additional piplite install options + */ + [key: string]: any; + } + /** * Initialization options for a worker. */ @@ -90,6 +105,11 @@ export namespace IPyodideWorkerKernel { */ mountDrive: boolean; + /** + * Default options to pass to piplite.install + */ + pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions; + /** * additional options to provide to `loadPyodide` * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index d5b91829..c337497e 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -65,8 +65,13 @@ export class PyodideRemoteKernel { throw new Error('Uninitialized'); } - const { pipliteWheelUrl, disablePyPIFallback, pipliteUrls, loadPyodideOptions } = - this._options; + const { + pipliteWheelUrl, + disablePyPIFallback, + pipliteUrls, + loadPyodideOptions, + pipliteInstallDefaultOptions, + } = this._options; const preloaded = (loadPyodideOptions || {}).packages || []; @@ -81,12 +86,31 @@ export class PyodideRemoteKernel { `); } + const pythonConfig = [ + 'import piplite.piplite', + 'piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS = {', + ' "keep_going": False,', + ' "deps": True,', + ' "credentials": None,', + ' "pre": False,', + ' "verbose": False,', + '}', + `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, + `piplite.piplite._PIPLITE_INTERNAL_FLAGS = {"disable_pypi": ${disablePyPIFallback ? 'True' : 'False'}}`, + ]; + + if (pipliteInstallDefaultOptions) { + for (const [key, value] of Object.entries(pipliteInstallDefaultOptions)) { + if (value !== undefined) { + pythonConfig.push( + `piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS["${key}"] = ${JSON.stringify(value)}`, + ); + } + } + } + // get piplite early enough to impact pyodide-kernel dependencies - await this._pyodide.runPythonAsync(` - import piplite.piplite - piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'} - piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)} - `); + await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } protected async initKernel(options: IPyodideWorkerKernel.IOptions): Promise {