Skip to content

Support passing of index URLs to piplite (CLI, runtime configuration, and requirements files) #169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 30 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
939dc42
Add an `--index-url` flag for the `piplite` CLI
agriyakhetarpal Feb 13, 2025
bed093c
Add `pipliteInstallDefaultOptions` to schema, types
agriyakhetarpal Feb 13, 2025
5d45a40
Fix a typo
agriyakhetarpal Feb 13, 2025
83757c0
Add effective index URLs to pass to piplite
agriyakhetarpal Feb 13, 2025
a9b539d
Add `--index-url`, `-i`, to CLI flags
agriyakhetarpal Feb 13, 2025
95b4700
Handle index URLs with requirements files
agriyakhetarpal Feb 13, 2025
32e0f51
Fix JS prettier lint errors
agriyakhetarpal Feb 13, 2025
8014816
Now fix Python linter errors
agriyakhetarpal Feb 13, 2025
48d1970
Fix typo
agriyakhetarpal Feb 13, 2025
7207454
Log what index URL is being used if verbose
agriyakhetarpal Feb 13, 2025
ffe3907
Fix, allow adding index URL inside a requirements file
agriyakhetarpal Feb 13, 2025
1c8e574
Mark CLI alias for index_urls in docstring
agriyakhetarpal Feb 14, 2025
e7d3818
Hopefully fix Python linter
agriyakhetarpal Feb 14, 2025
a5e9565
Handle tuple unpacking better
agriyakhetarpal Feb 14, 2025
c04d84f
Try to fix index URLs in requirements file
agriyakhetarpal Feb 14, 2025
23852ca
Rename `indexUrls` to `index_urls`
agriyakhetarpal Feb 14, 2025
b3f7808
Single source of truth for installation defaults
agriyakhetarpal Feb 14, 2025
d0fd31a
Fix Python formatting
agriyakhetarpal Feb 14, 2025
a9a62b3
Revert "Fix Python formatting"
agriyakhetarpal Feb 14, 2025
2d7aed6
Revert "Single source of truth for installation defaults"
agriyakhetarpal Feb 14, 2025
a1bcf66
Reapply "Single source of truth for installation defaults"
agriyakhetarpal Feb 14, 2025
a8cf844
Reapply "Fix Python formatting"
agriyakhetarpal Feb 14, 2025
d2192fe
Fix boolean capitalisation b/w JS/TS and Python
agriyakhetarpal Feb 14, 2025
4a7116c
Add a TS fix
agriyakhetarpal Feb 14, 2025
443c206
Fix index URLs and requirements files again
agriyakhetarpal Feb 14, 2025
e4e7a30
Some more fixes for install order precedence
agriyakhetarpal Feb 14, 2025
fec4e1b
More fixes
agriyakhetarpal Feb 14, 2025
98576dc
Simplify handling yet again
agriyakhetarpal Feb 15, 2025
d694c00
Fix URL handling that can lead to silent failures
agriyakhetarpal Feb 15, 2025
3fc2381
Temporarily remove NumPy, add SPNW index URL
agriyakhetarpal Feb 15, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion examples/jupyter-lite.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
"appName": "JupyterLite Pyodide Kernel",
"litePluginSettings": {
"@jupyterlite/pyodide-kernel-extension:kernel": {
"pipliteInstallDefaultOptions": {
"index_urls": "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
},
"loadPyodideOptions": {
"packages": ["matplotlib", "micropip", "numpy", "sqlite3", "ssl"],
"packages": ["micropip", "sqlite3", "ssl"],
"lockFileURL": "https://cdn.jsdelivr.net/pyodide/v0.27.2/full/pyodide-lock.json?from-lite-config=1"
}
}
Expand Down
15 changes: 15 additions & 0 deletions packages/pyodide-kernel-extension/schema/kernel.v0.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,21 @@
"default": [],
"format": "uri"
},
"pipliteInstallDefaultOptions": {
"type": "object",
"description": "Default options to pass to piplite.install",
"default": {},
"properties": {
"index_urls": {
"type": "array",
"items": {
"type": "string",
"format": "uri"
},
"description": "Base URLs of extra indices to use"
}
}
},
"loadPyodideOptions": {
"type": "object",
"description": "additional options to provide to `loadPyodide`, see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide",
Expand Down
8 changes: 8 additions & 0 deletions packages/pyodide-kernel-extension/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ const kernel: JupyterLiteServerPlugin<void> = {
const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href);
const disablePyPIFallback = !!config.disablePyPIFallback;
const loadPyodideOptions = config.loadPyodideOptions || {};
const pipliteInstallDefaultOptions = config.pipliteInstallDefaultOptions || {};

// Parse any configured index URLs
const index_urls = pipliteInstallDefaultOptions.index_urls || [];

for (const [key, value] of Object.entries(loadPyodideOptions)) {
if (key.endsWith('URL') && typeof value === 'string') {
Expand Down Expand Up @@ -99,6 +103,10 @@ const kernel: JupyterLiteServerPlugin<void> = {
mountDrive,
loadPyodideOptions,
contentsManager,
pipliteInstallDefaultOptions: {
index_urls,
...pipliteInstallDefaultOptions,
},
});
},
});
Expand Down
160 changes: 129 additions & 31 deletions packages/pyodide-kernel/py/piplite/piplite/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ async def install(
deps: bool = True, # --no-deps
credentials: str | None = None, # no CLI alias
pre: bool = False, # --pre
index_urls: list[str] | str | None = None, # no CLI alias
index_urls: list[str] | str | None = None, # -i and --index-url
*,
verbose: bool | int | None = None,
):
Expand All @@ -26,11 +26,37 @@ async def install(
import re
import sys
import typing
from typing import Optional, List, Tuple
from dataclasses import dataclass

from argparse import ArgumentParser
from pathlib import Path


@dataclass
class RequirementsContext:
"""Track state while parsing requirements files."""

index_url: Optional[str] = None
requirements: List[str] = None

def __post_init__(self):
if self.requirements is None:
self.requirements = []

def add_requirement(self, req: str):
"""Add a requirement with the currently active index URL."""
self.requirements.append((req, self.index_url))


REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*"

# Matches a pip-style index URL, with support for quote enclosures
INDEX_URL_PREFIX = (
r'^(--index-url|-i)\s*=?\s*(?:"([^"]*)"|\047([^\047]*)\047|([^\s]*))\s*$'
)


__all__ = ["get_transformed_code"]


Expand Down Expand Up @@ -76,6 +102,12 @@ def _get_parser() -> ArgumentParser:
action="store_true",
help="whether pre-release packages should be considered",
)
parser.add_argument(
"--index-url",
"-i",
type=str,
help="the index URL to use for package lookup",
)
parser.add_argument(
"packages",
nargs="*",
Expand All @@ -102,7 +134,6 @@ async def get_transformed_code(argv: list[str]) -> typing.Optional[str]:

async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict]:
"""Get the arguments to `piplite` subcommands from CLI-like tokens."""

parser = _get_parser()

try:
Expand All @@ -111,66 +142,133 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict
return None, {}

kwargs = {}

action = args.action

if action == "install":
kwargs["requirements"] = args.packages
# CLI index URL, if provided, is the only one we'll use
cli_index_url = args.index_url
all_requirements = []
last_seen_file_index = None

if args.packages:
all_requirements.extend((pkg, cli_index_url) for pkg in args.packages)

# Process requirements files
for req_file in args.requirements or []:
context = RequirementsContext()

if not Path(req_file).exists():
warn(f"piplite could not find requirements file {req_file}")
continue

# Process the file and capture any index URL it contains
for line_no, line in enumerate(
Path(req_file).read_text(encoding="utf-8").splitlines()
):
await _packages_from_requirements_line(
Path(req_file), line_no + 1, line, context
)

# Keep track of the last index URL we saw in any requirements file
if context.index_url is not None:
last_seen_file_index = context.index_url

# Add requirements - if CLI provided an index URL, use that instead
if cli_index_url:
all_requirements.extend(
(req, cli_index_url) for req, _ in context.requirements
)
else:
all_requirements.extend(context.requirements)

if all_requirements:
kwargs["requirements"] = []

# Add all requirements
kwargs["requirements"].extend(req for req, _ in all_requirements)

# Use index URL with proper precedence:
# 1. CLI index URL if provided
# 2. Otherwise, last seen index URL from any requirements file
effective_index = cli_index_url or last_seen_file_index
if effective_index:
kwargs["index_urls"] = effective_index

# Other CLI flags remain unchanged
if args.pre:
kwargs["pre"] = True

if args.no_deps:
kwargs["deps"] = False

if args.verbose:
kwargs["keep_going"] = True

for req_file in args.requirements or []:
kwargs["requirements"] += await _packages_from_requirements_file(
Path(req_file)
)

return action, kwargs


async def _packages_from_requirements_file(req_path: Path) -> list[str]:
"""Extract (potentially nested) package requirements from a requirements file."""
async def _packages_from_requirements_file(
req_path: Path,
) -> Tuple[List[Tuple[str, Optional[List[str]]]], List[str]]:
"""Extract package requirements and index URLs from a requirements file.

This function processes a requirements file to collect both package requirements
and any index URLs specified in it (with support for nested requirements).

Returns:
A tuple of:
- List of (requirement, index_urls) pairs, where index_urls is a list of URLs
that should be used for this requirement
- List of index URLs found in the file
"""

if not req_path.exists():
warn(f"piplite could not find requirements file {req_path}")
return []
return [], []

requirements = []
context = RequirementsContext()

for line_no, line in enumerate(req_path.read_text(encoding="utf").splitlines()):
requirements += await _packages_from_requirements_line(
req_path, line_no + 1, line
)
for line_no, line in enumerate(req_path.read_text(encoding="utf-8").splitlines()):
await _packages_from_requirements_line(req_path, line_no + 1, line, context)

return requirements
return context.requirements, context.index_urls


async def _packages_from_requirements_line(
req_path: Path, line_no: int, line: str
) -> list[str]:
req_path: Path, line_no: int, line: str, context: RequirementsContext
) -> None:
"""Extract (potentially nested) package requirements from line of a
requirements file.

`micropip` has a sufficient pep508 implementation to handle most cases
"""
req = line.strip().split("#")[0].strip()
# is it another requirement file?
if not req:
return

# Handle nested requirements file
req_file_match = re.match(REQ_FILE_PREFIX, req)
if req_file_match:
if req_file_match[2].startswith("/"):
sub_req = Path(req)
sub_path = req_file_match[2]
if sub_path.startswith("/"):
sub_req = Path(sub_path)
else:
sub_req = req_path.parent / req_file_match[2]
return await _packages_from_requirements_file(sub_req)
sub_req = req_path.parent / sub_path
nested_context = RequirementsContext()
await _packages_from_requirements_file(sub_req, nested_context)
# Use the last index URL from nested file, if one was found
if nested_context.index_url:
context.index_url = nested_context.index_url
context.requirements.extend(nested_context.requirements)
return

# Check for index URL - this becomes the new active index URL.
index_match = re.match(INDEX_URL_PREFIX, req)
if index_match:
url = next(group for group in index_match.groups()[1:] if group is not None)
context.index_url = url.strip()
return

if req.startswith("-"):
warn(f"{req_path}:{line_no}: unrecognized requirement: {req}")
req = None
if not req:
return []
return [req]
return

context.add_requirement(req)
72 changes: 57 additions & 15 deletions packages/pyodide-kernel/py/piplite/piplite/piplite.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,25 @@
#: a cache of available packages
_PIPLITE_INDICES = {}

#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS
_PIPLITE_DISABLE_PYPI = False

#: a well-known file name respected by the rest of the build chain
ALL_JSON = "/all.json"

#: default arguments for piplite.install
_PIPLITE_DEFAULT_INSTALL_ARGS = {
"requirements": None,
"keep_going": False,
"deps": True,
"credentials": None,
"pre": False,
"index_urls": None,
"verbose": False,
}

# Internal flags that affect package lookup behavior
_PIPLITE_INTERNAL_FLAGS = {
"disable_pypi": False, # don't fall back to pypi.org if package not found in _PIPLITE_URLS
}


class PiplitePyPIDisabled(ValueError):
"""An error for when PyPI is disabled at the site level, but a download was
Expand Down Expand Up @@ -94,7 +107,7 @@ async def _query_package(
if pypi_json_from_index:
return pypi_json_from_index

if _PIPLITE_DISABLE_PYPI:
if _PIPLITE_INTERNAL_FLAGS["disable_pypi"]:
raise PiplitePyPIDisabled(
f"{name} could not be installed: PyPI fallback is disabled"
)
Expand All @@ -116,17 +129,46 @@ async def _install(
*,
verbose: bool | int = False,
):
"""Invoke micropip.install with a patch to get data from local indexes"""
with patch("micropip.package_index.query_package", _query_package):
return await micropip.install(
requirements=requirements,
keep_going=keep_going,
deps=deps,
credentials=credentials,
pre=pre,
index_urls=index_urls,
verbose=verbose,
)
"""Invoke micropip.install with a patch to get data from local indexes.

This function handles the installation of Python packages, respecting index URLs
from various sources (CLI, requirements files, or defaults) while maintaining
precedence order provided for indices.

Arguments maintain the same semantics as micropip.install, but with additional
handling of index URLs and installation defaults.
"""
try:
install_args = _PIPLITE_DEFAULT_INSTALL_ARGS.copy()

provided_args = {
"requirements": requirements,
"keep_going": keep_going,
"deps": deps,
"credentials": credentials,
"pre": pre,
"verbose": verbose,
}

if index_urls is not None:
provided_args["index_urls"] = index_urls

install_args.update({k: v for k, v in provided_args.items() if v is not None})

if verbose:
logger.info(f"Installing with arguments: {install_args}")
if install_args.get("index_urls"):
logger.info(f"Using index URL: {install_args['index_urls']}")

with patch("micropip.package_index.query_package", _query_package):
return await micropip.install(**install_args)

except Exception as e:
if install_args.get("index_urls"):
logger.error(
f"Failed to install using index URLs {install_args['index_urls']}: {e}"
)
raise


def install(
Expand Down
5 changes: 5 additions & 0 deletions packages/pyodide-kernel/src/kernel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,11 @@ export namespace PyodideKernel {
*/
mountDrive: boolean;

/**
* Default options to pass to piplite.install
*/
pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions;

/**
* additional options to provide to `loadPyodide`
* @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide
Expand Down
Loading
Loading