Merge latest main

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

.github/workflows/pypi-release.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -37,7 +37,7 @@ jobs:
         run: python -m build --sdist --wheel --outdir dist/
 
       - name: Upload built archives
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pypi_archives
           path: dist/*
@@ -51,7 +51,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Download built archives
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pypi_archives
           path: dist

CHANGELOG.rst

@@ -1,7 +1,7 @@
 Changelog
 =========
 
-v0.13.0
+v0.13.2
 -----------
 
 - Speed up downloads with asyncio
@@ -33,6 +33,15 @@ v0.13.0
 - Merge latest skeleton and adopt ruff for code formatting.
 
 
+v0.13.1
+-----------
+
+- Fix ResolutionImpossible for lief==0.15.1 #202
+- Add license expression data from pypi API #208
+- Add python 3.13 in python-inspector #196
+- Update homepage_url and fix CI and tests
+
+
 v0.12.1
 -----------
 

azure-pipelines.yml

@@ -9,46 +9,46 @@ jobs:
     parameters:
       job_name: ubuntu20_cpython
       image_name: ubuntu-20.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: ubuntu22_cpython
       image_name: ubuntu-22.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-posix.yml
     parameters:
-      job_name: macos12_cpython
-      image_name: macOS-12
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      job_name: macos14_cpython
+      image_name: macOS-14
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: macos13_cpython
       image_name: macos-13
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs --reruns 2
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2019_cpython
       image_name: windows-2019
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs  --reruns 2
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2022_cpython
       image_name: windows-2022
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      python_versions: ["3.9", "3.10", "3.11", "3.12"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs --reruns 2

requirements-dev.txt

@@ -38,4 +38,5 @@ tomli==1.2.3
 tqdm==4.64.0
 twine==3.8.0
 typed-ast==1.5.4
-webencodings==0.5.1
+webencodings==0.5.1
+pytest-asyncio==0.21.1

requirements.txt

@@ -10,7 +10,7 @@ idna==3.3
 importlib-metadata==4.12.0
 intbitset==3.1.0
 packageurl-python==0.10.0
-packaging==21.3
+packaging==24.2
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
@@ -26,3 +26,5 @@ text-unidecode==1.3
 toml==0.10.2
 urllib3==1.26.11
 zipp==3.8.1
+aiohttp==3.11.14
+aiofiles==23.2.1

setup.cfg

@@ -69,6 +69,8 @@ install_requires =
     toml >= 0.10.0
     mock >= 3.0.5
     packvers >= 21.5
+    aiohttp >= 3.8
+    aiofiles >= 23.1
     pydantic >= 2.10.0
     pydantic_settings >= 2.8.0
 
@@ -89,6 +91,7 @@ testing =
     black
     isort
     pytest-rerunfailures
+    pytest-asyncio >= 0.21
 
 docs =
     Sphinx>=5.0.2

src/python_inspector/api.py

@@ -8,13 +8,14 @@
 # See https://aboutcode-orgnexB/python-inspector for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
+import asyncio
 import os
 from netrc import netrc
 from typing import Dict
 from typing import List
 from typing import NamedTuple
 from typing import Sequence
+from typing import Tuple
 
 from packageurl import PackageURL
 from packvers.requirements import Requirement
@@ -26,6 +27,7 @@
 from _packagedcode.pypi import PipRequirementsFileHandler
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import can_process_dependent_package
+from _packagedcode.pypi import get_resolved_purl
 from python_inspector import dependencies
 from python_inspector import pyinspector_settings as settings
 from python_inspector import utils
@@ -39,6 +41,7 @@
 from python_inspector.resolution import get_python_version_from_env_tag
 from python_inspector.resolution import get_reqs_insecurely
 from python_inspector.resolution import get_requirements_from_python_manifest
+from python_inspector.utils import Candidate
 from python_inspector.utils_pypi import PLATFORMS_BY_OS
 from python_inspector.utils_pypi import Environment
 from python_inspector.utils_pypi import PypiSimpleRepository
@@ -54,7 +57,7 @@ class Resolution(NamedTuple):
     ``files`` is a parsed list of input file data.
     """
 
-    resolution: Dict
+    resolution: List[Dict]
     packages: List[PackageData]
     files: List[Dict]
 
@@ -286,21 +289,27 @@ def resolve_dependencies(
         pdt_output=pdt_output,
         analyze_setup_py_insecurely=analyze_setup_py_insecurely,
         ignore_errors=ignore_errors,
+        verbose=verbose,
+        printer=printer,
     )
 
-    packages = []
+    async def gather_pypi_data():
+        async def get_pypi_data(package):
+            data = await get_pypi_data_from_purl(
+                package, repos=repos, environment=environment, prefer_source=prefer_source
+            )
 
-    for package in purls:
-        packages.extend(
-            [
-                pkg.to_dict()
-                for pkg in list(
-                    get_pypi_data_from_purl(
-                        package, repos=repos, environment=environment, prefer_source=prefer_source
-                    )
-                )
-            ],
-        )
+            if verbose:
+                printer(f"  retrieved package '{package}'")
+
+            return data
+
+        if verbose:
+            printer(f"retrieve package data from pypi:")
+
+        return await asyncio.gather(*[get_pypi_data(package) for package in purls])
+
+    packages = [pkg.to_dict() for pkg in asyncio.run(gather_pypi_data()) if pkg is not None]
 
     if verbose:
         printer("done!")
@@ -316,14 +325,16 @@ def resolve_dependencies(
 
 
 def resolve(
-    direct_dependencies,
-    environment,
-    repos=tuple(),
-    as_tree=False,
-    max_rounds=200000,
-    pdt_output=False,
-    analyze_setup_py_insecurely=False,
-    ignore_errors=False,
+    direct_dependencies: List[DependentPackage],
+    environment: Environment,
+    repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(),
+    as_tree: bool = False,
+    max_rounds: int = 200000,
+    pdt_output: bool = False,
+    analyze_setup_py_insecurely: bool = False,
+    ignore_errors: bool = False,
+    verbose: bool = False,
+    printer=print,
 ):
     """
     Resolve dependencies given a ``direct_dependencies`` list of
@@ -350,6 +361,8 @@ def resolve(
         pdt_output=pdt_output,
         analyze_setup_py_insecurely=analyze_setup_py_insecurely,
         ignore_errors=ignore_errors,
+        verbose=verbose,
+        printer=printer,
     )
 
     return resolved_dependencies, packages
@@ -364,32 +377,77 @@ def get_resolved_dependencies(
     pdt_output: bool = False,
     analyze_setup_py_insecurely: bool = False,
     ignore_errors: bool = False,
-):
+    verbose: bool = False,
+    printer=print,
+) -> Tuple[List[Dict], List[str]]:
     """
     Return resolved dependencies of a ``requirements`` list of Requirement for
-    an ``enviroment`` Environment. The resolved dependencies are formatted as
+    an ``environment`` Environment. The resolved dependencies are formatted as
     parent/children or a nested tree if ``as_tree`` is True.
 
     Used the provided ``repos`` list of PypiSimpleRepository.
-    If empty, use instead the PyPI.org JSON API exclusively instead
+    If empty, use instead the PyPI.org JSON API exclusively instead.
     """
+    provider = PythonInputProvider(
+        environment=environment,
+        repos=repos,
+        analyze_setup_py_insecurely=analyze_setup_py_insecurely,
+        ignore_errors=ignore_errors,
+    )
+
+    # gather version data for all requirements concurrently in advance.
+
+    async def gather_version_data():
+        async def get_version_data(name: str):
+            versions = await provider.fill_versions_for_package(name)
+
+            if verbose:
+                printer(f"  retrieved versions for package '{name}'")
+
+            return versions
+
+        if verbose:
+            printer(f"versions:")
+
+        return await asyncio.gather(
+            *[get_version_data(requirement.name) for requirement in requirements]
+        )
+
+    asyncio.run(gather_version_data())
+
+    # gather dependencies for all pinned requirements concurrently in advance.
+
+    async def gather_dependencies():
+        async def get_dependencies(requirement: Requirement):
+            purl = PackageURL(type="pypi", name=requirement.name)
+            resolved_purl = get_resolved_purl(purl=purl, specifiers=requirement.specifier)
+
+            if resolved_purl:
+                purl = resolved_purl.purl
+                candidate = Candidate(requirement.name, purl.version, requirement.extras)
+                await provider.fill_requirements_for_package(purl, candidate)
+
+                if verbose:
+                    printer(f"  retrieved dependencies for requirement '{str(purl)}'")
+
+        if verbose:
+            printer(f"dependencies:")
+
+        return await asyncio.gather(
+            *[get_dependencies(requirement) for requirement in requirements]
+        )
+
+    asyncio.run(gather_dependencies())
+
     resolver = Resolver(
-        provider=PythonInputProvider(
-            environment=environment,
-            repos=repos,
-            analyze_setup_py_insecurely=analyze_setup_py_insecurely,
-            ignore_errors=ignore_errors,
-        ),
+        provider=provider,
         reporter=BaseReporter(),
     )
     resolver_results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)
     package_list = get_package_list(results=resolver_results)
     if pdt_output:
-        return (format_pdt_tree(resolver_results), package_list)
-    return (
-        format_resolution(resolver_results, as_tree=as_tree),
-        package_list,
-    )
+        return format_pdt_tree(resolver_results), package_list
+    return format_resolution(resolver_results, as_tree=as_tree), package_list
 
 
 def get_requirements_from_direct_dependencies(