feat: add first part of rattler_pypi_interop crate

crates/rattler_pypi_interop/Cargo.toml

@@ -0,0 +1,77 @@
+[package]
+name = "rattler_pypi_interop"
+version = "0.1.0"
+categories.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+edition.workspace = true
+readme.workspace = true
+
+[features]
+default = ["native-tls"]
+native-tls = ['reqwest/native-tls']
+rustls-tls = ['reqwest/rustls-tls']
+
+[dependencies]
+bytes = { workspace = true }
+ciborium = "0.2.2"
+csv = "1.3.0"
+data-encoding = "2.5.0"
+dirs = { workspace = true }
+fslock = { workspace = true }
+dunce = { workspace = true }
+elsa = "1.10.0"
+fs4 = "0.8.2"
+futures = { workspace = true }
+html-escape = "0.2.13"
+http = { workspace = true }
+http-cache-semantics = { workspace = true, features = ["serde", "reqwest"] }
+include_dir = "0.7.3"
+indexmap = { workspace = true, features = ["serde"] }
+itertools = { workspace = true }
+miette = "7.2.0"
+mime = "0.3.17"
+once_cell = { workspace = true }
+parking_lot = { workspace = true }
+peg = "0.8.2"
+pep440_rs = { version = "0.7.3" }
+pep508_rs = { version = "0.9.2" }
+rattler_digest = { path="../rattler_digest", version = "1.0.5", features = ["serde"] }
+regex = { workspace = true }
+reqwest = { workspace = true, default-features = false, features = ["json", "stream", "blocking", "rustls-tls"] }
+reqwest-middleware = { workspace = true }
+serde = {workspace = true }
+serde_json = { workspace = true }
+serde_with = {workspace = true }
+tempfile = { workspace = true }
+thiserror = { workspace = true }
+tl = "0.7.8"
+tokio = { workspace = true, features = ["process", "rt-multi-thread"] }
+tokio-util = { workspace = true, features = ["compat"] }
+tracing = { workspace = true, features = ["attributes"] }
+url = { workspace = true, features = ["serde"] }
+zip = { workspace = true, default-features = true }
+pathdiff = { workspace = true }
+async_zip = { version = "0.0.16", features = ["tokio", "deflate"] }
+tar = { workspace = true }
+flate2 = "1.0.28"
+pyproject-toml = "0.13.0"
+configparser = "3.0.4"
+fs-err = "2.11.0"
+async_http_range_reader = "0.9.1"
+which = "6.0.1"
+
+[dev-dependencies]
+anyhow = "1.0.82"
+axum = "0.7.5"
+criterion = "0.5"
+insta = { version = "1.38.0", features = ["ron", "redactions"] }
+miette = { version = "7.2.0", features = ["fancy"] }
+once_cell = "1.19.0"
+rstest = "0.19.0"
+tokio = { version = "1.37.0", features = ["rt", "rt-multi-thread"] }
+tokio-macros = { version = "2.5.0"}
+tokio-test = "0.4.4"
+tower-http = { version = "0.5.2", features = ["add-extension"] }
+tracing-test = "0.2.4"

crates/rattler_pypi_interop/src/lib.rs

@@ -0,0 +1,28 @@
+#![deny(missing_docs, dead_code)]
+
+//! This crate provides a set of functions and data types for working with Python packages installed via
+//! `PyPI` compatible installers (e.g. `pip`, `uv`, `poetry`, etc.).
+//!
+//! Some of the things you can do with this crate include:
+//!
+//! - Querying a `PyPI` repository for package metadata.
+//! - Reading Python package metadata from locally installed packages.
+//!
+//! <div class="warning">
+//!
+//!   This crate ports over a subset of functionality originally available in the
+//!   [`rattler_installs_packages`] crate that is also part of the [`rip`] CLI tool.
+//!
+//! </div>
+//!
+//! [`rip`]: https://github.com/prefix-dev/rip/
+//! [`rattler_installs_packages`]: https://docs.rs/rattler_installs_packages/latest/rattler_installs_packages/
+
+// Private modules
+// mod utils;
+
+// Public modules
+// pub mod artifacts;
+// pub mod index;
+// pub mod python_env;
+pub mod types;

crates/rattler_pypi_interop/src/types/core_metadata.rs

@@ -0,0 +1,210 @@
+// Implementation comes from https://github.com/njsmith/posy/blob/main/src/vocab/core_metadata.rs
+// Licensed under MIT or Apache-2.0
+
+use super::extra::ParseExtraError;
+use crate::{
+    types::Extra, types::PackageName, types::ParsePackageNameError, types::RFC822ish,
+    types::Version, types::VersionSpecifiers,
+};
+use once_cell::sync::Lazy;
+use pep440_rs::{VersionParseError, VersionSpecifiersParseError};
+use pep508_rs::Requirement;
+use std::{collections::HashSet, str::FromStr};
+use thiserror::Error;
+
+/// Holds the parsed PKG-INFO file.
+pub struct PackageInfo {
+    /// The parsed PKG-INFO file.
+    pub parsed: RFC822ish,
+}
+
+impl PackageInfo {
+    /// Parse the PKG-INFO file from bytes.
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, WheelCoreMetaDataError> {
+        let s = String::from_utf8_lossy(bytes);
+        Ok(Self {
+            parsed: RFC822ish::from_str(&s)?,
+        })
+    }
+
+    /// Create a new `PackageInfo` from a parsed `RFC822ish`.
+    pub fn new(parsed: RFC822ish) -> Self {
+        Self { parsed }
+    }
+}
+
+#[derive(Debug, Clone, serde::Serialize)]
+
+/// The core metadata of a wheel.
+pub struct WheelCoreMetadata {
+    /// The name of the package
+    pub name: PackageName,
+    /// Version w.r.t to PEP440
+    pub version: Version,
+    /// Version of the metadata
+    pub metadata_version: MetadataVersion,
+    /// Requirements for this distribution
+    /// Matches the Requires-Dist field
+    pub requires_dist: Vec<Requirement>,
+    /// Python requirement
+    pub requires_python: Option<VersionSpecifiers>,
+    /// Extras provided by this distribution
+    pub extras: HashSet<Extra>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
+/// Wrapper around a PEP440 version
+/// specifically for the metadata version
+pub struct MetadataVersion(pub Version);
+
+impl MetadataVersion {
+    /// We consider that this implements PEP643
+    /// if the version is 2.3 or higher.
+    pub fn implements_pep643(&self) -> bool {
+        static VERSION_2_2: Lazy<MetadataVersion> = Lazy::new(|| {
+            MetadataVersion(Version::from_str("2.2").expect("cannot parse 2.2 version string"))
+        });
+
+        if self < &VERSION_2_2 {
+            return false;
+        }
+        true
+    }
+}
+
+#[derive(Debug, Error)]
+#[allow(missing_docs)]
+pub enum WheelCoreMetaDataError {
+    #[error(transparent)]
+    FailedToParseMetadata(#[from] <RFC822ish as FromStr>::Err),
+
+    #[error("missing key {0} in METADATA")]
+    MissingKey(String),
+
+    #[error("duplicate key {0} in METADATA")]
+    DuplicateKey(String),
+
+    #[error("invalid Metadata-Version: {0}")]
+    InvalidMetadataVersion(VersionParseError),
+
+    #[error("invalid Version: {0}")]
+    InvalidVersion(VersionParseError),
+
+    #[error("invalid Requires-Python: {0}")]
+    InvalidRequiresPython(#[source] VersionSpecifiersParseError),
+
+    #[error("unsupported METADATA version {0}")]
+    UnsupportedVersion(Version),
+
+    #[error(transparent)]
+    InvalidPackageName(#[from] ParsePackageNameError),
+
+    #[error("invalid extra identifier '{0}'")]
+    InvalidExtra(String, #[source] ParseExtraError),
+
+    #[error("{0}")]
+    FailedToParse(String),
+}
+
+impl TryFrom<&[u8]> for WheelCoreMetadata {
+    type Error = WheelCoreMetaDataError;
+
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        PackageInfo::from_bytes(value)?.try_into()
+    }
+}
+
+impl TryFrom<PackageInfo> for WheelCoreMetadata {
+    type Error = WheelCoreMetaDataError;
+
+    fn try_from(value: PackageInfo) -> Result<Self, Self::Error> {
+        let (name, version, metadata_version, mut parsed) = parse_common(value)?;
+
+        let mut requires_dist = Vec::new();
+        for req_str in parsed.take_all("Requires-Dist") {
+            match req_str.parse() {
+                Err(e) => {
+                    tracing::warn!("ignoring Requires-Dist: {req_str}, failed to parse: {e}");
+                }
+                Ok(req) => requires_dist.push(req),
+            }
+        }
+
+        let requires_python = parsed
+            .maybe_take("Requires-Python")
+            .map_err(|_err| WheelCoreMetaDataError::DuplicateKey(String::from("Requires-Python")))?
+            .as_deref()
+            .map(VersionSpecifiers::from_str)
+            .transpose()
+            .map_err(WheelCoreMetaDataError::InvalidRequiresPython)?;
+
+        let mut extras: HashSet<Extra> = HashSet::new();
+        for extra in parsed.take_all("Provides-Extra").drain(..) {
+            extras.insert(
+                extra
+                    .parse()
+                    .map_err(|e| WheelCoreMetaDataError::InvalidExtra(extra, e))?,
+            );
+        }
+
+        Ok(WheelCoreMetadata {
+            name,
+            version,
+            metadata_version,
+            requires_dist,
+            requires_python,
+            extras,
+        })
+    }
+}
+
+fn parse_common(
+    input: PackageInfo,
+) -> Result<(PackageName, Version, MetadataVersion, RFC822ish), WheelCoreMetaDataError> {
+    let mut parsed = input.parsed;
+
+    let next_major_metadata_version: Lazy<Version> = Lazy::new(|| Version::from_str("3").unwrap());
+
+    // Quoth https://packaging.python.org/specifications/core-metadata:
+    // "Automated tools consuming metadata SHOULD warn if metadata_version
+    // is greater than the highest version they support, and MUST fail if
+    // metadata_version has a greater major version than the highest
+    // version they support (as described in PEP 440, the major version is
+    // the value before the first dot)."
+    //
+    // We do the MUST, but I think I disagree about warning on
+    // unrecognized minor revisions. If it's a minor revision, then by
+    // definition old software is supposed to be able to handle it "well
+    // enough". The only purpose of the warning would be to alert users
+    // that they might want to upgrade, or to alert the tool authors that
+    // there's a new metadata release. But for users, there are better
+    // ways to nudge them to upgrade (e.g. checking on startup, like
+    // pip does), and new metadata releases are so rare and so
+    // much-discussed beforehand that if a tool's authors don't know
+    // about it it's because the tool is abandoned anyway.
+    let metadata_version = parsed
+        .take("Metadata-Version")
+        .map_err(|_err| WheelCoreMetaDataError::MissingKey(String::from("Metadata-Version")))?;
+    let metadata_version: Version = metadata_version
+        .parse()
+        .map_err(WheelCoreMetaDataError::InvalidMetadataVersion)?;
+    if metadata_version >= *next_major_metadata_version {
+        return Err(WheelCoreMetaDataError::UnsupportedVersion(metadata_version));
+    }
+
+    let version_str = parsed
+        .take("Version")
+        .map_err(|_err| WheelCoreMetaDataError::MissingKey(String::from("Version")))?;
+
+    Ok((
+        parsed
+            .take("Name")
+            .map_err(|_err| WheelCoreMetaDataError::MissingKey(String::from("Name")))?
+            .parse()?,
+        version_str
+            .parse()
+            .map_err(WheelCoreMetaDataError::InvalidVersion)?,
+        MetadataVersion(metadata_version),
+        parsed,
+    ))
+}