pyproject.toml

[project]
name = "mlx-benchmarks"
version = "0.7.0"  # x-release-please-version
description = "Benchmark harness for MLX-quantized LLMs on Apple Silicon — envelope v1 + HF dataset publisher"
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.11"
authors = [{ name = "Jacob Evans" }]
keywords = ["mlx", "benchmarks", "lm-eval", "llm", "apple-silicon"]
classifiers = [
    "Development Status :: 4 - Beta",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: MacOS",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
dependencies = [
    "huggingface-hub>=1.0.0",
    "jsonschema[format]>=4.23.0",
    "psutil>=5.9",
    "pyarrow>=24.0.0",
    "lm-eval[api]==0.4.12",
    # tokenizers powers the lm-eval converter's tok/s computation. The library
    # is also a transitive dep of transformers (and therefore lm-eval), but
    # declaring it explicitly keeps mypy happy across Python minor versions and
    # documents the direct usage.
    "tokenizers>=0.20",
]

[project.optional-dependencies]
framework-eval = [
    "openai>=1.0.0",
    "qwen-agent>=0.0.14",
    "soundfile>=0.13.0",
    "smolagents>=1.0.0",
    "google-adk>=2.0.0",
]
viewer = [
    "gradio>=6.12.0",
    "plotly>=6.7.0",
    "pandas>=2.0",
]
# Note: vllm benchmarking targets an external vllm server. The vllm Python
# package itself is Linux/CUDA-only, isn't installable on the Apple-Silicon
# dev platform, and isn't needed by the converter (which only parses JSON
# output). To run vllm `benchmark_serving`, install vllm in your own env.

[project.scripts]
mlx-bench-publish = "mlx_benchmarks.cli:main"

[project.urls]
Homepage = "https://github.com/JacobPEvans/mlx-benchmarks"
Dataset  = "https://huggingface.co/datasets/JacobPEvans/mlx-benchmarks"
Issues   = "https://github.com/JacobPEvans/mlx-benchmarks/issues"

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["src/mlx_benchmarks"]

# Ship schema.json inside the installed package dir (NOT a shared-data path),
# so `mlx_benchmarks.envelope._find_schema_path()` resolves it via
# `importlib.resources.files("mlx_benchmarks") / "schema.json"` after pip install.
[tool.hatch.build.targets.wheel.force-include]
"schema.json" = "mlx_benchmarks/schema.json"

[tool.hatch.build.targets.sdist]
include = [
    "src/mlx_benchmarks/**",
    "schema.json",
    "examples/**",
    "tests/**",
    "scripts/**",
    "configs/**",
    "README.md",
    "LICENSE",
    "CHANGELOG.md",
    "pyproject.toml",
]

[dependency-groups]
dev = [
    "mypy>=1.11",
    "pre-commit>=3.8",
    "pytest>=8.3",
    "pytest-cov>=5.0",
    "ruff>=0.6.9",
    "types-jsonschema",
    "types-psutil",
]

[tool.ruff]
line-length = 110
target-version = "py311"
src = ["src", "tests", "scripts"]

[tool.ruff.lint]
select = [
    "E",    # pycodestyle errors
    "F",    # pyflakes
    "I",    # isort
    "B",    # flake8-bugbear
    "UP",   # pyupgrade
    "SIM",  # flake8-simplify
    "RUF",  # ruff-specific
    "PIE",  # flake8-pie
    "PTH",  # flake8-use-pathlib
    "C4",   # flake8-comprehensions
    "ARG",  # unused arguments
]
ignore = [
    "E501",  # line length — ruff format handles it; avoids noise on URLs/long strings
]

[tool.ruff.lint.per-file-ignores]
"tests/**" = ["ARG"]  # fixtures often accept unused args

[tool.ruff.format]
docstring-code-format = true

[tool.mypy]
python_version = "3.11"
strict = true
pretty = true
show_error_codes = true
warn_unused_ignores = true
files = ["src/mlx_benchmarks"]

[[tool.mypy.overrides]]
module = [
    "huggingface_hub",
    "huggingface_hub.*",
    "pyarrow",
    "pyarrow.parquet",
    "gradio",
    "plotly",
    "plotly.*",
]
ignore_missing_imports = true

[tool.pytest.ini_options]
addopts = "-ra -q --strict-markers --strict-config"
testpaths = ["tests"]
pythonpath = ["src"]
filterwarnings = [
    "error",
    "ignore::DeprecationWarning:huggingface_hub.*",
    "ignore::DeprecationWarning:jsonschema.*",
]

[tool.coverage.run]
branch = true
source = ["mlx_benchmarks"]