ainativelang/Makefile at main · sbhooley/ainativelang · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
.PHONY: conformance benchmark benchmark-ci benchmark-deps check-apollo-promoter postgres-it postgres-it-deps mysql-it mysql-it-deps redis-it redis-it-deps dynamodb-it dynamodb-it-deps airtable-it airtable-it-deps supabase-it supabase-it-deps sync-venvs

# Refresh .venv-py310 and .venv-ainl with the same editable install (CI + OpenClaw parity).
sync-venvs:
	bash scripts/sync_dual_venvs.sh

# Apollo X promoter: strict IR + gateway integration tests (see scripts/check_apollo_promoter.sh).
check-apollo-promoter:
	bash scripts/check_apollo_promoter.sh

# Automated Conformance Test Suite (snapshot-driven).
# Usage:
#   make conformance
#   SNAPSHOT_UPDATE=1 make conformance
# Match CI Python 3.10 + deps (pytest-xdist, syrupy), e.g.:
#   make conformance PYTHON=./.venv-py310/bin/python
CONFORMANCE_DIR ?= tests/conformance

# Prefer .venv-py310 (CI), then .venv-ainl (OpenClaw), then .venv, then system python3.
# Shell-based (nested $(if $(wildcard …)) breaks some Make parsers on long lines).
PYTHON ?= $(shell if test -x "$(CURDIR)/.venv-py310/bin/python"; then echo "$(CURDIR)/.venv-py310/bin/python"; elif test -x "$(CURDIR)/.venv-ainl/bin/python"; then echo "$(CURDIR)/.venv-ainl/bin/python"; elif test -x "$(CURDIR)/.venv/bin/python"; then echo "$(CURDIR)/.venv/bin/python"; else echo python3; fi)
PYTEST ?= $(PYTHON) -m pytest
SNAPSHOT_UPDATE ?= 0
CONFORMANCE_LOG ?= tests/snapshots/conformance/last_run.log
CONFORMANCE_SUMMARY ?= tests/snapshots/conformance/summary.md
CONFORMANCE_BADGE ?= tests/snapshots/conformance/conformance_badge.svg

conformance:
	@echo "==> Running conformance suite in $(CONFORMANCE_DIR)"
	@mkdir -p tests/snapshots/conformance
	@if [ "$(SNAPSHOT_UPDATE)" = "1" ]; then \
	  $(PYTEST) -n auto $(CONFORMANCE_DIR) --snapshot-update | tee "$(CONFORMANCE_LOG)"; \
	else \
	  $(PYTEST) -n auto $(CONFORMANCE_DIR) | tee "$(CONFORMANCE_LOG)"; \
	fi
	@$(PYTHON) -c 'import re, pathlib; log=pathlib.Path("$(CONFORMANCE_LOG)").read_text(encoding="utf-8"); mp=re.search(r"([0-9]+) passed", log); ms=re.search(r"([0-9]+) skipped", log); mf=re.search(r"([0-9]+) failed", log); p=int(mp.group(1)) if mp else 0; s=int(ms.group(1)) if ms else 0; f=int(mf.group(1)) if mf else 0; status="passing" if f==0 else "failing"; color_hex="#4c1" if f==0 else "#e05d44"; summary=pathlib.Path("$(CONFORMANCE_SUMMARY)"); summary.write_text(f"# Conformance Summary\n\n- Passed: {p}\n- Skipped: {s}\n- Failed: {f}\n- Status: {status}\n", encoding="utf-8"); badge=pathlib.Path("$(CONFORMANCE_BADGE)"); svg=f"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"180\" height=\"20\" role=\"img\" aria-label=\"conformance: {status}\"><rect width=\"110\" height=\"20\" fill=\"#555\"/><rect x=\"110\" width=\"70\" height=\"20\" fill=\"{color_hex}\"/><text x=\"55\" y=\"14\" fill=\"#fff\" font-family=\"Verdana\" font-size=\"11\" text-anchor=\"middle\">conformance</text><text x=\"145\" y=\"14\" fill=\"#fff\" font-family=\"Verdana\" font-size=\"11\" text-anchor=\"middle\">{status}</text></svg>"; badge.write_text(svg, encoding="utf-8"); print(f"==> Wrote {summary} and {badge}")'

# Benchmarks need tiktoken + psutil (pyproject [benchmark] extra). Installed on demand below.
# ainl-lang baseline is Python 3.10+ (see pyproject.toml requires-python); fail fast with a clear message.
benchmark-deps: ## Install [benchmark] extra if tiktoken/psutil missing in $(PYTHON)
	@$(PYTHON) scripts/check_python_baseline.py
	@$(PYTHON) -c "import tiktoken, psutil" 2>/dev/null || \
	  (echo "==> Installing optional [benchmark] deps (tiktoken, psutil)..." && \
	   $(PYTHON) -m pip install -q -e ".[benchmark]")

benchmark: benchmark-deps ## Run all benchmarks locally (updates default JSON + markdown)
	@echo "==> PYTHON for benchmarks: $(PYTHON) (override with make benchmark PYTHON=...)"
	@echo "==> benchmark_size (compare baselines, gpt-4o cost)"
	@$(PYTHON) scripts/benchmark_size.py --compare-baselines --cost-model gpt-4o --mode wide
	@echo "==> benchmark_runtime (compare baselines, reliability 5, gpt-4o)"
	@$(PYTHON) scripts/benchmark_runtime.py --compare-baselines --reliability-runs 5 --cost-model gpt-4o

# CI-style: JSON only, smaller runtime sampling, headline profile + minimal_emit size slice
benchmark-ci: benchmark-deps ## Run benchmarks like CI (JSON only, tooling/*_ci.json)
	@echo "==> PYTHON for benchmark-ci: $(PYTHON) (override with make benchmark-ci PYTHON=...)"
	@echo "==> benchmark_size (CI slice, json only)"
	@$(PYTHON) scripts/benchmark_size.py --mode minimal_emit --profile-name canonical_strict_valid \
		--compare-baselines --cost-model gpt-4o \
		--json-out tooling/benchmark_size_ci.json --skip-markdown-inject
	@echo "==> benchmark_runtime (CI-friendly warmup/runs, json only)"
	@$(PYTHON) scripts/benchmark_runtime.py --profile-name canonical_strict_valid \
		--compare-baselines --reliability-runs 10 --cost-model gpt-4o \
		--warmup 1 --runs 2 \
		--json-out tooling/benchmark_runtime_ci.json --skip-markdown-inject

# Turnkey postgres integration test path (uses tests/fixtures/docker-compose.postgres.yml).
postgres-it-deps:
	@$(PYTHON) -c "import psycopg" 2>/dev/null || \
	  (echo "==> Installing optional [postgres] deps..." && $(PYTHON) -m pip install -q -e ".[postgres]")

postgres-it: postgres-it-deps
	@AINL_TEST_USE_DOCKER_POSTGRES=1 $(PYTHON) -m pytest -m integration tests/test_postgres_adapter_integration.py

# Turnkey mysql integration test path (uses tests/fixtures/docker-compose.mysql.yml).
mysql-it-deps:
	@$(PYTHON) -c "import pymysql" 2>/dev/null || \
	  (echo "==> Installing optional [mysql] deps..." && $(PYTHON) -m pip install -q -e ".[mysql]")

mysql-it: mysql-it-deps
	@AINL_TEST_USE_DOCKER_MYSQL=1 $(PYTHON) -m pytest -m integration tests/test_mysql_adapter_integration.py

# Turnkey redis integration test path (uses tests/fixtures/docker-compose.redis.yml).
redis-it-deps:
	@$(PYTHON) -c "import redis" 2>/dev/null || \
	  (echo "==> Installing optional [redis] deps..." && $(PYTHON) -m pip install -q -e ".[redis]")

redis-it: redis-it-deps
	@AINL_TEST_USE_DOCKER_REDIS=1 $(PYTHON) -m pytest -m integration tests/test_redis_adapter_integration.py

# Turnkey dynamodb integration test path (uses tests/fixtures/docker-compose.dynamodb.yml).
dynamodb-it-deps:
	@$(PYTHON) -c "import boto3" 2>/dev/null || \
	  (echo "==> Installing optional [dynamodb] deps..." && $(PYTHON) -m pip install -q -e ".[dynamodb]")

dynamodb-it: dynamodb-it-deps
	@AINL_TEST_USE_DOCKER_DYNAMODB=1 $(PYTHON) -m pytest -m integration tests/test_dynamodb_adapter_integration.py

# Airtable integration path (real API key/base; no docker fixture).
airtable-it-deps:
	@$(PYTHON) -c "import httpx" 2>/dev/null || \
	  (echo "==> Installing optional [airtable] deps..." && $(PYTHON) -m pip install -q -e ".[airtable]")

airtable-it: airtable-it-deps
	@AINL_TEST_USE_AIRTABLE=1 $(PYTHON) -m pytest -m integration tests/test_airtable_adapter_integration.py

# Supabase integration path (uses Postgres fixture + optional real Supabase URL/keys).
supabase-it-deps:
	@$(PYTHON) -c "import httpx" 2>/dev/null || \
	  (echo "==> Installing optional [supabase] deps..." && $(PYTHON) -m pip install -q -e ".[supabase]")

supabase-it: supabase-it-deps
	@AINL_TEST_USE_DOCKER_SUPABASE=1 $(PYTHON) -m pytest -m integration tests/test_supabase_adapter_integration.py