diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3540cfb9..f97b4fa2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,6 +72,21 @@ jobs: restore-keys: | docs-full-site-${{ github.repository }}- + # Tag-scoped caches are invisible on main; merge live Pages so releases survive. + - name: Merge versions from live GitHub Pages + if: github.event_name == 'push' + shell: bash + run: | + SITE_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}" + SKIP_VERSION="main" + if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then + SKIP_VERSION="${GITHUB_REF_NAME}" + fi + python3 ${GITHUB_WORKSPACE}/docs/scripts/merge_published_site.py \ + --build-dir ${GITHUB_WORKSPACE}/docs/build/html \ + --site-base-url "${SITE_URL}" \ + --skip-version "${SKIP_VERSION}" + - name: Build docs shell: bash run: | @@ -102,7 +117,7 @@ jobs: else echo "Building dev docs for main branch..." - # Only rebuild main/ — all other version dirs come from the cache + # Only rebuild main/ — other versions come from cache + live Pages merge rm -rf build/html/main sphinx-build source build/html/main cd build/html @@ -112,9 +127,9 @@ jobs: python3 ${GITHUB_WORKSPACE}/docs/scripts/generate_versions_json.py \ --build-dir . - # Save the updated full site so the next run can restore all versions + # Default-branch cache only (tag-scoped caches are not visible on main). - name: Save full multi-version docs site - if: github.event_name == 'push' + if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: path: docs/build/html @@ -145,17 +160,22 @@ jobs: --extra-index-url https://download.blender.org/pypi/ echo "Unit test Start" export HF_ENDPOINT=https://hf-mirror.com + pytest tests/docs -q --confcutdir=tests/docs pytest tests publish: if: github.event_name == 'push' needs: build runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} permissions: pages: write id-token: write steps: - name: Deploy GitHub Pages + id: deployment uses: actions/deploy-pages@v4 diff --git a/.github/workflows/tests/test_docs_publish.yml b/.github/workflows/tests/test_docs_publish.yml index c75015ed..2560048b 100644 --- a/.github/workflows/tests/test_docs_publish.yml +++ b/.github/workflows/tests/test_docs_publish.yml @@ -4,6 +4,13 @@ on: workflow_dispatch: jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run merge_published_site unit tests + run: pytest tests/docs -q --confcutdir=tests/docs + # ----------------------------------------------------------------------- # Scenario A: push to main — existing v0.1.0, v0.2.0 must survive # Simulates: cache holds v0.1.0 + v0.2.0, build adds/updates main/ @@ -49,6 +56,68 @@ jobs: " echo "PASS: main_push — existing versions preserved" + # ----------------------------------------------------------------------- + # Scenario D: main push after tag — stale cache (main only) + live Pages + # This is the production bug: tag cache is not on main; merge fixes it. + # ----------------------------------------------------------------------- + test-main-after-tag-merge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Stale default-branch cache (main/ only) + run: | + mkdir -p docs/build/html/main + echo "stale main" > docs/build/html/main/index.html + + - name: Mock live GitHub Pages (has tag release v0.3.0) + run: | + PUBLISHED="${GITHUB_WORKSPACE}/mock-published-site" + mkdir -p "${PUBLISHED}/v0.3.0" "${PUBLISHED}/main" + echo "v0.3.0 live" > "${PUBLISHED}/v0.3.0/index.html" + echo "main live" > "${PUBLISHED}/main/index.html" + python3 -c " + import json, pathlib + root = pathlib.Path('${PUBLISHED}') + manifest = { + 'latest': 'v0.3.0', + 'versions': [ + {'name': 'v0.3.0', 'url': './v0.3.0/index.html', 'type': 'tag'}, + {'name': 'main', 'url': './main/index.html', 'type': 'branch'}, + ], + } + (root / 'versions.json').write_text(json.dumps(manifest, indent=2)) + " + + - name: Merge published (skip main — will rebuild) + run: | + python3 ${GITHUB_WORKSPACE}/docs/scripts/merge_published_site.py \ + --build-dir ${GITHUB_WORKSPACE}/docs/build/html \ + --published-root ${GITHUB_WORKSPACE}/mock-published-site \ + --skip-version main + + - name: Rebuild main/ only + run: | + rm -rf docs/build/html/main + mkdir -p docs/build/html/main + echo "main rebuilt" > docs/build/html/main/index.html + python3 ${GITHUB_WORKSPACE}/docs/scripts/generate_versions_json.py \ + --build-dir ${GITHUB_WORKSPACE}/docs/build/html + + - name: Assert — v0.3.0 preserved after main push + run: | + [ -d docs/build/html/v0.3.0 ] || (echo "FAIL: v0.3.0 missing after merge!" && exit 1) + grep -q "v0.3.0 live" docs/build/html/v0.3.0/index.html + grep -q "main rebuilt" docs/build/html/main/index.html + python3 -c " + import json + d = json.load(open('docs/build/html/versions.json')) + names = [v['name'] for v in d['versions']] + assert 'v0.3.0' in names and 'main' in names, names + assert d['latest'] == 'v0.3.0', d['latest'] + " + echo "PASS: main_after_tag — release dir restored from published mock" + # ----------------------------------------------------------------------- # Scenario B: tag push v0.3.0 — new version added, old dirs untouched # ----------------------------------------------------------------------- diff --git a/docs/scripts/merge_published_site.py b/docs/scripts/merge_published_site.py new file mode 100644 index 00000000..612f49a6 --- /dev/null +++ b/docs/scripts/merge_published_site.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- +"""Merge version directories from the live docs site into a local build tree. + +CI restores an Actions cache and rebuilds only one version (``main`` or a tag). +Tag-scoped cache entries are not visible on ``main`` pushes, so the cache alone +cannot hold all versions. This script fills *missing* version directories from +the currently published GitHub Pages site (or a local directory in tests). +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import urlopen + +__all__ = ["load_versions_manifest", "merge_published_site"] + + +def load_versions_manifest( + *, + site_base_url: str | None = None, + published_root: Path | None = None, +) -> dict[str, Any] | None: + """Load ``versions.json`` from a local tree or the live site URL.""" + if published_root is not None: + manifest_path = published_root / "versions.json" + if not manifest_path.is_file(): + return None + return json.loads(manifest_path.read_text(encoding="utf-8")) + + if not site_base_url: + return None + + manifest_url = f"{site_base_url.rstrip('/')}/versions.json" + try: + with urlopen(manifest_url, timeout=30) as response: + if response.status != 200: + return None + return json.loads(response.read().decode("utf-8")) + except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as exc: + print(f"No published manifest at {manifest_url}: {exc}", file=sys.stderr) + return None + + +def _copy_local_version(src: Path, dest: Path) -> None: + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(src, dest) + + +def _download_version_wget(site_base_url: str, version: str, dest: Path) -> None: + """Download one version subtree with wget (available in CI containers).""" + url = f"{site_base_url.rstrip('/')}/{version}/" + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + shutil.rmtree(dest) + + # -nH: no host-based dirs; -np: stay under version URL; -P: output prefix + result = subprocess.run( + [ + "wget", + "-q", + "-r", + "-l", + "50", + "-np", + "-nH", + "-P", + str(dest.parent), + url, + ], + check=False, + ) + if result.returncode != 0: + print(f"wget failed for {url} (exit {result.returncode})", file=sys.stderr) + return + + # wget may create dest.parent// or nest extra path segments — normalize + if not dest.is_dir(): + candidates = list(dest.parent.glob(f"*/{version}")) + if len(candidates) == 1 and candidates[0].is_dir(): + candidates[0].rename(dest) + else: + nested = dest.parent / version + if nested.is_dir() and nested != dest: + nested.rename(dest) + + +def merge_published_site( + build_dir: Path, + *, + site_base_url: str | None = None, + published_root: Path | None = None, + skip_versions: frozenset[str] | None = None, +) -> list[str]: + """Copy missing version dirs from published site into ``build_dir``. + + Args: + build_dir: Sphinx output root (``docs/build/html``). + site_base_url: Live Pages base, e.g. ``https://org.github.io/Repo``. + published_root: Local published tree for tests (``versions.json`` + dirs). + skip_versions: Version names to leave for a fresh build (e.g. ``main``). + + Returns: + Names of versions merged from the published site. + """ + build_dir = build_dir.resolve() + build_dir.mkdir(parents=True, exist_ok=True) + skip = skip_versions or frozenset() + + manifest = load_versions_manifest( + site_base_url=site_base_url, + published_root=published_root, + ) + if not manifest: + print("No published versions manifest; skipping merge.") + return [] + + merged: list[str] = [] + for entry in manifest.get("versions", []): + name = entry.get("name") + if not name or name in skip: + continue + if (build_dir / name).is_dir(): + continue + + if published_root is not None: + src = published_root / name + if not src.is_dir(): + print( + f"Published root missing directory {name}; skip.", file=sys.stderr + ) + continue + print(f"Merging local published version: {name}") + _copy_local_version(src, build_dir / name) + merged.append(name) + elif site_base_url: + print(f"Downloading published version: {name}") + _download_version_wget(site_base_url, name, build_dir / name) + if (build_dir / name).is_dir(): + merged.append(name) + else: + print( + "Neither published_root nor site_base_url set; cannot merge.", + file=sys.stderr, + ) + + return merged + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Merge missing doc version dirs from live GitHub Pages into build/html" + ) + parser.add_argument( + "--build-dir", + type=Path, + default=Path("build/html"), + help="Local docs build directory (default: build/html)", + ) + parser.add_argument( + "--site-base-url", + default=None, + help="Published site base URL, e.g. https://org.github.io/EmbodiChain", + ) + parser.add_argument( + "--published-root", + type=Path, + default=None, + help="Local directory mirroring published site (for tests)", + ) + parser.add_argument( + "--skip-version", + action="append", + default=[], + help="Version to skip (repeatable); rebuilt in the same CI run", + ) + args = parser.parse_args() + + merged = merge_published_site( + args.build_dir, + site_base_url=args.site_base_url, + published_root=args.published_root, + skip_versions=frozenset(args.skip_version), + ) + if merged: + print(f"Merged versions: {', '.join(merged)}") + else: + print("No versions merged from published site.") + + +if __name__ == "__main__": + main() diff --git a/docs/source/quick_start/docs.md b/docs/source/quick_start/docs.md index 1a8aef4d..12d8cb3d 100644 --- a/docs/source/quick_start/docs.md +++ b/docs/source/quick_start/docs.md @@ -53,4 +53,6 @@ python3 scripts/generate_versions_json.py --build-dir build/html This generates both `versions.json` (for the sidebar version selector) and `index.html` (redirects to the latest stable version, falling back to `main`). -> Old release versions beyond `DOCS_MAX_VERSIONS` (default: 4) are automatically pruned during CI builds. +> Old release versions beyond `DOCS_MAX_VERSIONS` (default: 5 in CI) are automatically pruned during CI builds. +> +> CI merges missing version directories from the live GitHub Pages site before each build so a `main` push cannot wipe docs built for release tags. See `docs/scripts/merge_published_site.py` and `tests/docs/test_merge_published_site.py`. diff --git a/tests/docs/__init__.py b/tests/docs/__init__.py new file mode 100644 index 00000000..dd650e90 --- /dev/null +++ b/tests/docs/__init__.py @@ -0,0 +1,15 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- diff --git a/tests/docs/conftest.py b/tests/docs/conftest.py new file mode 100644 index 00000000..d0a9f91f --- /dev/null +++ b/tests/docs/conftest.py @@ -0,0 +1,39 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = _REPO_ROOT / "docs" / "scripts" / "merge_published_site.py" + + +def _load_merge_module(): + spec = importlib.util.spec_from_file_location("merge_published_site", _SCRIPT) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load {_SCRIPT}") + module = importlib.util.module_from_spec(spec) + sys.modules["merge_published_site"] = module + spec.loader.exec_module(module) + return module + + +_merge = _load_merge_module() +load_versions_manifest = _merge.load_versions_manifest +merge_published_site = _merge.merge_published_site diff --git a/tests/docs/test_merge_published_site.py b/tests/docs/test_merge_published_site.py new file mode 100644 index 00000000..e80369fc --- /dev/null +++ b/tests/docs/test_merge_published_site.py @@ -0,0 +1,154 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Tests for multi-version docs merge (CI GitHub Pages).""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from .conftest import load_versions_manifest, merge_published_site + + +def _write_published_site(root: Path, versions: list[str], latest: str) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = { + "latest": latest, + "versions": [ + { + "name": v, + "url": f"./{v}/index.html", + "type": "tag" if v.startswith("v") else "branch", + } + for v in versions + ], + } + (root / "versions.json").write_text(json.dumps(manifest), encoding="utf-8") + for v in versions: + d = root / v + d.mkdir(parents=True, exist_ok=True) + (d / "index.html").write_text(f"{v} published", encoding="utf-8") + + +@pytest.fixture +def published_site(tmp_path: Path) -> Path: + published = tmp_path / "published" + _write_published_site(published, ["v0.1.0", "v0.2.0", "main"], latest="v0.2.0") + return published + + +@pytest.fixture +def build_dir(tmp_path: Path) -> Path: + build = tmp_path / "build" / "html" + build.mkdir(parents=True) + (build / "main").mkdir() + (build / "main" / "index.html").write_text( + "stale main from cache", encoding="utf-8" + ) + return build + + +def test_load_manifest_from_local(published_site: Path) -> None: + manifest = load_versions_manifest(published_root=published_site) + assert manifest is not None + assert manifest["latest"] == "v0.2.0" + assert len(manifest["versions"]) == 3 + + +def test_merge_fills_missing_tags_from_published( + build_dir: Path, published_site: Path +) -> None: + """Simulates main push: cache only has main/, live site has release tags.""" + merged = merge_published_site( + build_dir, + published_root=published_site, + skip_versions=frozenset({"main"}), + ) + assert merged == ["v0.1.0", "v0.2.0"] + assert ( + (build_dir / "v0.1.0" / "index.html") + .read_text(encoding="utf-8") + .startswith("v0.1.0") + ) + assert (build_dir / "v0.2.0").is_dir() + assert (build_dir / "main" / "index.html").read_text(encoding="utf-8") == ( + "stale main from cache" + ) + + +def test_merge_does_not_overwrite_existing_version( + build_dir: Path, published_site: Path +) -> None: + (build_dir / "v0.2.0").mkdir() + (build_dir / "v0.2.0" / "index.html").write_text( + "v0.2.0 local cache", encoding="utf-8" + ) + merged = merge_published_site( + build_dir, + published_root=published_site, + skip_versions=frozenset({"main"}), + ) + assert merged == ["v0.1.0"] + assert "local cache" in (build_dir / "v0.2.0" / "index.html").read_text( + encoding="utf-8" + ) + + +def test_merge_skip_version_for_fresh_tag_build( + build_dir: Path, published_site: Path +) -> None: + """Simulates tag push: do not pull the tag being built from published.""" + merged = merge_published_site( + build_dir, + published_root=published_site, + skip_versions=frozenset({"v0.3.0"}), + ) + assert "v0.3.0" not in merged + assert (build_dir / "v0.1.0").is_dir() + + +def test_main_push_after_tag_preserves_releases( + build_dir: Path, published_site: Path, tmp_path: Path +) -> None: + """End-to-end: stale cache + published site (post-tag) + rebuild main/.""" + _write_published_site( + published_site, + ["v0.1.0", "v0.2.0", "v0.3.0", "main"], + latest="v0.3.0", + ) + (published_site / "v0.3.0" / "index.html").write_text( + "v0.3.0 published", encoding="utf-8" + ) + + merge_published_site( + build_dir, + published_root=published_site, + skip_versions=frozenset({"main"}), + ) + + shutil.rmtree(build_dir / "main") + (build_dir / "main").mkdir() + (build_dir / "main" / "index.html").write_text( + "main rebuilt", encoding="utf-8" + ) + + for name in ("v0.1.0", "v0.2.0", "v0.3.0"): + assert (build_dir / name).is_dir(), f"missing {name} after main push simulation" + assert "rebuilt" in (build_dir / "main" / "index.html").read_text(encoding="utf-8")