change(ci): Overhaul CI test flow management

Author: lucasssvaz

.github/CODEOWNERS

@@ -72,9 +72,9 @@
 /libraries/Wire/ @me-no-dev
 /libraries/Zigbee/ @P-R-O-C-H-Y
 
-# CI JSON
+# CI YAML
 # Keep this after other libraries and tests to avoid being overridden.
-**/ci.json @lucasssvaz
+**/ci.yml @lucasssvaz
 
 # The CODEOWNERS file should be owned by the developers of the ESP32 Arduino Core.
 # Leave this entry as the last one to avoid being overridden.

.github/scripts/generate_missing_junits.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from xml.etree.ElementTree import Element, SubElement, ElementTree
+import yaml
+
+
+def parse_array(value) -> list[str]:
+    if isinstance(value, list):
+        return [str(x) for x in value]
+    if not isinstance(value, str):
+        return []
+    txt = value.strip()
+    if not txt:
+        return []
+    # Try JSON
+    try:
+        return [str(x) for x in json.loads(txt)]
+    except Exception:
+        pass
+    # Normalize single quotes then JSON
+    try:
+        fixed = txt.replace("'", '"')
+        return [str(x) for x in json.loads(fixed)]
+    except Exception:
+        pass
+    # Fallback: CSV
+    return [p.strip() for p in txt.strip("[]").split(",") if p.strip()]
+
+
+def _parse_ci_yml(content: str) -> dict:
+    if not content:
+        return {}
+    try:
+        data = yaml.safe_load(content) or {}
+        if not isinstance(data, dict):
+            return {}
+        return data
+    except Exception:
+        return {}
+
+
+def _fqbn_counts_from_yaml(ci: dict) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    if not isinstance(ci, dict):
+        return counts
+    fqbn = ci.get("fqbn")
+    if not isinstance(fqbn, dict):
+        return counts
+    for target, entries in fqbn.items():
+        if isinstance(entries, list):
+            counts[str(target)] = len(entries)
+        elif entries is not None:
+            # Single value provided as string
+            counts[str(target)] = 1
+    return counts
+
+
+def _sdkconfig_meets(ci_cfg: dict, sdk_text: str) -> bool:
+    if not sdk_text:
+        return True
+    for req in ci_cfg.get("requires", []):
+        if not req or not isinstance(req, str):
+            continue
+        if not any(line.startswith(req) for line in sdk_text.splitlines()):
+            return False
+    req_any = ci_cfg.get("requires_any", [])
+    if req_any:
+        if not any(any(line.startswith(r.strip()) for line in sdk_text.splitlines()) for r in req_any if isinstance(r, str)):
+            return False
+    return True
+
+
+def expected_from_artifacts(build_root: Path) -> dict[tuple[str, str, str, str], int]:
+    """Compute expected runs using ci.yml and sdkconfig found in build artifacts.
+    Returns mapping (platform, target, type, sketch) -> expected_count
+    """
+    expected: dict[tuple[str, str, str, str], int] = {}
+    if not build_root.exists():
+        return expected
+    print(f"[DEBUG] Scanning build artifacts in: {build_root}", file=sys.stderr)
+    for artifact_dir in build_root.iterdir():
+        if not artifact_dir.is_dir():
+            continue
+        m = re.match(r"test-bin-([A-Za-z0-9_\-]+)-([A-Za-z0-9_\-]+)", artifact_dir.name)
+        if not m:
+            continue
+        target = m.group(1)
+        test_type = m.group(2)
+        print(f"[DEBUG] Artifact group target={target} type={test_type} dir={artifact_dir}", file=sys.stderr)
+
+        # Group build*.tmp directories by sketch to avoid processing same ci.yml multiple times
+        # Structure: test-bin-<target>-<type>/<sketch>/build*.tmp/
+        sketches_processed = set()
+
+        for ci_path in artifact_dir.rglob("ci.yml"):
+            if not ci_path.is_file():
+                continue
+            build_tmp = ci_path.parent
+            if not re.search(r"build\d*\.tmp$", build_tmp.name):
+                continue
+
+            # Path structure is: test-bin-<target>-<type>/<sketch>/build*.tmp/ci.yml
+            sketch = build_tmp.parent.name
+
+            # Skip if we already processed this sketch (same ci.yml in multiple build*.tmp)
+            if sketch in sketches_processed:
+                continue
+            sketches_processed.add(sketch)
+
+            print(f"[DEBUG]  Processing sketch={sketch} from artifact {artifact_dir.name}", file=sys.stderr)
+
+            sdk_path = build_tmp / "sdkconfig"
+            try:
+                ci_text = ci_path.read_text(encoding="utf-8")
+            except Exception as e:
+                print(f"[DEBUG]   Skip (failed to read ci.yml: {e})", file=sys.stderr)
+                continue
+            try:
+                sdk_text = sdk_path.read_text(encoding="utf-8", errors="ignore") if sdk_path.exists() else ""
+            except Exception:
+                sdk_text = ""
+
+            ci = _parse_ci_yml(ci_text)
+            fqbn_counts = _fqbn_counts_from_yaml(ci)
+
+            # Determine allowed platforms for this test
+            # Performance tests are only run on hardware
+            if test_type == "performance":
+                allowed_platforms = ["hardware"]
+            else:
+                allowed_platforms = []
+                platforms_cfg = ci.get("platforms") if isinstance(ci, dict) else None
+                for plat in ("hardware", "wokwi", "qemu"):
+                    dis = None
+                    if isinstance(platforms_cfg, dict):
+                        dis = platforms_cfg.get(plat)
+                    if dis is False:
+                        continue
+                    allowed_platforms.append(plat)
+
+            # Requirements check
+            minimal = {
+                "requires": ci.get("requires") or [],
+                "requires_any": ci.get("requires_any") or [],
+            }
+            if not _sdkconfig_meets(minimal, sdk_text):
+                print(f"[DEBUG]   Skip (requirements not met): target={target} type={test_type} sketch={sketch}", file=sys.stderr)
+                continue
+
+            # Expected runs = number from fqbn_counts in ci.yml (how many FQBNs for this target)
+            exp_runs = fqbn_counts.get(target, 0) or 1
+            print(f"[DEBUG]   ci.yml specifies {exp_runs} FQBN(s) for target={target}", file=sys.stderr)
+
+            for plat in allowed_platforms:
+                expected[(plat, target, test_type, sketch)] = exp_runs
+                print(f"[DEBUG]   Expected: plat={plat} target={target} type={test_type} sketch={sketch} runs={exp_runs}", file=sys.stderr)
+
+        if len(sketches_processed) == 0:
+            print(f"[DEBUG]  No sketches found in this artifact group", file=sys.stderr)
+    return expected
+
+
+def scan_executed_xml(xml_root: Path, valid_types: set[str]) -> dict[tuple[str, str, str, str], int]:
+    """Return executed counts per (platform, target, type, sketch).
+    Type/sketch/target are inferred from .../<type>/<sketch>/<target>/<file>.xml
+    """
+    counts: dict[tuple[str, str, str, str], int] = {}
+    if not xml_root.exists():
+        print(f"[DEBUG] Results root not found: {xml_root}", file=sys.stderr)
+        return counts
+    print(f"[DEBUG] Scanning executed XMLs in: {xml_root}", file=sys.stderr)
+    for xml_path in xml_root.rglob("*.xml"):
+        if not xml_path.is_file():
+            continue
+        rel = str(xml_path)
+        platform = "hardware"
+        if "test-results-wokwi-" in rel:
+            platform = "wokwi"
+        elif "test-results-qemu-" in rel:
+            platform = "qemu"
+        # Expect .../<type>/<sketch>/<target>/*.xml
+        parts = xml_path.parts
+        t_idx = -1
+        for i, p in enumerate(parts):
+            if p in valid_types:
+                t_idx = i
+        if t_idx == -1 or t_idx + 3 >= len(parts):
+            continue
+        test_type = parts[t_idx]
+        sketch = parts[t_idx + 1]
+        target = parts[t_idx + 2]
+        key = (platform, target, test_type, sketch)
+        old_count = counts.get(key, 0)
+        counts[key] = old_count + 1
+        print(f"[DEBUG]  Executed XML #{old_count + 1}: plat={platform} target={target} type={test_type} sketch={sketch} file={xml_path.name}", file=sys.stderr)
+    print(f"[DEBUG] Executed entries discovered: {len(counts)}", file=sys.stderr)
+    return counts
+
+
+def write_missing_xml(out_root: Path, platform: str, target: str, test_type: str, sketch: str, missing_count: int):
+    out_tests_dir = out_root / f"test-results-{platform}" / "tests" / test_type / sketch / target
+    out_tests_dir.mkdir(parents=True, exist_ok=True)
+    # Create one XML per missing index
+    for idx in range(missing_count):
+        suite_name = f"{test_type}_{platform}_{target}_{sketch}"
+        root = Element("testsuite", name=suite_name, tests="1", failures="0", errors="1")
+        case = SubElement(root, "testcase", classname=f"{test_type}.{sketch}", name="missing-run")
+        error = SubElement(case, "error", message="Expected test run missing")
+        error.text = "This placeholder indicates an expected test run did not execute."
+        tree = ElementTree(root)
+        out_file = out_tests_dir / f"{sketch}_missing_{idx}.xml"
+        tree.write(out_file, encoding="utf-8", xml_declaration=True)
+
+
+def main():
+    # Args: <build_artifacts_dir> <test_results_dir> <output_junit_dir>
+    if len(sys.argv) != 4:
+        print(f"Usage: {sys.argv[0]} <build_artifacts_dir> <test_results_dir> <output_junit_dir>", file=sys.stderr)
+        return 2
+
+    build_root = Path(sys.argv[1]).resolve()
+    results_root = Path(sys.argv[2]).resolve()
+    out_root = Path(sys.argv[3]).resolve()
+
+    # Validate inputs
+    if not build_root.is_dir():
+        print(f"ERROR: Build artifacts directory not found: {build_root}", file=sys.stderr)
+        return 2
+    if not results_root.is_dir():
+        print(f"ERROR: Test results directory not found: {results_root}", file=sys.stderr)
+        return 2
+    # Ensure output directory exists
+    try:
+        out_root.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        print(f"ERROR: Failed to create output directory {out_root}: {e}", file=sys.stderr)
+        return 2
+
+    # Read matrices from environment variables injected by workflow
+    hw_enabled = (os.environ.get("HW_TESTS_ENABLED", "false").lower() == "true")
+    wokwi_enabled = (os.environ.get("WOKWI_TESTS_ENABLED", "false").lower() == "true")
+    qemu_enabled = (os.environ.get("QEMU_TESTS_ENABLED", "false").lower() == "true")
+
+    hw_targets = parse_array(os.environ.get("HW_TARGETS", "[]"))
+    wokwi_targets = parse_array(os.environ.get("WOKWI_TARGETS", "[]"))
+    qemu_targets = parse_array(os.environ.get("QEMU_TARGETS", "[]"))
+
+    hw_types = parse_array(os.environ.get("HW_TYPES", "[]"))
+    wokwi_types = parse_array(os.environ.get("WOKWI_TYPES", "[]"))
+    qemu_types = parse_array(os.environ.get("QEMU_TYPES", "[]"))
+
+    expected = expected_from_artifacts(build_root)  # (platform, target, type, sketch) -> expected_count
+    executed_types = set(hw_types + wokwi_types + qemu_types)
+    executed = scan_executed_xml(results_root, executed_types)      # (platform, target, type, sketch) -> count
+    print(f"[DEBUG] Expected entries computed: {len(expected)}", file=sys.stderr)
+
+    # Filter expected by enabled platforms and target/type matrices
+    enabled_plats = set()
+    if hw_enabled:
+        enabled_plats.add("hardware")
+    if wokwi_enabled:
+        enabled_plats.add("wokwi")
+    if qemu_enabled:
+        enabled_plats.add("qemu")
+
+    # Build platform-specific target and type sets
+    plat_targets = {
+        "hardware": set(hw_targets),
+        "wokwi": set(wokwi_targets),
+        "qemu": set(qemu_targets),
+    }
+    plat_types = {
+        "hardware": set(hw_types),
+        "wokwi": set(wokwi_types),
+        "qemu": set(qemu_types),
+    }
+
+    missing_total = 0
+    extra_total = 0
+    for (plat, target, test_type, sketch), exp_count in expected.items():
+        if plat not in enabled_plats:
+            continue
+        # Check if target and type are valid for this specific platform
+        if target not in plat_targets.get(plat, set()):
+            continue
+        if test_type not in plat_types.get(plat, set()):
+            continue
+        got = executed.get((plat, target, test_type, sketch), 0)
+        if got < exp_count:
+            print(f"[DEBUG] Missing: plat={plat} target={target} type={test_type} sketch={sketch} expected={exp_count} got={got}", file=sys.stderr)
+            write_missing_xml(out_root, plat, target, test_type, sketch, exp_count - got)
+            missing_total += (exp_count - got)
+        elif got > exp_count:
+            print(f"[DEBUG] Extra runs: plat={plat} target={target} type={test_type} sketch={sketch} expected={exp_count} got={got}", file=sys.stderr)
+            extra_total += (got - exp_count)
+
+    # Check for executed tests that were not expected at all
+    for (plat, target, test_type, sketch), got in executed.items():
+        if (plat, target, test_type, sketch) not in expected:
+            print(f"[DEBUG] Unexpected test: plat={plat} target={target} type={test_type} sketch={sketch} got={got} (not in expected)", file=sys.stderr)
+
+    print(f"Generated {missing_total} placeholder JUnit files for missing runs.", file=sys.stderr)
+    if extra_total > 0:
+        print(f"WARNING: {extra_total} extra test runs detected (more than expected).", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
+

.github/scripts/get_affected.py

@@ -63,7 +63,7 @@
 Build file patterns
 --------------------
 - **build_files**: Core Arduino build system files (platform.txt, variants/**, etc.)
-- **sketch_build_files**: Sketch-specific files (ci.json, *.csv in example directories)
+- **sketch_build_files**: Sketch-specific files (ci.yml, *.csv in example directories)
 - **idf_build_files**: Core IDF build system files (CMakeLists.txt, idf_component.yml, etc.)
 - **idf_project_files**: Project-specific IDF files (per-example CMakeLists.txt, sdkconfig, etc.)
 
@@ -128,7 +128,7 @@
 # Files that are used by the sketch build system.
 # If any of these files change, the sketch should be recompiled.
 sketch_build_files = [
-    "libraries/*/examples/**/ci.json",
+    "libraries/*/examples/**/ci.yml",
     "libraries/*/examples/**/*.csv",
 ]
 
@@ -150,7 +150,7 @@
 # If any of these files change, the example that uses them should be recompiled.
 idf_project_files = [
     "idf_component_examples/*/CMakeLists.txt",
-    "idf_component_examples/*/ci.json",
+    "idf_component_examples/*/ci.yml",
     "idf_component_examples/*/*.csv",
     "idf_component_examples/*/sdkconfig*",
     "idf_component_examples/*/main/*",

.github/scripts/on-push-idf.sh

@@ -17,9 +17,9 @@ fi
 
 for example in $affected_examples; do
     example_path="$PWD/components/arduino-esp32/$example"
-    if [ -f "$example_path/ci.json" ]; then
+    if [ -f "$example_path/ci.yml" ]; then
         # If the target is listed as false, skip the sketch. Otherwise, include it.
-        is_target=$(jq -r --arg target "$IDF_TARGET" '.targets[$target]' "$example_path/ci.json")
+        is_target=$(yq eval ".targets.${IDF_TARGET}" "$example_path/ci.yml" 2>/dev/null)
         if [[ "$is_target" == "false" ]]; then
             printf "\n\033[93mSkipping %s for target %s\033[0m\n\n" "$example" "$IDF_TARGET"
             continue