Trying coverage tests on self-hosted runners and coveralls

.github/workflows/self_hosted_build_and_test.yml

@@ -2,40 +2,249 @@ name: Self-Hosted Build and Test
 
 on:
   push:
-    branches: [ develop ]
+    branches:
+      - "**"
   workflow_dispatch:
     inputs:
       ref:
         description: "Branch, tag, or SHA to test (leave empty for current ref)"
         required: false
         default: ""
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.inputs.ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_REQUIRED_VERSION: "3.12.2"
+  FC: gfortran-13
+  BUILD_DIR: build-coverage
+  ANNUAL_SIMULATION_EXCLUDE_REGEX: >-
+    ^integration\.(GSHP-GLHE-BoreholeFieldDesign|PythonPluginSolarHeating|_5ZoneAirCooled_annual|_5ZoneAirCooled_LeapYear_annual|UnitarySystem_MultiSpeedDX_EconoStaging|UnitarySystem_VariableSpeedDX_SZVAV|_ResidentialBase|_ExternalInterface-functionalmockupunit-to-actuator|_ExternalInterface-functionalmockupunit-to-schedule|_ExternalInterface-functionalmockupunit-to-variable)$
+  INTEGRATION_EXCLUDE_REGEX: >-
+    ^integration\.(UnitaryHybridAC_DedicatedOutsideAir|HospitalLowEnergy|GSHP-GLHE-CalcGFunctions|HospitalBaselineReheatReportEMS|HospitalBaseline|RefBldgOutPatientNew2004_Chicago|ASHRAE901_ApartmentHighRise_STD2019_Denver|UnitarySystem_VariableSpeedDX_SZVAV|ASHRAE901_OutPatientHealthCare_STD2019_Denver|UnitarySystem_MultiSpeedDX_EconoStaging|RefBldgSecondarySchoolNew2004_Chicago|RefrigeratedWarehouse|HeatPumpWaterHeaterStratified|ASHRAE901_OfficeLarge_STD2019_Denver_Chiller205|ASHRAE901_OfficeLarge_STD2019_Denver_Chiller205_Detailed|_5ZoneAirCooled_LeapYear_annual|_5ZoneAirCooled_annual|_SmallOffice_Dulles|EcoroofOrlando|EcoroofOrlando_NoSitePrec)$
+
 jobs:
-  build_and_test:
-    name: Build and Test (Ubuntu matrix)
-    strategy:
-      fail-fast: false
-      matrix:
-        runner:
-          - [self-hosted, linux, x64, ubuntu-24.04]
-    runs-on: ${{ matrix.runner }}
+  coverage:
+    name: Coverage
+    runs-on: [ self-hosted, linux, x64, ubuntu-24.04 ]
+    permissions:
+      contents: read
+      statuses: write
     steps:
       - name: Checkout EnergyPlus
         uses: actions/checkout@v6
         with:
           ref: ${{ github.event.inputs.ref || github.ref }}
 
-      - name: Configure
+      - name: Resolve checked out SHA
+        id: checkout-sha
+        shell: bash
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Setup runner
+        id: setup-runner
+        uses: ./.github/actions/setup-runner
+        with:
+          python-version: ${{ env.PYTHON_REQUIRED_VERSION }}
+          python-arch: x64
+
+      - name: Restore ccache
+        uses: actions/cache/restore@v5
+        id: cacheccache-restore
+        with:
+          path: |
+            ${{ steps.setup-runner.outputs.ccache-dir }}
+          key: ccache-self-hosted-linux-${{ steps.setup-runner.outputs.compiler-id
+            }}-coverage-${{ github.event.inputs.ref || github.ref_name }}-${{
+            github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            ccache-self-hosted-linux-${{ steps.setup-runner.outputs.compiler-id }}-coverage-${{ github.event.inputs.ref || github.ref_name }}-
+            ccache-self-hosted-linux-${{ steps.setup-runner.outputs.compiler-id }}-coverage-
+            ccache-self-hosted-linux-${{ steps.setup-runner.outputs.compiler-id }}-
+            ccache-self-hosted-linux-
+
+      - name: Install Linux coverage tools
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get -qq install -y lcov
+
+      - name: Show restored ccache stats
+        shell: bash
+        run: |
+          ccache --show-stats -vv || ccache --show-stats
+          ccache --zero-stats
+          ccache -p
+
+      - name: Configure coverage build
+        run: >
+          cmake -S . -B ${{ env.BUILD_DIR }} -G "Unix Makefiles"
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLINK_WITH_PYTHON:BOOL=ON
+          -DPYTHON_CLI:BOOL=OFF -DPython_REQUIRED_VERSION:STRING=${{
+          env.PYTHON_REQUIRED_VERSION }} -DPython_ROOT_DIR:PATH=${{
+          steps.setup-runner.outputs.python-root-dir }} -DBUILD_FORTRAN:BOOL=ON
+          -DBUILD_TESTING=ON -DBUILD_PACKAGE=OFF
+          -DDOCUMENTATION_BUILD=DoNotBuild -DENABLE_REGRESSION_TESTING:BOOL=OFF
+          -DTEST_ANNUAL_SIMULATION:BOOL=OFF -DENABLE_COVERAGE:BOOL=ON
+          -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DENABLE_PCH:BOOL=OFF
+          -DFORCE_DEBUG_ARITHM_GCC_OR_CLANG:BOOL=ON -DCOMMIT_SHA:STRING=${{
+          steps.checkout-sha.outputs.sha }}
+
+      - name: Build coverage targets
+        run: cmake --build ${{ env.BUILD_DIR }} -j "${{ steps.setup-runner.outputs.nproc
+          }}"
+
+      - name: Show post-build ccache stats
+        shell: bash
+        run: ccache --show-stats -vv || ccache --show-stats
+
+      - name: Run unit tests
+        run: >
+          ctest --test-dir ${{ env.BUILD_DIR }} -E "integration.*"
+          --output-on-failure -j "${{ steps.setup-runner.outputs.nproc }}"
+
+      - name: Generate unit test coverage results
+        working-directory: ${{ env.BUILD_DIR }}
+        run: |
+          set -o pipefail
+          mkdir -p coverage/unit
+          lcov -c -d . -o coverage/unit/lcov.output --base-directory ${{ github.workspace }} --ignore-errors source,source,mismatch,mismatch
+          ${{ steps.setup-runner.outputs.python-root-dir }}/bin/python \
+            ${{ github.workspace }}/scripts/dev/normalize_lcov_paths.py \
+            coverage/unit/lcov.output \
+            --workspace ${{ github.workspace }} \
+            --build-directory ${{ github.workspace }}/${{ env.BUILD_DIR }} \
+            --absolute-output coverage/unit/lcov.output.filtered \
+            --relative-output coverage/unit/lcov.output.coveralls
+          genhtml coverage/unit/lcov.output.filtered -o coverage/unit/lcov-html --demangle-cpp --function-coverage --ignore-errors source,source,unmapped,unmapped | tee coverage/unit/cover.txt
+
+      - name: Process unit test coverage summary
+        working-directory: ${{ env.BUILD_DIR }}/coverage/unit
+        continue-on-error: true
+        run: ${{ steps.setup-runner.outputs.python-root-dir }}/bin/python ${{
+          github.workspace }}/scripts/dev/gha_coverage_summary.py
+
+      - name: Add unit test coverage summary to job summary
+        if: ${{ always() && hashFiles(format('{0}/{1}/coverage/unit/cover.md',
+          github.workspace, env.BUILD_DIR)) != '' }}
+        run: cat ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/unit/cover.md >>
+          "$GITHUB_STEP_SUMMARY"
+
+      - name: Clear coverage counters before integration tests
+        run: ${{ steps.setup-runner.outputs.python-root-dir }}/bin/python ${{
+          github.workspace }}/scripts/dev/clear_coverage_results.py ${{
+          env.BUILD_DIR }}
+
+      - name: Run integration tests excluding slowest and annual-only tests
         run: >
-          cmake -S . -B build
-          -G "Unix Makefiles"
-          -DCMAKE_BUILD_TYPE=Release
-          -DBUILD_TESTING=ON
-          -DBUILD_PACKAGE=OFF
-          -DDOCUMENTATION_BUILD=DoNotBuild
-
-      - name: Build
-        run: cmake --build build -j "$(nproc)"
-
-      - name: Run tests
-        run: ctest --test-dir build -j "$(nproc)"
+          ctest --test-dir ${{ env.BUILD_DIR }} -R "integration.*" -E "${{
+          env.INTEGRATION_EXCLUDE_REGEX }}|${{
+          env.ANNUAL_SIMULATION_EXCLUDE_REGEX }}" --output-on-failure -j "${{
+          steps.setup-runner.outputs.nproc }}"
+
+      - name: Generate integration test coverage results
+        working-directory: ${{ env.BUILD_DIR }}
+        run: |
+          set -o pipefail
+          mkdir -p coverage/integration
+          lcov -c -d . -o coverage/integration/lcov.output --base-directory ${{ github.workspace }} --ignore-errors source,source,mismatch,mismatch
+          ${{ steps.setup-runner.outputs.python-root-dir }}/bin/python \
+            ${{ github.workspace }}/scripts/dev/normalize_lcov_paths.py \
+            coverage/integration/lcov.output \
+            --workspace ${{ github.workspace }} \
+            --build-directory ${{ github.workspace }}/${{ env.BUILD_DIR }} \
+            --absolute-output coverage/integration/lcov.output.filtered \
+            --relative-output coverage/integration/lcov.output.coveralls
+          genhtml coverage/integration/lcov.output.filtered -o coverage/integration/lcov-html --demangle-cpp --function-coverage --ignore-errors source,source,unmapped,unmapped | tee coverage/integration/cover.txt
+
+      - name: Process integration test coverage summary
+        working-directory: ${{ env.BUILD_DIR }}/coverage/integration
+        continue-on-error: true
+        run: ${{ steps.setup-runner.outputs.python-root-dir }}/bin/python ${{
+          github.workspace }}/scripts/dev/gha_coverage_summary.py
+
+      - name: Add integration test coverage summary to job summary
+        if: ${{ always() && hashFiles(format('{0}/{1}/coverage/integration/cover.md',
+          github.workspace, env.BUILD_DIR)) != '' }}
+        run: cat ${{ github.workspace }}/${{ env.BUILD_DIR
+          }}/coverage/integration/cover.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload unit test coverage to Coveralls
+        id: coveralls-unit
+        if: ${{ success() &&
+          hashFiles(format('{0}/{1}/coverage/unit/lcov.output.coveralls',
+          github.workspace, env.BUILD_DIR)) != '' }}
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: ${{ github.workspace }}/${{ env.BUILD_DIR
+            }}/coverage/unit/lcov.output.coveralls
+          format: lcov
+          flag-name: unit-tests
+          parallel: true
+          build-number: ${{ github.run_id }}
+          git-branch: ${{ github.event.inputs.ref || github.ref_name }}
+          git-commit: ${{ steps.checkout-sha.outputs.sha }}
+          fail-on-error: true
+
+      - name: Upload integration test coverage to Coveralls
+        id: coveralls-integration
+        if: ${{ success() &&
+          hashFiles(format('{0}/{1}/coverage/integration/lcov.output.coveralls',
+          github.workspace, env.BUILD_DIR)) != '' }}
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: ${{ github.workspace }}/${{ env.BUILD_DIR
+            }}/coverage/integration/lcov.output.coveralls
+          format: lcov
+          flag-name: integration-tests
+          parallel: true
+          build-number: ${{ github.run_id }}
+          git-branch: ${{ github.event.inputs.ref || github.ref_name }}
+          git-commit: ${{ steps.checkout-sha.outputs.sha }}
+          fail-on-error: true
+
+      - name: Finish Coveralls coverage build
+        if: ${{ success() && steps.coveralls-unit.conclusion == 'success' &&
+          steps.coveralls-integration.conclusion == 'success' }}
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          parallel-finished: true
+          build-number: ${{ github.run_id }}
+          git-branch: ${{ github.event.inputs.ref || github.ref_name }}
+          git-commit: ${{ steps.checkout-sha.outputs.sha }}
+          fail-on-error: true
+
+      - name: Upload unit test coverage artifacts
+        if: ${{ always() &&
+          hashFiles(format('{0}/{1}/coverage/unit/lcov.output.filtered',
+          github.workspace, env.BUILD_DIR)) != '' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit_test_coverage_results
+          path: |
+            ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/unit/lcov.output.filtered
+            ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/unit/lcov.output.coveralls
+            ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/unit/lcov-html
+
+      - name: Upload integration test coverage artifacts
+        if: ${{ always() &&
+          hashFiles(format('{0}/{1}/coverage/integration/lcov.output.filtered',
+          github.workspace, env.BUILD_DIR)) != '' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration_test_coverage_results
+          path: |
+            ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/integration/lcov.output.filtered
+            ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/integration/lcov.output.coveralls
+            ${{ github.workspace }}/${{ env.BUILD_DIR }}/coverage/integration/lcov-html
+
+      - name: Save ccache
+        if: always() && steps.cacheccache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v5
+        with:
+          path: |
+            ${{ steps.setup-runner.outputs.ccache-dir }}
+          key: ${{ steps.cacheccache-restore.outputs.cache-primary-key }}

scripts/dev/gha_coverage_summary.py

@@ -64,13 +64,26 @@
 #   lines......: 7.9% (28765 of 364658 lines)
 #   functions......: 19.6% (2224 of 11327 functions)
 
+import re
 from pathlib import Path
 
+ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
+
+
+def find_coverage_summary(text, label):
+    matches = re.findall(rf"^\s*{re.escape(label)}\.*:\s*(.+)$", text, re.MULTILINE)
+    if matches:
+        return matches[-1].strip()
+    tail = "\n".join(text.splitlines()[-10:])
+    raise RuntimeError(f"Could not find {label} coverage summary in cover.txt. Recent output:\n{tail}")
+
+
 cover_input = Path.cwd() / "cover.txt"
-lines = cover_input.read_text().strip().split("\n")
-line_coverage = lines[-2].strip().split(":")[1].strip()
-line_percent = line_coverage.split(" ")[0]
-function_coverage = lines[-1].strip().split(":")[1].strip()
+cover_text = cover_input.read_text(encoding="utf-8", errors="replace")
+cover_text = ANSI_ESCAPE.sub("", cover_text)
+line_coverage = find_coverage_summary(cover_text, "lines")
+line_percent = line_coverage.split()[0]
+function_coverage = find_coverage_summary(cover_text, "functions")
 cover_output = Path.cwd() / "cover.md"
 content = f"""
 <details>
@@ -79,4 +92,4 @@
   - {line_coverage}
   - {function_coverage}
 </details>"""
-cover_output.write_text(content)
+cover_output.write_text(content, encoding="utf-8")