|
| 1 | +on: |
| 2 | + workflow_call: |
| 3 | + inputs: |
| 4 | + artifact-prefix: |
| 5 | + description: | |
| 6 | + Prefix for charm package GitHub artifact(s) |
| 7 | + |
| 8 | + Use canonical/data-platform-workflows build_charm.yaml to build the charm(s) |
| 9 | + required: true |
| 10 | + type: string |
| 11 | + |
| 12 | +jobs: |
| 13 | + collect-integration-tests: |
| 14 | + name: Collect integration test spread jobs |
| 15 | + runs-on: ubuntu-latest |
| 16 | + timeout-minutes: 5 |
| 17 | + steps: |
| 18 | + - name: Checkout |
| 19 | + uses: actions/checkout@v4 |
| 20 | + - name: Set up environment |
| 21 | + run: | |
| 22 | + sudo snap install charmcraft --classic |
| 23 | + pipx install tox poetry |
| 24 | + - name: Collect spread jobs |
| 25 | + id: collect-jobs |
| 26 | + shell: python |
| 27 | + run: | |
| 28 | + import json |
| 29 | + import os |
| 30 | + import subprocess |
| 31 | +
|
| 32 | + spread_jobs = ( |
| 33 | + subprocess.run( |
| 34 | + ["charmcraft", "test", "--list", "github-ci"], capture_output=True, check=True, text=True |
| 35 | + ) |
| 36 | + .stdout.strip() |
| 37 | + .split("\n") |
| 38 | + ) |
| 39 | + jobs = [] |
| 40 | + for job in spread_jobs: |
| 41 | + # Example `job`: "github-ci:ubuntu-24.04:tests/spread/test_charm.py:juju36" |
| 42 | + _, runner, task, variant = job.split(":") |
| 43 | + # Example: "test_charm.py" |
| 44 | + task = task.removeprefix("tests/spread/") |
| 45 | + if runner.endswith("-arm"): |
| 46 | + architecture = "arm64" |
| 47 | + else: |
| 48 | + architecture = "amd64" |
| 49 | + # Example: "test_charm.py:juju36 | amd64" |
| 50 | + name = f"{task}:{variant} | {architecture}" |
| 51 | + # ":" character not valid in GitHub Actions artifact |
| 52 | + name_in_artifact = f"{task}-{variant}-{architecture}" |
| 53 | + jobs.append({ |
| 54 | + "spread_job": job, |
| 55 | + "name": name, |
| 56 | + "name_in_artifact": name_in_artifact, |
| 57 | + "runner": runner, |
| 58 | + }) |
| 59 | + output = f"jobs={json.dumps(jobs)}" |
| 60 | + print(output) |
| 61 | + with open(os.environ["GITHUB_OUTPUT"], "a") as file: |
| 62 | + file.write(output) |
| 63 | + - name: Generate Allure default test results |
| 64 | + if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }} |
| 65 | + run: tox run -e integration -- tests/integration --allure-default-dir=allure-default-results |
| 66 | + - name: Upload Allure default results |
| 67 | + # Default test results in case the integration tests time out or runner set up fails |
| 68 | + # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) |
| 69 | + if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }} |
| 70 | + uses: actions/upload-artifact@v4 |
| 71 | + with: |
| 72 | + name: allure-default-results-integration-test |
| 73 | + path: allure-default-results/ |
| 74 | + if-no-files-found: error |
| 75 | + outputs: |
| 76 | + jobs: ${{ steps.collect-jobs.outputs.jobs }} |
| 77 | + |
| 78 | + integration-test: |
| 79 | + strategy: |
| 80 | + fail-fast: false |
| 81 | + matrix: |
| 82 | + job: ${{ fromJSON(needs.collect-integration-tests.outputs.jobs) }} |
| 83 | + name: ${{ matrix.job.name }} |
| 84 | + needs: |
| 85 | + - collect-integration-tests |
| 86 | + runs-on: ${{ matrix.job.runner }} |
| 87 | + timeout-minutes: 217 # Sum of steps `timeout-minutes` + 5 |
| 88 | + steps: |
| 89 | + - name: Free up disk space |
| 90 | + timeout-minutes: 1 |
| 91 | + run: | |
| 92 | + printf '\nDisk usage before cleanup\n' |
| 93 | + df --human-readable |
| 94 | + # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 |
| 95 | + rm -r /opt/hostedtoolcache/ |
| 96 | + printf '\nDisk usage after cleanup\n' |
| 97 | + df --human-readable |
| 98 | + - name: Checkout |
| 99 | + timeout-minutes: 3 |
| 100 | + uses: actions/checkout@v4 |
| 101 | + - name: Set up environment |
| 102 | + timeout-minutes: 5 |
| 103 | + run: sudo snap install charmcraft --classic |
| 104 | + # TODO: remove when https://github.com/canonical/charmcraft/issues/2105 and |
| 105 | + # https://github.com/canonical/charmcraft/issues/2130 fixed |
| 106 | + - run: | |
| 107 | + sudo snap install go --classic |
| 108 | + go install github.com/snapcore/spread/cmd/spread@latest |
| 109 | + - name: Download packed charm(s) |
| 110 | + timeout-minutes: 5 |
| 111 | + uses: actions/download-artifact@v4 |
| 112 | + with: |
| 113 | + pattern: ${{ inputs.artifact-prefix }}-* |
| 114 | + merge-multiple: true |
| 115 | + - name: Run spread job |
| 116 | + timeout-minutes: 180 |
| 117 | + id: spread |
| 118 | + # TODO: replace with `charmcraft test` when |
| 119 | + # https://github.com/canonical/charmcraft/issues/2105 and |
| 120 | + # https://github.com/canonical/charmcraft/issues/2130 fixed |
| 121 | + run: ~/go/bin/spread -vv -artifacts=artifacts '${{ matrix.job.spread_job }}' |
| 122 | + - name: Upload Allure results |
| 123 | + timeout-minutes: 3 |
| 124 | + # Only upload results from one spread system & one spread variant |
| 125 | + # Allure can only process one result per pytest test ID. If parameterization is done via |
| 126 | + # spread instead of pytest, there will be overlapping pytest test IDs. |
| 127 | + if: ${{ (success() || (failure() && steps.spread.outcome == 'failure')) && startsWith(matrix.job.spread_job, 'github-ci:ubuntu-24.04:') && endsWith(matrix.job.spread_job, ':juju36') && github.event_name == 'schedule' && github.run_attempt == '1' }} |
| 128 | + uses: actions/upload-artifact@v4 |
| 129 | + with: |
| 130 | + name: allure-results-integration-test-${{ matrix.job.name_in_artifact }} |
| 131 | + path: artifacts/${{ matrix.job.spread_job }}/allure-results/ |
| 132 | + if-no-files-found: error |
| 133 | + - timeout-minutes: 1 |
| 134 | + if: ${{ success() || (failure() && steps.spread.outcome == 'failure') }} |
| 135 | + run: snap list |
| 136 | + - name: Select model |
| 137 | + timeout-minutes: 1 |
| 138 | + # `!contains(matrix.job.spread_job, 'juju29')` workaround for juju 2 error: |
| 139 | + # "ERROR cannot acquire lock file to read controller concierge-microk8s: unable to open |
| 140 | + # /tmp/juju-store-lock-3635383939333230: permission denied" |
| 141 | + # Unable to workaround error with `sudo rm /tmp/juju-*` |
| 142 | + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} |
| 143 | + id: juju-switch |
| 144 | + run: | |
| 145 | + # sudo needed since spread runs scripts as root |
| 146 | + # "testing" is default model created by concierge |
| 147 | + sudo juju switch testing |
| 148 | + mkdir ~/logs/ |
| 149 | + - name: juju status |
| 150 | + timeout-minutes: 1 |
| 151 | + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} |
| 152 | + run: sudo juju status --color --relations | tee ~/logs/juju-status.txt |
| 153 | + - name: juju debug-log |
| 154 | + timeout-minutes: 3 |
| 155 | + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} |
| 156 | + run: sudo juju debug-log --color --replay --no-tail | tee ~/logs/juju-debug-log.txt |
| 157 | + - name: jhack tail |
| 158 | + timeout-minutes: 3 |
| 159 | + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} |
| 160 | + run: sudo jhack tail --printer raw --replay --no-watch | tee ~/logs/jhack-tail.txt |
| 161 | + - name: Upload logs |
| 162 | + timeout-minutes: 5 |
| 163 | + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} |
| 164 | + uses: actions/upload-artifact@v4 |
| 165 | + with: |
| 166 | + name: logs-integration-test-${{ matrix.job.name_in_artifact }} |
| 167 | + path: ~/logs/ |
| 168 | + if-no-files-found: error |
| 169 | + - name: Disk usage |
| 170 | + timeout-minutes: 1 |
| 171 | + if: ${{ success() || (failure() && steps.spread.outcome == 'failure') }} |
| 172 | + run: df --human-readable |
| 173 | + |
| 174 | + allure-report: |
| 175 | + # TODO future improvement: use concurrency group for job |
| 176 | + name: Publish Allure report |
| 177 | + if: ${{ !cancelled() && github.event_name == 'schedule' && github.run_attempt == '1' }} |
| 178 | + needs: |
| 179 | + - integration-test |
| 180 | + runs-on: ubuntu-latest |
| 181 | + timeout-minutes: 5 |
| 182 | + steps: |
| 183 | + - name: Download Allure |
| 184 | + # Following instructions from https://allurereport.org/docs/install-for-linux/#install-from-a-deb-package |
| 185 | + run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb' |
| 186 | + env: |
| 187 | + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 188 | + - name: Install Allure |
| 189 | + run: | |
| 190 | + sudo apt-get update |
| 191 | + sudo apt-get install ./allure_*.deb -y |
| 192 | + # For first run, manually create branch with no history |
| 193 | + # (e.g. |
| 194 | + # git checkout --orphan gh-pages-beta |
| 195 | + # git rm -rf . |
| 196 | + # touch .nojekyll |
| 197 | + # git add .nojekyll |
| 198 | + # git commit -m "Initial commit" |
| 199 | + # git push origin gh-pages-beta |
| 200 | + # ) |
| 201 | + - name: Checkout GitHub pages branch |
| 202 | + uses: actions/checkout@v4 |
| 203 | + with: |
| 204 | + ref: gh-pages-beta |
| 205 | + path: repo/ |
| 206 | + - name: Download default test results |
| 207 | + # Default test results in case the integration tests time out or runner set up fails |
| 208 | + # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) |
| 209 | + uses: actions/download-artifact@v4 |
| 210 | + with: |
| 211 | + path: allure-default-results/ |
| 212 | + name: allure-default-results-integration-test |
| 213 | + - name: Download test results |
| 214 | + uses: actions/download-artifact@v4 |
| 215 | + with: |
| 216 | + path: allure-results/ |
| 217 | + pattern: allure-results-integration-test-* |
| 218 | + merge-multiple: true |
| 219 | + - name: Combine Allure default results & actual results |
| 220 | + # For every test: if actual result available, use that. Otherwise, use default result |
| 221 | + # So that, if actual result not available, Allure report will show "unknown"/"failed" test result |
| 222 | + # instead of omitting the test |
| 223 | + shell: python |
| 224 | + run: | |
| 225 | + import dataclasses |
| 226 | + import json |
| 227 | + import pathlib |
| 228 | +
|
| 229 | +
|
| 230 | + @dataclasses.dataclass(frozen=True) |
| 231 | + class Result: |
| 232 | + test_case_id: str |
| 233 | + path: pathlib.Path |
| 234 | +
|
| 235 | + def __eq__(self, other): |
| 236 | + if not isinstance(other, type(self)): |
| 237 | + return False |
| 238 | + return self.test_case_id == other.test_case_id |
| 239 | +
|
| 240 | +
|
| 241 | + actual_results = pathlib.Path("allure-results") |
| 242 | + default_results = pathlib.Path("allure-default-results") |
| 243 | +
|
| 244 | + results: dict[pathlib.Path, set[Result]] = { |
| 245 | + actual_results: set(), |
| 246 | + default_results: set(), |
| 247 | + } |
| 248 | + for directory, results_ in results.items(): |
| 249 | + for path in directory.glob("*-result.json"): |
| 250 | + with path.open("r") as file: |
| 251 | + id_ = json.load(file)["testCaseId"] |
| 252 | + results_.add(Result(id_, path)) |
| 253 | +
|
| 254 | + actual_results.mkdir(exist_ok=True) |
| 255 | +
|
| 256 | + missing_results = results[default_results] - results[actual_results] |
| 257 | + for default_result in missing_results: |
| 258 | + # Move to `actual_results` directory |
| 259 | + default_result.path.rename(actual_results / default_result.path.name) |
| 260 | + - name: Load test report history |
| 261 | + run: | |
| 262 | + if [[ -d repo/_latest/history/ ]] |
| 263 | + then |
| 264 | + echo 'Loading history' |
| 265 | + cp -r repo/_latest/history/ allure-results/ |
| 266 | + fi |
| 267 | + - name: Create executor.json |
| 268 | + shell: python |
| 269 | + run: | |
| 270 | + # Reverse engineered from https://github.com/simple-elf/allure-report-action/blob/eca283b643d577c69b8e4f048dd6cd8eb8457cfd/entrypoint.sh |
| 271 | + import json |
| 272 | +
|
| 273 | + DATA = { |
| 274 | + "name": "GitHub Actions", |
| 275 | + "type": "github", |
| 276 | + "buildOrder": ${{ github.run_number }}, # TODO future improvement: use run ID |
| 277 | + "buildName": "Run ${{ github.run_id }}", |
| 278 | + "buildUrl": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", |
| 279 | + "reportUrl": "../${{ github.run_number }}/", |
| 280 | + } |
| 281 | + with open("allure-results/executor.json", "w") as file: |
| 282 | + json.dump(DATA, file) |
| 283 | + - name: Generate Allure report |
| 284 | + run: allure generate |
| 285 | + - name: Create index.html |
| 286 | + shell: python |
| 287 | + run: | |
| 288 | + DATA = f"""<!DOCTYPE html> |
| 289 | + <meta charset="utf-8"> |
| 290 | + <meta http-equiv="cache-control" content="no-cache"> |
| 291 | + <meta http-equiv="refresh" content="0; url=${{ github.run_number }}"> |
| 292 | + """ |
| 293 | + with open("repo/index.html", "w") as file: |
| 294 | + file.write(DATA) |
| 295 | + - name: Update GitHub pages branch |
| 296 | + working-directory: repo/ |
| 297 | + # TODO future improvement: commit message |
| 298 | + run: | |
| 299 | + mkdir '${{ github.run_number }}' |
| 300 | + rm -f _latest |
| 301 | + ln -s '${{ github.run_number }}' _latest |
| 302 | + cp -r ../allure-report/. _latest/ |
| 303 | + git add . |
| 304 | + git config user.name "GitHub Actions" |
| 305 | + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" |
| 306 | + git commit -m "Allure report ${{ github.run_number }}" |
| 307 | + # Uses token set in checkout step |
| 308 | + git push origin gh-pages-beta |
0 commit comments