test-all.yml

#
# This workflow is run either by a manual workflow dispatch, or
# as part of a pull request check.
#
# It builds a version of GAP (which one depends on its inputs), then builds
# all packages (resp. all matching some glob), then runs the tests of all
# packages (resp. all matching some glob) with a testfile. The various
# test results are aggregated into a test-status JSON file and is uploaded
# as an artifact. Finally, the test results are compared against the latest
# "official" test run. A human-readable comparison report is generated as a
# MARKDOWN file and a summary of the changes as a test-status-diff JSON file
# which other workflows can process. Both files are uploaded as artifacts.
#
name: "Test packages"

on:
  workflow_dispatch:
    inputs:
      which-gap:
        description: 'Either a GAP branch name or a GAP version'
        required: true
        type: string
        default: master  # or 4.11.1 or ...
      which-gap-repo:
        description: 'URL of any fork of the GAP repository'
        required: false
        type: string
        default: https://github.com/gap-system/gap.git
      pkg-build-glob:
        description: 'Only build packages matching the given glob'
        required: false
        type: string
        default: "*"
      pkg-test-glob:
        description: 'Only test packages matching the given glob'
        required: false
        type: string
        default: "*"
      # see https://github.com/marketplace/actions/debugging-with-tmate
      #debug_enabled:
      #  description: 'Run the build with tmate debugging enabled'
      #  type: boolean
      #  default: false
  workflow_call:
    inputs:
      which-gap:
        description: 'Either a GAP branch name or a GAP version'
        required: true
        type: string
        default: master  # or 4.11.1 or ...
      which-gap-repo:
        description: 'URL of any fork of the GAP repository'
        required: false
        type: string
        default: https://github.com/gap-system/gap.git
      pkg-build-glob:
        description: 'Only build packages matching the given glob'
        required: false
        type: string
        default: "*"
      pkg-test-glob:
        description: 'Only test packages matching the given glob'
        required: false
        type: string
        default: "*"

jobs:
  build:
    # If you change this name, you must adjust the corresponding name
    # in tools/generate_test_status.py
    name: "Build GAP and packages"
    runs-on: ubuntu-20.04
    outputs:
      matrix: ${{ steps.get-names.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3

      # Cache the package archives we download across runs of this job
      # to speed up things considerably
      - name: "Cache archives"
        uses: actions/cache@v3
        with:
          path: _archives
          key: archives-${{ hashFiles('packages/*/meta.json') }}
          restore-keys: |
            archives-

      - name: "Install package distribution tools"
        run: python -m pip install -r tools/requirements.txt

      # TOOD: dependencies should come from a container
      - name: "Install binary package dependencies"
        run: |
          sudo apt-get update
          deps=$(tools/gather_dependencies.py packages/*/meta.json)
          if [[ -n "$deps" ]]; then
            echo "Installing required binary depedencies: $deps"
            sudo apt-get install --no-install-recommends $deps
          else
            echo "No required binary depedencies to be installed"
          fi

      - name: "Download packages"
        run: tools/download_packages.py packages/*/meta.json

      - name: "Cleanup archives"
        run: tools/cleanup_archives.py

      # the following step is useful for debugging
      - name: "Determine system architecture for -march=native"
        run: |
          echo "cc -march=native: $(tools/compiler_arch.sh cc)"
          echo "c++ -march=native: $(tools/compiler_arch.sh c++)"

      # Setup ccache, to speed up repeated compilation of the same binaries
      # (i.e., GAP and the packages)
      - name: "Setup ccache"
        uses: Chocobo1/setup-ccache-action@v1
        with:
          update_packager_index: false

      - name: "Install GAP"
        run: |
          echo "::group::fetch"
          whichgap="${{ github.event.inputs.which-gap || inputs.which-gap }}"
          whichgaprepo="${{ github.event.inputs.which-gap-repo || inputs.which-gap-repo }}"
          if [[ "${whichgap}" == 4.* ]]; then
            # assume it is a tag
            wget --quiet https://github.com/gap-system/gap/releases/download/v${whichgap}/gap-${whichgap}-core.tar.gz
            tar xf gap-${whichgap}-core.tar.gz
            rm gap-${whichgap}-core.tar.gz
            mv gap-${whichgap} $HOME/gap
          else
            git clone --depth=2 -b ${whichgap} ${whichgaprepo} $HOME/gap
          fi
          cd $HOME/gap
          echo "::endgroup::"

          echo "::group::autogen"
          ./autogen.sh
          echo "::endgroup::"

          echo "::group::configure"
          ./configure
          echo "::endgroup::"

          echo "::group::make"
          make -j4
          echo "::endgroup::"

          # put GAP into PATH
          ln -s $HOME/gap/gap /usr/local/bin/gap

      - name: "Extract packages"
        run: |
          mkdir -p $HOME/gap/pkg
          cd $HOME/gap/pkg
          for f in $GITHUB_WORKSPACE/_archives/* ; do
            echo "::group::$(basename $f)"
            ls -l $f
            case "$f" in
              *.tar.*)
                echo "Extracting $f"
                tar -xvf "$f"
              ;;
              *.zip)
                echo "Extracting $f"
                unzip "$f"
              ;;
              *)
                echo "Skipping $f"
              ;;
            esac
            echo "::endgroup::"
          done

      - name: "Build packages"
        run: |
          cd $HOME/gap/pkg

          # skip xgap: no X11 headers, and no means to test it
          rm -rf xgap*

          MAKEFLAGS=-j3 ../bin/BuildPackages.sh --strict ${{ github.event.inputs.pkg-build-glob || inputs.pkg-build-glob }}

      - name: "Test LoadAllPackages + GAP test suite"
        run: |
          gap -A --quitonbreak -r -c "
                SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG);
                LoadAllPackages();
                SetInfoLevel(InfoPackageLoading, PACKAGE_ERROR);
                ReadGapRoot(\"tst/testinstall.g\");
                QUIT;
                "

      - name: "Create tarball"
        run: |
          cd $HOME
          tar --exclude-vcs --exclude=build --exclude=.libs -cf gap.tar.zst gap

      - name: "Upload GAP with packages as artifact"
        uses: actions/upload-artifact@v3
        with:
          name: gap-${{ github.event.inputs.which-gap || inputs.which-gap }}
          path: /home/runner/gap.tar.zst

      - name: "Create jobs for matching packages"
        id: get-names
        run: |
            MATRIX="{\"package\":["
            SKIPPED=""
            for PKG in packages/${{ github.event.inputs.pkg-test-glob || inputs.pkg-test-glob }}/meta.json; do
              PKG=${PKG%"/meta.json"}
              PKG=${PKG#"packages/"}
              if ! jq -e -r '.TestFile' < packages/${PKG}/meta.json > /dev/null ; then
                echo "Skip ${PKG}: no TestFile"
                SKIPPED="${SKIPPED}${PKG} "
              elif [[ ${PKG} == xgap ]]; then
                echo "Skip xgap: no X11 headers, and no means to test it"
                SKIPPED="${SKIPPED}${PKG} "
              elif [[ ${PKG} == polycyclic ]]; then
                # HACK FIXME TODO: skip polycyclic for now
                echo "Skip polycyclic tests for now, as they run in an infinite (?) loop"
                echo "Re-enable them once there is a new polycyclic release"
                SKIPPED="${SKIPPED}${PKG} "
              else
                MATRIX="${MATRIX}\"${PKG}\","
              fi
            done
            MATRIX="${MATRIX}]}"
            echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

  test-package:
    # If you change this name, you must adjust the corresponding name
    # in tools/generate_test_status.py
    name: "${{ matrix.package }}"
    if: ${{ needs.build.outputs.matrix != '{"package":[]}' }}
    needs: build
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.build.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v3

      - name: "Install package distribution tools"
        run: python -m pip install -r tools/requirements.txt

      # the following step is useful for debugging
      - name: "Determine system architecture for -march=native"
        run: |
          echo "cc -march=native: $(tools/compiler_arch.sh cc)"
          echo "c++ -march=native: $(tools/compiler_arch.sh c++)"

      - name: "Download GAP from previous job"
        uses: actions/download-artifact@v3
        with:
          name: gap-${{ github.event.inputs.which-gap || inputs.which-gap }}

      - name: "Extract GAP artifact"
        run: |
          cd $HOME
          tar xvf $GITHUB_WORKSPACE/gap.tar.zst
          ln -s $HOME/gap/gap /usr/local/bin/gap
          cd $GITHUB_WORKSPACE

      - name: "Install binary package dependencies"
        run: |
          sudo apt-get update
          deps=$(tools/gather_dependencies.py ${{ matrix.package }})
          if [[ -n "$deps" ]]; then
            echo "Installing required binary depedencies: $deps"
            sudo apt-get install --no-install-recommends $deps
          else
            echo "No required binary depedencies to be installed"
          fi

      #- name: "Setup tmate session"
      #  uses: mxschmitt/action-tmate@v3
      #  if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}

      - name: "Run tests"
        timeout-minutes: 10
        id: tests-default
        run: |
          PKG=${{ matrix.package }}
          gap --quitonbreak -r -c "
                pkgname := \"$PKG\";

                SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG);
                LoadPackage(pkgname);
                SetInfoLevel(InfoPackageLoading, PACKAGE_ERROR);

                res:=TestPackage(pkgname);
                FORCE_QUIT_GAP(res);
                "

      - name: "Run tests with OnlyNeeded"
        timeout-minutes: 10
        if: always()
        id: tests-only-needed
        run: |
          PKG=${{ matrix.package }}
          gap -A --quitonbreak -r -c "
                pkgname := \"$PKG\";

                # WORKAROUNDS for various packages which need additional packages
                if pkgname = \"agt\" then
                  LoadPackage(\"grape\" : OnlyNeeded);
                elif pkgname = \"ctbllib\" then
                  LoadPackage(\"browse\" : OnlyNeeded);
                  LoadPackage(\"tomlib\" : OnlyNeeded);
                  LoadPackage(\"spinsym\" : OnlyNeeded);
                elif pkgname = \"gapdoc\" then
                  LoadPackage(\"browse\" : OnlyNeeded);
                elif pkgname = \"guarana\" then
                  LoadPackage(\"nq\" : OnlyNeeded);
                elif pkgname = \"hap\" then
                  # hap really needs all (?) its optional dependencies in its tests
                  # so we cheat and load it early
                  SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG);
                  LoadPackage(\"hap\");
                elif pkgname = \"packagemanager\" then
                  QUIT_GAP(true); # HACK: workaround for weird failures
                elif pkgname = \"sglppow\" then
                  LoadPackage(\"liepring\" : OnlyNeeded);
                elif pkgname = \"standardff\" then
                  LoadPackage(\"factint\" : OnlyNeeded);
                elif pkgname = \"ugaly\" then
                  LoadPackage(\"fga\" : OnlyNeeded);
                fi;

                SetInfoLevel(InfoPackageLoading, PACKAGE_DEBUG);
                LoadPackage(pkgname : OnlyNeeded);
                SetInfoLevel(InfoPackageLoading, PACKAGE_ERROR);

                res:=TestPackage(pkgname);
                FORCE_QUIT_GAP(res);
                "

  report:
    name: "Generate report"
    needs: test-package
    if: always()
    runs-on: ubuntu-20.04
    outputs:
      test-status: ${{ steps.test-status.outputs.test-status }}
    steps:
      - uses: actions/checkout@v3

      - name: "Install package distribution tools"
        run: python -m pip install -r tools/requirements.txt

      - name: "Generate test-status.json"
        id: test-status
        run: |
          # Is this a workflow_call?
          if [ "${{inputs.which-gap}}" != "" ]; then
            JOB_NAME_PREFIX="${{inputs.which-gap}} / "
          else
            JOB_NAME_PREFIX=""
          fi
          ROOT='data/reports'
          # relative path (with respect to ROOT) to generated test-status.json, i.e. the "id" entry of the json-file.
          DIR_REL=$(tools/generate_test_status.py ${{secrets.GITHUB_TOKEN}} ${{ github.repository }} "$GITHUB_RUN_ID" "$GITHUB_SHA" ${{ github.event.inputs.which-gap || inputs.which-gap }} "$JOB_NAME_PREFIX")
          DIR="${ROOT}/${DIR_REL}"
          echo "dir=${DIR}" >> $GITHUB_OUTPUT
          echo "dir-rel=${DIR_REL}" >> $GITHUB_OUTPUT

      - name: "Download latest report"
        id: download-latest-report
        run: |
          ROOT="data/reports"
          DIR_SYM_REL=latest-${{ github.event.inputs.which-gap || inputs.which-gap }}
          DIR_SYM="${ROOT}/${DIR_SYM_REL}"
          URL_SYM="https://raw.githubusercontent.com/${{ github.repository }}/${DIR_SYM}"
          # Check if file at url exists,
          # so we do not run into errors for the first run of the script
          # (when the first report is created and thus no latest report is available)
          if wget --spider "${URL_SYM}"; then
            # wget downloads the "symbolic link" as a plain file
            # containing as content the path that the symlink points to,
            # so we need to convert this into a real symbolic link.
            DIR_REL=$(wget -O - ${URL_SYM})
            DIR="${ROOT}/${DIR_REL}"
            ln -s ${DIR_REL} ${DIR_SYM}
            URL="https://raw.githubusercontent.com/${{ github.repository }}/${DIR}/test-status.json"
            wget -P ${DIR} ${URL}
          fi

      - name: "Generate report"
        id: report
        run: tools/generate_report.py ${{ steps.test-status.outputs.dir-rel }} latest-${{ github.event.inputs.which-gap || inputs.which-gap }}

      - name: "Upload report as artifact"
        uses: actions/upload-artifact@v3
        with:
          name: "report-${{ github.event.inputs.which-gap || inputs.which-gap }}"
          path: "${{ steps.test-status.outputs.dir }}"