diff --git a/.github/actions/url-health-check/action.yml b/.github/actions/url-health-check/action.yml new file mode 100644 index 00000000000..36d180d96e0 --- /dev/null +++ b/.github/actions/url-health-check/action.yml @@ -0,0 +1,16 @@ +name: 'URL Health Check' +description: 'Performs URL health check with retries' +inputs: + url: + description: 'Health check URL' + required: true +runs: + using: 'composite' + steps: + - shell: bash + run: | + for i in {1..20}; do + curl --fail -v "${{ inputs.url }}" && exit 0 + sleep 5 + done + exit 1 \ No newline at end of file diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh new file mode 100755 index 00000000000..b2bd8536de3 --- /dev/null +++ b/.github/not-on-master.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ "${GITHUB_REF_NAME}" == "master" ]; then + echo "Skipping this step on master..." +else + exec "$@" +fi diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 599e2b22e00..de1adff2ff8 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1,13 +1,242 @@ name: CI Pipeline on: + push: + branches: + - master + pull_request: + branches: + - '*' workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false + +env: + USER_NAME: root + USER_UID: 1000 + USER_GID: 1000 + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASS: ${{ secrets.DOCKER_PASS }} jobs: - foo: - runs-on: ubuntu-20.04 + static_frontend_code_checks: + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v3 with: - fetch-depth: 5 \ No newline at end of file + fetch-depth: 5 + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install frontend dependencies + run: corepack enable && yarn install --immutable + + - name: Lint frontend code and check formatting + run: yarn run check-frontend + + - name: Typecheck frontend code + run: yarn typecheck + + - name: Check for cyclic dependencies in frontend + run: yarn check-cyclic-dependencies + + + build_test_deploy: + runs-on: ubuntu-24.04 + env: + # The hostname used to communicate with the PostgreSQL service container + # POSTGRES_HOST: localhost + # POSTGRES_PORT: 5432 + PGPASSWORD: secret_password + services: + postgres: + image: postgres:10-alpine + env: + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 2s + --health-timeout 5s + --health-retries 30 + ports: + - 5432:5432 + steps: + + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 5 + + - name: "Custom environment variables" + run: | + if [[ ${{ github.ref_type }} == "branch" ]]; then + NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g') + echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV + DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}" + echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV + else + echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV + echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV + fi + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install frontend dependencies + run: corepack enable && yarn install --immutable + + - name: Build webknossos (webpack) + run: yarn build + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '21' + cache: 'sbt' + - name: Set up sbt + uses: sbt/setup-sbt@v1 + + - name: Install OS dependencies / libs + run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev + + # build-essential \ + # cmake \ + + # - name: Prepare dependency folders + # run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier + + # - name: Assert unique evolution numbers + # run: node tools/postgres/dbtool.js assert-unique-evolution-numbers + + # - name: Assert schema.sql and evolutions are equal + # run: | + # docker compose up -d postgres + # sleep 3 + # docker compose run compile tools/postgres/dbtool.js check-evolutions-schema + + + - name: Build webknossos (sbt) + run: | + if [ "${{ github.ref_name }}" == "master" ]; then + sbt -no-colors clean compile stage + else + sbt -no-colors -DfailOnWarning compile stage + fi + env: + POSTGRES_URL: jdbc:postgresql://localhost:5432/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + + - name: Build webknossos-datastore (sbt) + run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage + + - name: Build webknossos-tracingstore (sbt) + run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + + - name: Lint backend code and check formatting + run: .github/not-on-master.sh sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" + + - name: Run backend tests + run: .github/not-on-master.sh sbt -v "testOnly backend.*" + + - name: Checksum App Dirs + run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt + + - name: Set up Docker + uses: docker/setup-buildx-action@v2 + + - name: Build webknossos docker image + run: | + docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true + DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos + env: + POSTGRES_URL: jdbc:postgresql://localhost:5432/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + + - name: Build webknossos-datastore docker image + run: docker compose build --pull webknossos-datastore + + - name: Build webknossos-tracingstore docker image + run: docker compose build --pull webknossos-tracingstore + + # Enable tmate debugging of manually-triggered workflows if the input option was provided + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + + - name: Start webknossos, datastore, and tracingstore + run: | + mkdir -p fossildb/data && sudo chmod -R "0777" fossildb/data + mkdir -p fossildb/backup && sudo chmod -R "0777" fossildb/backup + docker compose up -d webknossos + docker compose up -d webknossos-datastore + docker compose up -d webknossos-tracingstore + + - name: Run webknossos smoke test + uses: ./.github/actions/url-health-check + with: + url: http://localhost:9000/api/health + + - name: Run webknossos-datastore smoke test + uses: ./.github/actions/url-health-check + with: + url: http://localhost:9090/data/health + + - name: Run webknossos-tracingstore smoke test + uses: ./.github/actions/url-health-check + with: + url: http://localhost:9050/tracings/health + + - name: Run end-to-end tests + uses: nick-invision/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + command: .github/not-on-master.sh yarn run test-e2e + env: + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + + - name: Stop webknossos, datastore, and tracingstore + run: docker compose down --volumes --remove-orphans + + - name: Push docker images + uses: nick-invision/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + docker compose push webknossos + docker compose push webknossos-datastore + docker compose push webknossos-tracingstore + + if [[ ${{ github.ref_type }} == "branch" ]]; then + docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + fi + docker logout + + - name: Report coverage + run: .github/not-on-master.sh yarn coverage || true + + # - name: Send Slack notification (master only) + # run: .circleci/slack-notification.sh diff --git a/Dockerfile b/Dockerfile index 2a553862cc5..d3e12dc63c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \ RUN mkdir -p /webknossos WORKDIR /webknossos +# Copy compiled Scala output from a previous build step, e.g. output of the Docker-dev image COPY target/universal/stage . RUN addgroup --system --gid 999 webknossos \ diff --git a/docker-compose.yml b/docker-compose.yml index 8d811af7dfe..84448787865 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "2.2" - services: # Production webknossos: @@ -105,8 +103,8 @@ services: - POSTGRES_USER=webknossos_user - POSTGRES_PASSWORD=secret_password - HOME=/root - - CIRCLE_TAG=${CIRCLE_TAG} - - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM} + - CI_TAG=${GITHUB_REF} + - CI_BUILD_NUM=${GITHUB_JOB} - NODE_OPTIONS=--max_old_space_size=2048 # This will be picked up by ava so that the tests fail if a snapshot doesn't exist during CI, see https://github.com/avajs/ava/issues/1585 - CI diff --git a/project/BuildInfoSettings.scala b/project/BuildInfoSettings.scala index a16d0223fe3..229e55b5754 100644 --- a/project/BuildInfoSettings.scala +++ b/project/BuildInfoSettings.scala @@ -13,8 +13,8 @@ object BuildInfoSettings { } val ciBuild: String = - if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else "" - val ciTag: String = if (System.getenv().containsKey("CIRCLE_TAG")) System.getenv().get("CIRCLE_TAG") else "" + if (System.getenv().containsKey("CI_BUILD_NUM")) System.getenv().get("CI_BUILD_NUM") else "" + val ciTag: String = if (System.getenv().containsKey("CI_TAG")) System.getenv().get("CI_TAG") else "" def commitHash: String = getStdoutFromCommand("git rev-parse HEAD", "") def commitDate: String = getStdoutFromCommand("git log -1 --format=%cd ", "") diff --git a/yarn.lock b/yarn.lock index a331738acb4..d6dbe4271e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8109,7 +8109,7 @@ __metadata: languageName: node linkType: hard -"lodash-es@npm:^4.17.21, lodash-es@npm:^4.2.1": +"lodash-es@npm:^4.17.21": version: 4.17.21 resolution: "lodash-es@npm:4.17.21" checksum: 10c0/fb407355f7e6cd523a9383e76e6b455321f0f153a6c9625e21a8827d10c54c2a2341bd2ae8d034358b60e07325e1330c14c224ff582d04612a46a4f0479ff2f2 @@ -8144,7 +8144,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21, lodash@npm:^4.2.1": +"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c @@ -11587,14 +11587,11 @@ __metadata: linkType: hard "redux@npm:^4.2.0": - version: 3.7.2 - resolution: "redux@npm:3.7.2" + version: 4.2.1 + resolution: "redux@npm:4.2.1" dependencies: - lodash: "npm:^4.2.1" - lodash-es: "npm:^4.2.1" - loose-envify: "npm:^1.1.0" - symbol-observable: "npm:^1.0.3" - checksum: 10c0/544456f95734de33326637b370894addb57d9de2524edf36a20e4a326d0a36a0e223979d027545c5aa8a8d7a2859363981f63d1146401b72df0d16f373dd09cb + "@babel/runtime": "npm:^7.9.2" + checksum: 10c0/136d98b3d5dbed1cd6279c8c18a6a74c416db98b8a432a46836bdd668475de6279a2d4fd9d1363f63904e00f0678a8a3e7fa532c897163340baf1e71bb42c742 languageName: node linkType: hard @@ -12835,13 +12832,6 @@ __metadata: languageName: node linkType: hard -"symbol-observable@npm:^1.0.3": - version: 1.2.0 - resolution: "symbol-observable@npm:1.2.0" - checksum: 10c0/009fee50798ef80ed4b8195048288f108b03de162db07493f2e1fd993b33fafa72d659e832b584da5a2427daa78e5a738fb2a9ab027ee9454252e0bedbcd1fdc - languageName: node - linkType: hard - "symbol-tree@npm:^3.2.4": version: 3.2.4 resolution: "symbol-tree@npm:3.2.4"