ci: run integration tests through actions /w runner set

.github/workflows/integration.yml

@@ -0,0 +1,195 @@
+---
+name: Integration Tests
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+      - main
+  pull_request:
+
+# Cancel in-progress jobs if a new commit is pushed to the same branch
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build Validation
+    runs-on: ubuntu-latest
+    container:
+      image: goalert/build-env:go1.24.2
+      volumes:
+        - /var/cache/arc:/var/cache/arc
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set git safe directory
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
+      - uses: actions/cache@v4
+        with:
+          key: cache-${{ hashFiles('go.mod','go.sum','package.json','bun.lock','*.version')}}
+          path: |
+            /go/pkg/mod
+            $HOME/.cache/goalert-gettool
+            $HOME/.cache/Cypress
+            $HOME/.cache/ms-playwright
+            $HOME/.bun/install/cache
+      - name: Build all binaries
+        run: make cache
+
+  check:
+    name: make check
+    # You need to use the INSTALLATION_NAME from the previous step
+    runs-on: ubuntu-latest
+    needs: build
+    container:
+      image: goalert/build-env:go1.24.2
+      volumes:
+        - /var/cache/arc:/var/cache/arc
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set git safe directory
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
+      - uses: actions/cache@v4
+        with:
+          key: cache-${{ hashFiles('go.mod','go.sum','package.json','bun.lock','*.version')}}
+          path: |
+            /go/pkg/mod
+            $HOME/.cache/goalert-gettool
+            $HOME/.cache/Cypress
+            $HOME/.cache/ms-playwright
+            $HOME/.bun/install/cache
+
+      # Run repo & linting checks
+      - name: Run Repo & Linting Checks
+        run: make check self-test
+
+  unit:
+    name: Unit Tests
+    # You need to use the INSTALLATION_NAME from the previous step
+    runs-on: ubuntu-latest
+    needs: build
+    container:
+      image: goalert/build-env:go1.24.2
+      volumes:
+        - /var/cache/arc:/var/cache/arc
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set git safe directory
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+      - uses: actions/cache@v4
+        with:
+          key: cache-${{ hashFiles('go.mod','go.sum','package.json','bun.lock','*.version')}}
+          path: |
+            /go/pkg/mod
+            $HOME/.cache/goalert-gettool
+            $HOME/.cache/Cypress
+            $HOME/.cache/ms-playwright
+            $HOME/.bun/install/cache
+
+      # Run repo & linting checks
+      - name: Run Tests
+        run: make test-unit test-components
+
+  # smoke:
+  #   name: Smoke Tests (behavioral tests)
+  #   # You need to use the INSTALLATION_NAME from the previous step
+  #   runs-on: arc-runner-set
+  #   needs: build
+  #   strategy:
+  #     matrix:
+  #       postgres_version: [13, 17]
+  #   container:
+  #     image: goalert/build-env:go1.24.2
+  #     volumes:
+  #       - /var/cache/arc:/var/cache/arc
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set git safe directory
+  #       run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+  #     - name: Restore cache
+  #       run: bash ./devtools/scripts/ci-cache.sh restore
+  #       env: { CI: '1' }
+
+  #     # Run repo & linting checks
+  #     - name: Run Smoke Tests
+  #       run: start_postgres ${{ matrix.postgres_version }} make smoketest
+
+  # playwright:
+  #   name: Playwright Tests
+  #   # You need to use the INSTALLATION_NAME from the previous step
+  #   runs-on: arc-runner-set
+  #   needs: build
+  #   strategy:
+  #     matrix:
+  #       postgres_version: [13, 17]
+  #   container:
+  #     image: goalert/build-env:go1.24.2
+  #     volumes:
+  #       - /var/cache/arc:/var/cache/arc
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set git safe directory
+  #       run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+  #     - name: Restore cache
+  #       run: bash ./devtools/scripts/ci-cache.sh restore
+  #       env: { CI: '1' }
+
+  #     # Run repo & linting checks
+  #     - name: Run Smoke Tests
+  #       run: bash -c 'start_postgres ${{ matrix.postgres_version }} make playwright-run'
+
+  # cy-wide:
+  #   name: Cypess Widescreen Tests
+  #   # You need to use the INSTALLATION_NAME from the previous step
+  #   runs-on: arc-runner-set
+  #   needs: build
+  #   strategy:
+  #     matrix:
+  #       postgres_version: [13, 17]
+  #   container:
+  #     image: goalert/build-env:go1.24.2
+  #     volumes:
+  #       - /var/cache/arc:/var/cache/arc
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set git safe directory
+  #       run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+  #     - name: Restore cache
+  #       run: bash ./devtools/scripts/ci-cache.sh restore
+  #       env: { CI: '1' }
+
+  #     # Run repo & linting checks
+  #     - name: Run Smoke Tests
+  #       run: bash -c 'start_postgres ${{ matrix.postgres_version }} make cy-wide-prod-run'
+
+  # cy-mobile:
+  #   name: Cypess Mobile Tests
+  #   # You need to use the INSTALLATION_NAME from the previous step
+  #   runs-on: arc-runner-set
+  #   needs: build
+  #   strategy:
+  #     matrix:
+  #       postgres_version: [13, 17]
+  #   container:
+  #     image: goalert/build-env:go1.24.2
+  #     volumes:
+  #       - /var/cache/arc:/var/cache/arc
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set git safe directory
+  #       run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+  #     - name: Restore cache
+  #       run: bash ./devtools/scripts/ci-cache.sh restore
+  #       env: { CI: '1' }
+
+  #     # Run repo & linting checks
+  #     - name: Run Smoke Tests
+  #       run: bash -c 'start_postgres ${{ matrix.postgres_version }} make cy-mobile-prod-run'

Makefile

@@ -97,6 +97,25 @@ $(BIN_DIR)/tools/mailpit: mailpit.version
 $(BIN_DIR)/tools/bun: bun.version
 	go tool gettool -t bun -v $(shell cat bun.version) -o $@
 
+.PHONY: setup-cypress
+setup-cypress: $(BIN_DIR)/tools/bun
+	bun x cypress install
+
+.PHONY: setup-playwright
+setup-playwright: $(BIN_DIR)/tools/bun
+	bun x playwright install
+
+.PHONY: deps
+deps: setup-cypress setup-playwright ## Install all dependencies
+	go get tool ./...
+
+.PHONY: cache
+cache: deps
+	$(MAKE) -j4 binaries
+
+.PHONY: binaries
+binaries: bin/goalert bin/goalert.cover bin/goalert-linux-amd64.tgz bin/goalert-linux-arm.tgz bin/goalert-linux-arm64.tgz bin/goalert-darwin-amd64.tgz bin/goalert-windows-amd64.zip ## Prime the build cache by building all binaries
+
 bun.lock: $(BIN_DIR)/tools/bun
 	$(BIN_DIR)/tools/bun install
 	touch "$@"
@@ -226,6 +245,10 @@ generate: $(NODE_DEPS) pkg/sysapi/sysapi.pb.go pkg/sysapi/sysapi_grpc.pb.go
 	$(SQLC) generate
 	go generate ./...
 
+.PHONY: self-test
+self-test:
+	$(MAKE) bin/goalert BUNDLE=1
+	./bin/goalert self-test --offline
 
 test-all: test-unit test-components test-smoke test-integration
 test-integration: playwright-run cy-wide-prod-run cy-mobile-prod-run
@@ -321,6 +344,7 @@ resetdb: config.json.bak ## Recreate the database leaving it empty (no migration
 	go tool resetdb --no-migrate
 
 clean: ## Clean up build artifacts
+	chmod +w -f -R bin || true
 	rm -rf bin node_modules web/src/node_modules .pnp.cjs .pnp.loader.mjs web/src/build/static .yarn storybook-static
 
 new-migration:

bun.version

@@ -1 +1 @@
-1.2.5
+1.2.9

devtools/ci/dockerfiles/build-env/Dockerfile

@@ -4,6 +4,7 @@ RUN apt-get update && apt-get install -y \
     apt-transport-https \
     ca-certificates \
     curl \
+    unzip \
     gnupg-agent \
     software-properties-common \
     build-essential \
@@ -26,7 +27,7 @@ RUN \
     rm -rf /var/lib/apt/lists/*
 
 # Postgres
-ENV PGDATA=/var/lib/postgresql/data PGUSER=postgres DB_URL=postgresql://postgres@?client_encoding=UTF8
+ENV PGDATA=/var/lib/postgresql/data PGUSER=postgres DB_URL=postgresql://postgres@?client_encoding=UTF8&sslmode=disable
 RUN mkdir -p ${PGDATA} /run/postgresql /var/log/postgresql &&\
     chown postgres ${PGDATA} /run/postgresql /var/log/postgresql &&\
     su postgres -c "/usr/lib/postgresql/13/bin/initdb $PGDATA" &&\

devtools/gettool/fetchfile.go

@@ -1,14 +1,65 @@
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 )
 
+var errMiss = fmt.Errorf("file not found in cache")
+
+func hash256(in string) string {
+	sum := sha256.Sum256([]byte(in))
+	return hex.EncodeToString(sum[:])
+}
+
+func getCacheFile(url string) (*os.File, int64, error) {
+	if cacheDir == "" {
+		return nil, 0, errMiss
+	}
+
+	file := filepath.Join(cacheDir, hash256(url)+".data")
+	fd, err := os.Open(file)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, 0, errMiss
+	}
+	info, err := fd.Stat()
+	if err != nil {
+		return nil, 0, fmt.Errorf("stat: %w", err)
+	}
+	return fd, info.Size(), nil
+}
+
+func mkCacheFile(url string) (*os.File, error) {
+	file := filepath.Join(cacheDir, hash256(url)+".data.tmp")
+	err := os.MkdirAll(cacheDir, 0o755)
+	if err != nil {
+		return nil, fmt.Errorf("create cache dir: %w", err)
+	}
+
+	fd, err := os.Create(file)
+	if err != nil {
+		return nil, fmt.Errorf("create cache file: %w", err)
+	}
+	return fd, nil
+}
+
 // fetchFile will download and open a file with the contents of `url`.
 func fetchFile(url string) (*os.File, int64, error) {
+	file, size, err := getCacheFile(url)
+	if err == nil {
+		return file, size, nil
+	}
+	if err != errMiss {
+		return nil, 0, fmt.Errorf("getCacheFile: %w", err)
+	}
+
 	resp, err := http.Get(url)
 	if err != nil {
 		return nil, 0, fmt.Errorf("fetch: %w", err)
@@ -18,16 +69,37 @@ func fetchFile(url string) (*os.File, int64, error) {
 		return nil, 0, fmt.Errorf("non-200 response: %s", resp.Status)
 	}
 
-	fd, err := os.CreateTemp("", "*.zip")
-	if err != nil {
-		return nil, 0, fmt.Errorf("create temp file: %w", err)
+	var fd *os.File
+	var cacheFailed bool
+	if cacheDir != "" {
+		fd, err = mkCacheFile(url)
+		if err != nil {
+			log.Println("failed to update cache:", err)
+		}
+	}
+
+	if fd == nil {
+		fd, err = os.CreateTemp("", "*.zip")
+		if err != nil {
+			return nil, 0, fmt.Errorf("create temp file: %w", err)
+		}
+		cacheFailed = true
 	}
 
 	n, err := io.Copy(fd, resp.Body)
 	if err != nil {
 		fd.Close()
 		return nil, 0, fmt.Errorf("download file '%s': %w", url, err)
 	}
+
+	if !cacheFailed {
+		err = os.Rename(fd.Name(), filepath.Join(cacheDir, hash256(url)+".data"))
+		if err != nil {
+			fd.Close()
+			return nil, 0, fmt.Errorf("rename cache file: %w", err)
+		}
+	}
+
 	_, err = fd.Seek(0, 0)
 	if err != nil {
 		fd.Close()

devtools/gettool/run.go → devtools/gettool/main.go

@@ -3,15 +3,30 @@ package main
 import (
 	"flag"
 	"log"
+	"os"
+	"path/filepath"
 )
 
+var cacheDir string
+
 func main() {
 	tool := flag.String("t", "", "Tool to fetch.")
 	version := flag.String("v", "", "Version of the tool to fetch.")
 	output := flag.String("o", "", "Output file/dir.")
+	flag.StringVar(&cacheDir, "c", os.Getenv("GETTOOL_CACHE"), "Cache dir.")
 	flag.Parse()
 	log.SetFlags(log.Lshortfile)
 
+	if cacheDir != "" {
+		// do nothing, already set
+	} else if base := os.Getenv("XDG_CACHE_HOME"); base != "" {
+		// use the XDG_CACHE_HOME
+		cacheDir = filepath.Join(base, "goalert-gettool")
+	} else if home := os.Getenv("HOME"); home != "" {
+		// use the HOME dir
+		cacheDir = filepath.Join(home, ".cache", "goalert-gettool")
+	}
+
 	if *tool == "" {
 		log.Fatal("-t flag is required")
 	}