build.yaml

repoPrefix: /repo

alwaysRunSteps:
  - merge_code

steps:
  - kind: buildImage2
    name: git_make_bash_image
    publishAs: git-make-bash
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ global.docker_root_image }}
        FROM $BASE_IMAGE
        RUN apt-get update && apt-get install -y git make bash
  - kind: runImage
    name: merge_code
    image:
      valueFrom: git_make_bash_image.image
    script: |
      set -ex
      cd /io
      mkdir -p repo
      cd repo
      {{ code.checkout_script }}
      make -C hail python-version-info env/SPARK_VERSION HAIL_BUILD_MODE=Release MILLOPTS=--no-daemon
      {% if code.get('pip_version') %}
      _make_pip_version=$(cat hail/env/HAIL_PIP_VERSION)
      [[ "$_make_pip_version" == "{{ code.get('pip_version') }}" ]] || { echo "HAIL_PIP_VERSION mismatch: CI parsed {{ code.get('pip_version') }} from Makefile but make produced $_make_pip_version"; exit 1; }
      {% endif %}
    outputs:
      - from: /io/repo/
        to: /repo
    dependsOn:
      - git_make_bash_image
  - kind: createNamespace
    name: default_ns
    namespaceName: default
    secrets:
      - name: auth-oauth2-client-secret
      - name: hail-ci-0-1-github-oauth-token
      - name: testns-test-gsa-key
      - name: testns-test-dev-gsa-key
      - name: testns-auth-gsa-key
      - name: testns-batch-gsa-key
      - name: testns-ci-gsa-key
      - name: testns-grafana-gsa-key
      - name: test-aws-key
        clouds:
          - gcp
      - name: test-azure-key
        clouds:
          - gcp
      - name: zulip-config
      - name: billing-monitor-gsa-key
        clouds:
          - gcp
      - name: hail-ci-0-1-service-account-key
      - name: test-dataproc-service-account-key
        clouds:
          - gcp
  - kind: runImage
    name: copy_third_party_images
    image: quay.io/skopeo/stable:v1.13.2
    script: |
      set -ex

      retry() {
          "$@" ||
              (sleep 2 && "$@") ||
              (sleep 5 && "$@");
      }

      REGISTRY={{ global.docker_prefix.split('/')[0] }}

      {% if global.cloud == "gcp" %}
      cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
      {% else %}
      echo "unknown cloud {{ global.cloud }}"
      exit 1
      {% endif %}

      cd /io/docker/third-party
      DOCKER_PREFIX={{ global.docker_prefix }} {% if global.dockerhub_prefix %}DOCKERHUB_PREFIX={{ global.dockerhub_prefix }} {% endif %}retry bash copy_images.sh
    inputs:
      - from: /repo/docker
        to: /io/docker
    scopes:
      - deploy
    dependsOn:
      - default_ns
      - merge_code
  - kind: buildImage2
    name: hail_ubuntu_image
    dockerFile: /io/repo/docker/hail-ubuntu/Dockerfile
    contextPath: /io/repo
    publishAs: hail-ubuntu
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/repo/docker/hail-ubuntu
      - from: /repo/gear/pinned-requirements.txt
        to: /io/repo/gear/pinned-requirements.txt
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: buildImage2
    name: hail_ubuntu_image_python_3_13
    dockerFile: /io/repo/docker/hail-ubuntu/Dockerfile
    contextPath: /io/repo
    publishAs: hail-ubuntu-python-3-13
    buildArgs:
      - name: PYTHON_VERSION
        value: "3.13"
      - name: SKIP_GCLOUD_PROFILER
        value: "true" # Python 3.13 does not support google-cloud-profiler yet
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/repo/docker/hail-ubuntu
      - from: /repo/gear/pinned-requirements.txt
        to: /io/repo/gear/pinned-requirements.txt
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: buildImage2
    name: hail_ubuntu_image_python_3_14
    dockerFile: /io/repo/docker/hail-ubuntu/Dockerfile
    contextPath: /io/repo
    publishAs: hail-ubuntu-python-3-14
    buildArgs:
      - name: PYTHON_VERSION
        value: "3.14"
      - name: SKIP_GCLOUD_PROFILER
        value: "true"
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/repo/docker/hail-ubuntu
      - from: /repo/gear/pinned-requirements.txt
        to: /io/repo/gear/pinned-requirements.txt
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: deploy
    name: deploy_batch_sa
    namespace:
      valueFrom: default_ns.name
    config: batch/service-account.yaml
    dependsOn:
      - default_ns
  - kind: deploy
    name: deploy_ci_agent
    namespace:
      valueFrom: default_ns.name
    config: ci/ci-agent.yaml
    dependsOn:
      - default_ns
  - kind: deploy
    name: deploy_test_batch_sa
    namespace:
      valueFrom: default_ns.name
    config: batch/test-sa.yaml
    dependsOn:
      - default_ns
  - kind: buildImage2
    name: base_image
    dockerFile: /io/repo/docker/Dockerfile.base
    contextPath: /io/repo
    publishAs: base
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/Dockerfile.base
        to: /io/repo/docker/Dockerfile.base
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: create_certs_image
    dockerFile: /io/tls/Dockerfile
    contextPath: /io/tls
    publishAs: create_certs_image
    inputs:
      - from: /repo/tls
        to: /io/tls
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: runImage
    name: create_ssl_config_hail_root
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: create_certs_image.image
    script: |

      {% if not deploy %}
      kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root \
        --template={% raw %}'{{index .data "hail-root-cert.pem"}}'{% endraw %} \
        | base64 --decode \
        | openssl x509 -checkend 0 -noout -in -

      if [ "$?" -ne 0 ]
      then
          kubectl delete secret -n {{ default_ns.name }} ssl-config-hail-root
      fi
      {% endif %}

      openssl req -new -x509 -subj /CN=hail-root -nodes -newkey rsa:4096 -keyout hail-root-key.pem -out hail-root-cert.pem
      until kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root
      do
          kubectl create secret generic -n {{ default_ns.name }} ssl-config-hail-root \
                  --from-file=hail-root-key.pem \
                  --from-file=hail-root-cert.pem
      done
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - create_certs_image
  - kind: runImage
    name: create_certs
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: create_certs_image.image
    script: |
      set -ex
      python3 create_certs.py \
              {{ default_ns.name }} \
              config.yaml \
              /ssl-config-hail-root/hail-root-key.pem \
              /ssl-config-hail-root/hail-root-cert.pem
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    secrets:
      - name: ssl-config-hail-root
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config-hail-root
    dependsOn:
      - default_ns
      - create_certs_image
      - create_ssl_config_hail_root
  - kind: buildImage2
    name: ci_utils_image
    dockerFile: /io/repo/ci/Dockerfile.ci-utils
    contextPath: /io/repo
    publishAs: ci-utils
    resources:
      storage: 20Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: create_test_database_server_config
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: create_certs_image.image
    script: |
      # Check if secret exists and whether its certificate has expired
      # First, set +ex to (e) allow the command to fail, (x) avoid logging the password
      set +ex
      echo 'Running: SECRET_JSON=$(kubectl get secret database-server-config -n {{ default_ns.name }} -o json)'
      SECRET_JSON=$(kubectl get secret database-server-config -n {{ default_ns.name }} -o json)
      SECRET_EXISTS_EXIT_CODE=$?
      echo "SECRET_EXISTS_EXIT_CODE=$SECRET_EXISTS_EXIT_CODE"
      set -ex

      if [ "$SECRET_EXISTS_EXIT_CODE" -eq 0 ]; then
          echo "Secret exists, checking certificate expiration..."

          set +x
          # set -e will abort the script so the error is surfaced rather than silently regenerating.
          CERT_FILE=$(mktemp)
          echo "$SECRET_JSON" | jq -r '.data["server-ca.pem"]' | base64 -d > "$CERT_FILE"
          set -x
          echo "Certificate expiration: $(openssl x509 -enddate -noout -in "$CERT_FILE")"

          # openssl x509 -checkend 0: exit code 0 = not expired, 1 = expired
          if openssl x509 -checkend 0 -noout -in "$CERT_FILE"; then
              echo "Certificate is still valid, skipping regeneration"
          else
              echo "Certificate has expired, regenerating certificates while preserving password..."
              echo "Running: EXISTING_PASSWORD=\$(echo \"\$SECRET_JSON\" | jq -r '.data[\"db-root-password\"]' | base64 -d)"
              set +x
              EXISTING_PASSWORD=$(echo "$SECRET_JSON" | jq -r '.data["db-root-password"]' | base64 -d)
              echo "Running: NAMESPACE={{ default_ns.name }} KEEP_PASSWORD=\"\$EXISTING_PASSWORD\" bash /create_test_db_config.sh"
              NAMESPACE={{ default_ns.name }} KEEP_PASSWORD="$EXISTING_PASSWORD" bash /create_test_db_config.sh
              set -x
          fi
          rm -f "$CERT_FILE"
      else
          echo "Secret does not exist, creating new one..."
          NAMESPACE={{ default_ns.name }} bash /create_test_db_config.sh
      fi
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - dev
      - test
    dependsOn:
      - default_ns
      - create_certs_image
  - kind: deploy
    name: deploy_test_db
    namespace:
      valueFrom: default_ns.name
    config: docker/mysql/db.yaml
    wait:
      - kind: Service
        name: db
        for: alive
        resource_type: statefulset
    scopes:
      - dev
      - test
    dependsOn:
      - default_ns
      - create_test_database_server_config
  - kind: buildImage2
    name: admin_pod_image
    dockerFile: /io/repo/admin-pod/Dockerfile
    contextPath: /io/repo
    publishAs: admin-pod
    inputs:
      - from: /repo/admin-pod
        to: /io/repo/admin-pod
    dependsOn:
      - hail_ubuntu_image
  - kind: runImage
    name: build_ui
    image: node:24-slim@sha256:d45d78e7929b46875bbd4e29bea672d5bc48186c6c3588306521c815e78352d6
    script: |
      set -ex
      cd /io/repo/services/ui
      npm ci
      npm run build
    inputs:
      - from: /repo/services/ui
        to: /io/repo/services/ui
    outputs:
      - from: /io/repo/services/ui/dist/auth
        to: /compiled-js/auth
      - from: /io/repo/services/ui/dist/batch
        to: /compiled-js/batch
      - from: /io/repo/services/ui/dist/batch_driver
        to: /compiled-js/batch_driver
      - from: /io/repo/services/ui/dist/ci
        to: /compiled-js/ci
      - from: /io/repo/services/ui/dist/monitoring
        to: /compiled-js/monitoring
      - from: /io/repo/services/ui/dist/shared
        to: /compiled-js/shared
    dependsOn:
      - merge_code
  - kind: buildImage2
    name: auth_image
    dockerFile: /io/repo/auth/Dockerfile
    contextPath: /io/repo
    publishAs: auth
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/auth
        to: /io/repo/auth
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /compiled-js/auth
        to: /io/repo/auth/auth/static/compiled-js
      - from: /compiled-js/shared
        to: /io/repo/auth/auth/static/compiled-js
    dependsOn:
      - hail_ubuntu_image
      - merge_code
      - build_ui
  - kind: runImage
    name: delete_auth_tables
    image:
      valueFrom: auth_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} auth
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - auth_image
      - merge_code
      - create_test_database_server_config
  - kind: createDatabase2
    name: auth_database
    databaseName: auth
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/000-initial.sql
      - name: copy-paste-tokens
        script: /io/sql/001-copy-paste-tokens.sql
      - name: drop-bucket
        script: /io/sql/002-drop-bucket.sql
      - name: add-trial-billing-project
        script: /io/sql/003-add-trial-billing-project.sql
      - name: add-roles
        script: /io/sql/004-add-roles.sql
      - name: rename-gsa-identity
        script: /io/sql/005-rename-gsa-identity.sql
      - name: support-azure-oauth
        script: /io/sql/006-support-azure-oauth.sql
      - name: change-username-collation
        script: /io/sql/007-change-username-collation.sql
      - name: add-hail-identity-uid
        script: /io/sql/008-add-hail-identity-uid.sql
      - name: add-users-last-active
        script: /io/sql/009-add-users-last-active.sql
      - name: refactor-roles
        script: /io/sql/010-refactor-roles.sql
      - name: refactor-system-permissions
        script: /io/sql/011-refactor-system-permissions.sql
    inputs:
      - from: /repo/auth/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: auth
    dependsOn:
      - default_ns
      - merge_code
      - delete_auth_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: runImage
    name: create_deploy_config
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex

      # k8s deploy config
      {% if default_ns.name == "default" %}
      cat > deploy-config.json <<EOF
      {"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"{{ global.domain }}"}
      EOF
      {% else %}
      cat > deploy-config.json <<EOF
      {"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"internal.{{ global.domain }}","base_path":"/{{ default_ns.name }}"}
      EOF
      {% endif %}
      kubectl -n {{ default_ns.name }} create secret generic deploy-config \
              --from-file=./deploy-config.json \
              --save-config --dry-run=client -o yaml \
          | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - ci_utils_image
  - kind: deploy
    name: deploy_admin_pod
    namespace:
      valueFrom: default_ns.name
    config: admin-pod/admin-pod.yaml
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - admin_pod_image
      - create_deploy_config
      - create_certs
  - kind: runImage
    name: create_session_key
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      # create session keys
      N=$(kubectl -n {{ default_ns.name }} get secret --ignore-not-found=true --no-headers session-secret-key | wc -l | tr -d '[:space:]')
      if [[ $N != 0 ]]; then
        exit
      fi
      mkdir /session-secret-key
      cat > generate-session-key.py <<EOF
      import base64
      from cryptography import fernet
      with open('/session-secret-key/session-secret-key', 'wb') as f:
          f.write(base64.urlsafe_b64decode(fernet.Fernet.generate_key()))
      EOF
      python3 generate-session-key.py
      kubectl -n {{ default_ns.name }} create secret generic session-secret-key --from-file=/session-secret-key/session-secret-key
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - ci_utils_image
  - kind: runImage
    name: create_test_gsa_keys
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      kubectl -n {{ default_ns.name }} get -o json secret testns-test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-test-dev-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-dev-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-auth-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "auth-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-batch-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "batch-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-ci-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "ci-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-grafana-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "grafana-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
    scopes:
      - test
      - dev
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - ci_utils_image
  - kind: deploy
    name: deploy_auth_driver_service_account
    namespace:
      valueFrom: default_ns.name
    config: auth/auth-driver-service-account.yaml
    dependsOn:
      - default_ns
  - kind: runImage
    name: create_accounts
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: auth_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_SCOPE={{ scope }}
      export GOOGLE_APPLICATION_CREDENTIALS=/auth-gsa-key/key.json

      python3 /io/bootstrap_create_accounts.py
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    secrets:
      - name:
          valueFrom: auth_database.user_secret_name
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /database-server-config
      - name: auth-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /auth-gsa-key
      - name: global-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /global-config
    inputs:
      - from: /repo/ci/bootstrap_create_accounts.py
        to: /io/bootstrap_create_accounts.py
    dependsOn:
      - default_ns
      - deploy_test_batch_sa
      - auth_database
      - auth_image
      - create_deploy_config
      - deploy_auth_driver_service_account
      - create_test_gsa_keys
      - create_test_database_server_config
  - kind: buildImage2
    name: hailgenetics_vep_grch37_85_image
    dockerFile: /io/repo/docker/hailgenetics/vep/grch37/85/Dockerfile
    contextPath: /io/repo/docker/vep/
    publishAs: hailgenetics/vep-grch37-85
    inputs:
      - from: /repo/docker/hailgenetics/vep
        to: /io/repo/docker/hailgenetics/vep
      - from: /repo/docker/vep
        to: /io/repo/docker/vep
    dependsOn:
      - merge_code
  - kind: buildImage2
    name: hailgenetics_vep_grch38_95_image
    dockerFile: /io/repo/docker/hailgenetics/vep/grch38/95/Dockerfile
    contextPath: /io/repo/docker/vep/
    publishAs: hailgenetics/vep-grch38-95
    inputs:
      - from: /repo/docker/hailgenetics/vep
        to: /io/repo/docker/hailgenetics/vep
      - from: /repo/docker/vep
        to: /io/repo/docker/vep
    dependsOn:
      - merge_code
  - kind: buildImage2
    name: monitoring_image
    dockerFile: /io/repo/monitoring/Dockerfile
    contextPath: /io/repo
    publishAs: monitoring
    inputs:
      - from: /repo/monitoring
        to: /io/repo/monitoring
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /compiled-js/monitoring
        to: /io/repo/monitoring/monitoring/static/compiled-js
      - from: /compiled-js/shared
        to: /io/repo/monitoring/monitoring/static/compiled-js
    dependsOn:
      - hail_ubuntu_image
      - merge_code
      - build_ui
  - kind: runImage
    name: check_ui
    scopes:
      - test
      - dev
    image: node:24-slim@sha256:d45d78e7929b46875bbd4e29bea672d5bc48186c6c3588306521c815e78352d6
    script: |
      set -ex
      cd /io/repo/services/ui
      export FORCE_COLOR=1
      npm ci
      npm test
      npm run check
    inputs:
      - from: /repo/services/ui
        to: /io/repo/services/ui
    dependsOn:
      - merge_code
  - kind: buildImage2
    name: batch_image
    dockerFile: /io/repo/batch/Dockerfile
    contextPath: /io/repo
    publishAs: batch
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/batch
        to: /io/repo/batch
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /compiled-js/batch
        to: /io/repo/batch/batch/front_end/static/compiled-js
      - from: /compiled-js/shared
        to: /io/repo/batch/batch/front_end/static/compiled-js
      - from: /compiled-js/batch_driver
        to: /io/repo/batch/batch/driver/static/compiled-js
      - from: /compiled-js/shared
        to: /io/repo/batch/batch/driver/static/compiled-js
    dependsOn:
      - hail_ubuntu_image
      - merge_code
      - build_ui
      - check_ui
  - kind: buildImage2
    name: hailgenetics_hailtop_image
    dockerFile: /io/repo/docker/hailgenetics/hailtop/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hailtop
    inputs:
      - from: /repo/docker/hailgenetics/hailtop
        to: /io/repo/docker/hailgenetics/hailtop
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
    dependsOn:
      - merge_code
      - hail_ubuntu_image
  - kind: buildImage2
    name: ci_image
    dockerFile: /io/repo/ci/Dockerfile
    contextPath: /io/repo
    publishAs: ci
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /compiled-js/ci
        to: /io/repo/ci/ci/static/compiled-js
      - from: /compiled-js/shared
        to: /io/repo/ci/ci/static/compiled-js
    dependsOn:
      - hail_ubuntu_image
      - merge_code
      - build_ui
      - check_ui
  - kind: buildImage2
    name: hail_buildkit_image
    dockerFile: /io/repo/ci/buildkit/Dockerfile
    contextPath: /io/repo/ci
    publishAs: hail-buildkit
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: runImage
    name: download_mill_and_maven_deps
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail

      export HAIL_BUILD_MODE=Release
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      # force download of mill binary and coursier dependencies
      sh mill --no-daemon 'hail.__.{prepareOffline,scalaCompilerClasspath,javaHome}'
      tar -C /io -czf /io/mill-cache.tgz cache
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
    outputs:
      - from: /io/mill-cache.tgz
        to: /mill-cache.tgz
      - from: /io/repo/hail/out
        to: /derived/root/hail/out
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: build_hail_jar_and_wheel
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail

      tar -C /io -xzf /io/mill-cache.tgz

      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=Release
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      time retry sh mill --no-daemon --version

      # We've encountered the following sporadic error in CI between `mill`
      # finishing assembling the jar and `Make` copying the jar:
      #
      # cp -f out/assembly.dest/out.jar python/hail/backend/hail-all-spark.jar
      # cp: cannot stat 'out/assembly.dest/out.jar': No such file or directory
      #
      # This was likely from running mill in server mode; mill main exits before
      # the jar has finished being written.
      time retry make shadowJar
      if [ ! -f out/hail/assembly.dest/out.jar ]; then
          echo 'no out.jar found after mill assembly returned. going to sleep'
          sleep 5
      fi

      time retry make DEV_CLARIFIER=hail-ci-build wheel

      # Check wheel size is small enough for pypi (< 200 MiB)
      BYTES=$(du build/deploy/dist/hail-*-py3-none-any.whl | awk '{print $1}')
      (( BYTES < 209715200 )) || exit 1
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/.git
        to: /io/repo/.git
      - from: /repo/README.md
        to: /io/repo/README.md
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
      - from: /derived/root/hail/out
        to: /io/repo/hail/out
    outputs:
      - from: /io/repo/hail/out/hail/assembly.dest/out.jar
        to: /derived/release/hail/out/hail/assembly.dest/out.jar
      - from: /io/repo/hail/build/deploy/dist
        to: /derived/release/hail/build/deploy/dist
    dependsOn:
      - base_image
      - merge_code
      - download_mill_and_maven_deps
  - kind: runImage
    name: test_install_in_isolation_python3_12
    scopes:
      - test
      - dev
    image:
      valueFrom: base_image.image
    script: |
      set -ex
      cd /io/
      
      uv venv --python 3.12 --seed _venv
      source _venv/bin/activate
      
      pip install /io/wheel/hail-*-py3-none-any.whl
      cat << EOF | python3
      import hail as hl
      hl.utils.range_table(10)._force_count()
      EOF
    inputs:
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
    dependsOn:
      - base_image
      - build_hail_jar_and_wheel
  - kind: runImage
    name: upload_temporary_hailctl_artifacts
    image:
      valueFrom: ci_utils_image.image
    resources:
      memory: standard
      cpu: '1'
    script: |
      set -ex
      cd /io/repo/hail

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      make HAIL_BUILD_MODE=Release DEV_CLARIFIER=hail-ci-build upload-artifacts \
          -o $(ls build/deploy/dist/hail-*-py3-none-any.whl)
    dependsOn:
      - ci_utils_image
      - default_ns
      - merge_code
      - build_hail_jar_and_wheel
    inputs:
      - from: /repo
        to: /io/repo
      - from: /derived/release/hail/build/deploy/dist
        to: /io/repo/hail/build/deploy/dist
    secrets:
      - name: test-dataproc-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dataproc-service-account-key
    forceIfLabeled:
      - run-dataproc-tests
    onlyIfRelease: true
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
  - kind: runImage
    name: build_hail_debug_jar_and_wheel
    scopes:
      - test
      - dev
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail

      tar -C /io -xzf /io/mill-cache.tgz

      export MILLOPTS='--no-daemon'
      export HAIL_BUILD_MODE=CI
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      time retry sh mill --no-daemon --version

      # See `build_hail_jar_and_wheel`
      time retry make shadowJar
      if [ ! -f out/hail/assembly.dest/out.jar ]; then
          echo 'no out.jar found after mill assembly returned. going to sleep'
          sleep 5
      fi

      time retry make wheel
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/.git
        to: /io/repo/.git
      - from: /repo/README.md
        to: /io/repo/README.md
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
      - from: /derived/root/hail/out
        to: /io/repo/hail/out
    outputs:
      - from: /io/repo/hail/out
        to: /derived/debug/hail/out
      - from: /io/repo/hail/build/deploy/dist
        to: /derived/debug/hail/build/deploy/dist
    dependsOn:
      - base_image
      - merge_code
      - download_mill_and_maven_deps
  - kind: runImage
    name: build_debug_hail_test
    scopes:
      - test
      - dev
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail

      tar -C /io -xzf /io/mill-cache.tgz

      export HAIL_BUILD_MODE=CI
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      time sh mill --no-daemon hail.test.{compile,testForkGrouping}
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/.git
        to: /io/repo/.git
      - from: /derived/debug/hail/out
        to: /io/repo/hail/out
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
    outputs:
      - from: /io/repo/hail/out
        to: /derived/debug/hail/out
    dependsOn:
      - base_image
      - merge_code
      - build_hail_debug_jar_and_wheel
      - download_mill_and_maven_deps
  - kind: runImage
    name: build_hail_test_artifacts
    scopes:
      - test
      - dev
    image:
      valueFrom: base_image.image
    script: |
      set -ex
      cd /io/repo/hail

      time tar czf test.tar.gz -C python test pytest.ini
      time tar czf resources.tar.gz -C hail/test resources
      time tar czf data.tar.gz -C python/hail/docs data
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
    outputs:
      - from: /io/repo/hail/test.tar.gz
        to: /test.tar.gz
      - from: /io/repo/hail/resources.tar.gz
        to: /resources.tar.gz
      - from: /io/repo/hail/data.tar.gz
        to: /data.tar.gz
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: hail_run_image
    dockerFile: /io/repo/hail/Dockerfile.hail-run
    contextPath: /io/repo
    publishAs: hail-run
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/hail/Dockerfile.hail-run
        to: /io/repo/hail/Dockerfile.hail-run
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/repo/hail/python/dev/pinned-requirements.txt
      - from: /repo/docker/core-site.xml
        to: /io/repo/docker/core-site.xml
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: jvm_entryway_jar
    image:
      valueFrom: base_image.image
    script: |
      set -ex

      cd /io/batch/jvm-entryway
      time retry sh mill --no-daemon --version
      sh mill --no-daemon palantirformat --check
      sh mill --no-daemon assembly
    inputs:
      - from: /repo/batch
        to: /io/batch
    outputs:
      - from: /io/batch/jvm-entryway/out/assembly.dest/out.jar
        to: /jvm-entryway.jar
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: batch_worker_image
    dockerFile: /io/repo/batch/Dockerfile.worker
    contextPath: /io/repo
    publishAs: batch-worker
    inputs:
      - from: /repo/letsencrypt/subdomains.txt
        to: /io/repo/letsencrypt/subdomains.txt
      - from: /repo/docker
        to: /io/repo/docker
      - from: /repo/batch
        to: /io/repo/batch
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /jvm-entryway.jar
        to: /io/repo/batch/jvm-entryway/out/assembly.dest/out.jar
    dependsOn:
      - merge_code
      - hail_ubuntu_image
      - jvm_entryway_jar
  - kind: runImage
    name: upload_test_resources_to_blob_storage
    scopes:
      - test
      - dev
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      cd /io/repo/hail/
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      python3 -m hailtop.aiotools.copy 'null' '[
      {"from": "hail/test/resources",
       "to":   "{{ global.test_storage_uri }}/{{ token }}/test/resources"},
      {"from": "python/hail/docs/data",
       "to":   "{{ global.test_storage_uri }}/{{ token }}/doctest/data"}
      ]'
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    inputs:
      - from: /repo/hail/hail/test/resources
        to: /io/repo/hail/hail/test/resources
      - from: /repo/hail/python/hail/docs/data
        to: /io/repo/hail/python/hail/docs/data
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - create_test_gsa_keys
      - merge_code
  - kind: runImage
    name: test_hail_java
    scopes:
      - test
      - dev
    numSplits: 5
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail

      tar -C /io -xzf /io/mill-cache.tgz

      export HAIL_BUILD_MODE='CI'
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      sh mill --no-daemon -j1 hail.test.test-main --split $HAIL_RUN_IMAGE_SPLIT_INDEX --totalSplits 5
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/.git
        to: /io/repo/.git
      - from: /derived/debug/hail/out
        to: /io/repo/hail/out
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - create_test_gsa_keys
      - build_debug_hail_test
      - build_hail_test_artifacts
      - download_mill_and_maven_deps
  - kind: runImage
    name: test_hail_python
    scopes:
      - test
      - dev
    numSplits: 22
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      python3 -m pip install --no-dependencies /io/debug-wheel/hail-*-py3-none-any.whl

      # pyspark/conf/core-site.xml already points at /gsa-key/key.json
      mv /test-gsa-key/key.json /gsa-key/key.json

      export HAIL_QUERY_N_CORES=2
      export OMP_NUM_THREADS=2
      export OPENBLAS_NUM_THREADS=2
      export MKL_NUM_THREADS=2
      export VECLIB_MAXIMUM_THREADS=2
      export NUMEXPR_NUM_THREADS=2

      export HAIL_TEST_RESOURCES_DIR=$(realpath ./resources)
      export HAIL_DOCTEST_DATA_DIR=$(realpath ./data)
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=120 \
              test/hail
    inputs:
      - from: /derived/debug/hail/build/deploy/dist
        to: /io/debug-wheel
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - create_test_gsa_keys
      - build_hail_debug_jar_and_wheel
      - build_hail_test_artifacts
    clouds:
      - gcp
  - kind: runImage
    name: test_hailtop_python_fs
    scopes:
      - test
      - dev
    numSplits: 5
    image:
      valueFrom: hailgenetics_hailtop_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz

      python3 -m pip install -r test-requirements.txt

      export HAIL_TEST_GCS_BUCKET={{ global.hail_test_gcs_bucket }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_TEST_S3_BUCKET=hail-test-dy5rg
      export AWS_SHARED_CREDENTIALS_FILE=/test-aws-key/credentials

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=120 \
              test/hailtop/inter_cloud
    timeout: 1200
    inputs:
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/test-requirements.txt
      - from: /test.tar.gz
        to: /io/test.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-aws-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-aws-key
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - create_test_gsa_keys
      - build_hail_test_artifacts
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_unchecked_allocator
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      # pyspark/conf/core-site.xml already points at /gsa-key/key.json
      mv /test-gsa-key/key.json /gsa-key/key.json

      export HAIL_TEST_RESOURCES_DIR=$(realpath ./resources)
      export HAIL_DOCTEST_DATA_DIR=$(realpath ./data)
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              -m unchecked_allocator \
              --ignore=test/hailtop/batch/ \
              --ignore=test/hailtop/inter_cloud \
              --timeout=120 \
              test
    inputs:
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - build_hail_jar_and_wheel
      - build_hail_test_artifacts
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_local_backend
    scopes:
      - test
      - dev
    numSplits: 22
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      python3 -m pip install --no-dependencies /io/debug-wheel/hail-*-py3-none-any.whl
      mkdir -p /io/tmp

      # The test should use the test credentials, not CI's credentials
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_QUERY_N_CORES=2
      export OMP_NUM_THREADS=2
      export OPENBLAS_NUM_THREADS=2
      export MKL_NUM_THREADS=2
      export VECLIB_MAXIMUM_THREADS=2
      export NUMEXPR_NUM_THREADS=2

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_TEST_STORAGE_URI=/io/tmp/
      export HAIL_TEST_RESOURCES_DIR=$(realpath ./resources)
      export HAIL_DOCTEST_DATA_DIR=$(realpath ./data)
      export HAIL_LOCAL_BACKEND_HEAP_SIZE=3G

      hailctl config set query/backend local

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --ignore=test/hailtop/ \
              --timeout=120 \
              test
    timeout: 1800
    inputs:
      - from: /derived/debug/hail/build/deploy/dist
        to: /io/debug-wheel
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - create_test_gsa_keys
      - build_hail_debug_jar_and_wheel
      - build_hail_test_artifacts
  - kind: runImage
    name: test_python_docs
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      export HAIL_QUERY_N_CORES=2
      export OMP_NUM_THREADS=2
      export OPENBLAS_NUM_THREADS=2
      export MKL_NUM_THREADS=2
      export VECLIB_MAXIMUM_THREADS=2
      export NUMEXPR_NUM_THREADS=2

      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      export HAIL_LOCATION=$(dirname $(python3 -c 'import hail; print(hail.__file__)'))

      mv /io/docs $HAIL_LOCATION/docs

      # NB: do not use --timeout with doctest because there is no way to override it.
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --doctest-modules \
              --doctest-glob='*.rst' \
              --doctest-continue-on-failure \
              --ignore=$HAIL_LOCATION/test \
              --ignore=$HAIL_LOCATION/docs/conf.py \
              --ignore=$HAIL_LOCATION/docs/doctest_write_data.py \
              --rootdir $HAIL_LOCATION \
              $HAIL_LOCATION
    inputs:
      - from: /repo/hail/python/hail/docs
        to: /io/docs
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
    dependsOn:
      - build_hail_jar_and_wheel
      - hail_run_image
    clouds:
      - gcp
  - kind: runImage
    name: make_docs
    resources:
      memory: standard
      cpu: '4'
    image:
      valueFrom: hail_run_image.image
    script: |
      set -ex
      set -o pipefail
      export HAIL_USE_PRIVATE_LENS_DATA=1
      export HAIL_SHORT_VERSION='0.2'
      export SPHINXOPTS='-tgenerate_notebook_outputs'

      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/hail
      export HAIL_WEBSITE_DIR=/io/website/website/

      sed -E "s/\(hail\#([0-9]+)\)/(\[#\1](https:\/\/github.com\/hail-is\/hail\/pull\/\1))/g" \
        < python/hail/docs/change_log.md \
        | pandoc -o python/hail/docs/change_log.rst --fail-if-warnings --verbose

      make -C python/hail/docs BUILDDIR=_build clean html
      make -C python/hailtop/batch/docs BUILDDIR=_build clean html

      mkdir -p www/docs
      mv python/hail/docs/_build/html www/docs/0.2
      mv python/hailtop/batch/docs/_build/html www/docs/batch

      HAIL_CACHE_VERSION=$(cat env/HAIL_VERSION)
      find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +;

      tar czf /io/www.tar.gz www
    inputs:
      - from: /repo/hail
        to: /io/hail
      - from: /repo/website/website
        to: /io/website/website
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
    outputs:
      - from: /io/www.tar.gz
        to: /www.tar.gz
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_run_image
  - kind: buildImage2
    name: hailgenetics_hail_image
    dockerFile: /io/repo/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hail
    buildArgs:
      - name: BASE_IMAGE
        value:
          valueFrom: hail_ubuntu_image.image
      - name: HAIL_WHEEL_DIR
        value: /wheel
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/repo/docker/hailgenetics/hail
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /derived/release/hail/build/deploy/dist
        to: /io/repo/wheel
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_ubuntu_image
  - kind: buildImage2
    name: hailgenetics_hail_image_python_3_13
    dockerFile: /io/repo/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hail
    buildArgs:
      - name: BASE_IMAGE
        value:
          valueFrom: hail_ubuntu_image_python_3_13.image
      - name: HAIL_WHEEL_DIR
        value: /wheel
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/repo/docker/hailgenetics/hail
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /derived/release/hail/build/deploy/dist
        to: /io/repo/wheel
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_ubuntu_image_python_3_13
  - kind: buildImage2
    name: hailgenetics_hail_image_python_3_14
    dockerFile: /io/repo/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hail
    buildArgs:
      - name: BASE_IMAGE
        value:
          valueFrom: hail_ubuntu_image_python_3_14.image
      - name: HAIL_WHEEL_DIR
        value: /wheel
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/repo/docker/hailgenetics/hail
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /derived/release/hail/build/deploy/dist
        to: /io/repo/wheel
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_ubuntu_image_python_3_14
  - kind: runImage
    name: test_hailgenetics_hail_image
    scopes:
      - test
      - dev
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex
      python3 -c 'import sys; assert sys.version_info.major == 3 and sys.version_info.minor == 12, sys.version_info'
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import scipy'
    dependsOn:
      - hailgenetics_hail_image
  - kind: runImage
    name: test_hailgenetics_hail_image_python_3_13
    scopes:
      - test
      - dev
    image:
      valueFrom: hailgenetics_hail_image_python_3_13.image
    script: |
      set -ex
      python3 -c 'import sys; assert sys.version_info.major == 3 and sys.version_info.minor == 13, sys.version_info'
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import scipy'
    dependsOn:
      - hailgenetics_hail_image_python_3_13
  - kind: runImage
    name: test_hailgenetics_hail_image_python_3_14
    image:
      valueFrom: hailgenetics_hail_image_python_3_14.image
    script: |
      set -ex
      python3 -c 'import sys; assert sys.version_info.major == 3 and sys.version_info.minor == 14, sys.version_info'
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import scipy'
    dependsOn:
      - hailgenetics_hail_image_python_3_14
  - kind: buildImage2
    name: hail_dev_image
    dockerFile: /io/repo/docker/Dockerfile.hail-dev
    contextPath: /io/repo
    publishAs: hail-dev
    inputs:
      - from: /repo/docker/Dockerfile.hail-dev
        to: /io/repo/docker/Dockerfile.hail-dev
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/repo/hail/python/dev/pinned-requirements.txt
    dependsOn:
      - hailgenetics_hail_image
      - merge_code
  - kind: buildImage2
    name: hail_linting_image
    contextPath: /io/repo
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        RUN hail-apt-get-install git make openjdk-21-jre-headless
        COPY hail/python/pinned-requirements.txt hail-requirements.txt
        COPY hail/python/dev/pinned-requirements.txt dev-requirements.txt
        COPY gear/pinned-requirements.txt gear-requirements.txt
        COPY web_common/pinned-requirements.txt web_common-requirements.txt
        COPY batch/pinned-requirements.txt batch-requirements.txt
        COPY ci/pinned-requirements.txt ci-requirements.txt
        RUN python3 -m pip install \
            -r hail-requirements.txt \
            -r dev-requirements.txt \
            -r gear-requirements.txt \
            -r web_common-requirements.txt \
            -r batch-requirements.txt \
            -r ci-requirements.txt \
            && \
            python3 -m pyright --version # Force the installation of node and npm pyright in the image
    publishAs: linting
    inputs:
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/repo/hail/python/dev/pinned-requirements.txt
      - from: /repo/gear/pinned-requirements.txt
        to: /io/repo/gear/pinned-requirements.txt
      - from: /repo/web_common/pinned-requirements.txt
        to: /io/repo/web_common/pinned-requirements.txt
      - from: /repo/batch/pinned-requirements.txt
        to: /io/repo/batch/pinned-requirements.txt
      - from: /repo/ci/pinned-requirements.txt
        to: /io/repo/ci/pinned-requirements.txt
    dependsOn:
      - merge_code
      - hail_ubuntu_image
  - kind: runImage
    name: check_pip_requirements
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_linting_image.image
    script: |
      set -ex
      cd /io/repo
      chmod 755 ./check_pip_requirements.sh
      make check-pip-requirements
    inputs:
      - from: /repo/.git
        to: /io/repo/.git
      - from: /repo/Makefile
        to: /io/repo/Makefile
      - from: /repo/config.mk
        to: /io/repo/config.mk
      - from: /repo/check_pip_requirements.sh
        to: /io/repo/check_pip_requirements.sh
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/hail/python/dataproc-pre-installed-requirements.txt
        to: /io/repo/hail/python/dataproc-pre-installed-requirements.txt
      - from: /repo/hail/python/requirements.txt
        to: /io/repo/hail/python/requirements.txt
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /repo/hail/python/dev
        to: /io/repo/hail/python/dev
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /repo/batch
        to: /io/repo/batch
      - from: /repo/ci
        to: /io/repo/ci
    dependsOn:
      - hail_linting_image
      - merge_code
  - kind: runImage
    name: check_services
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_linting_image.image
    script: |
      set -ex
      cd /io/repo
      {% if 'target_sha' in code %}
      export HAIL_TARGET_SHA={{ code.target_sha }}
      {% endif %}
      make -k check-services
    inputs:
      - from: /repo/.git
        to: /io/repo/.git
      - from: /repo/Makefile
        to: /io/repo/Makefile
      - from: /repo/config.mk
        to: /io/repo/config.mk
      - from: /repo/pylintrc
        to: /io/repo/pylintrc
      - from: /repo/pyproject.toml
        to: /io/repo/pyproject.toml
      - from: /repo/check-sql.sh
        to: /io/repo/check-sql.sh
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/hail/python/dataproc-pre-installed-requirements.txt
        to: /io/repo/hail/python/dataproc-pre-installed-requirements.txt
      - from: /repo/auth
        to: /io/repo/auth
      - from: /repo/batch
        to: /io/repo/batch
      - from: /repo/ci
        to: /io/repo/ci
      - from: /repo/monitoring
        to: /io/repo/monitoring
      - from: /repo/website
        to: /io/repo/website
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_linting_image
      - merge_code
  - kind: runImage
    name: check_hail
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_linting_image.image
    resources:
      memory: 16G
    script: |
      set -ex
      cd /io/repo
      tar -C /io -xzf /io/mill-cache.tgz
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      export MILLOPTS=--no-daemon
      # This also compiles with scala 2.13, so asserts there are no compiler errors or warnings
      make check-hail
    inputs:
      - from: /repo/.git
        to: /io/repo/.git
      - from: /repo/Makefile
        to: /io/repo/Makefile
      - from: /repo/pylintrc
        to: /io/repo/pylintrc
      - from: /repo/pyproject.toml
        to: /io/repo/pyproject.toml
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/config.mk
        to: /io/repo/config.mk
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
      - from: /derived/root/hail/out/hail
        to: /io/repo/hail/out/hail
      - from: /derived/root/hail/out/Env
        to: /io/repo/hail/out/Env
    dependsOn:
      - hail_linting_image
      - merge_code
      - download_mill_and_maven_deps
  - kind: runImage
    name: get_pip_versioned_docs
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex

      cd /io/repo

      # dev deploy elides the hail-is remote, add it and retrieve the tags
      git remote add hail-is https://github.com/hail-is/hail.git

      export HAIL_PIP_VERSION=$(cat hail/env/HAIL_PIP_VERSION)

      if git ls-remote --exit-code --tags hail-is $HAIL_PIP_VERSION
      then
        # In this case, we want to get the docs from Google Storage.
        python3 -m hailtop.aiotools.copy 'null' '[
        {"from": "gs://hail-common/website/'$HAIL_PIP_VERSION'/www.tar.gz",
        "to": "www.tar.gz"}
        ]'
      else
        cp /io/www.tar.gz .
      fi

      mkdir -p hail/build/www/docs/0.2
      tar -xvf www.tar.gz -C hail/build/www/docs --strip-components 2 --no-same-owner

      python3 -m hailtop.aiotools.copy 'null' '[
      {"from": "gs://hail-common/builds/0.1/docs/hail-0.1-docs-5a6778710097.tar.gz",
       "to": "hail-0.1-docs-5a6778710097.tar.gz"}
      ]'

      mkdir -p hail/build/www/docs/0.1
      tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2 --no-same-owner

      tar czf /io/docs.tar.gz -C hail/build/www .
    resources:
      memory: standard
      cpu: '2'
    inputs:
      - from: /repo/hail/env/HAIL_PIP_VERSION
        to: /io/repo/hail/env/HAIL_PIP_VERSION
      - from: /repo/.git
        to: /io/repo/.git
      - from: /www.tar.gz
        to: /io/www.tar.gz
    outputs:
      - from: /io/docs.tar.gz
        to: /docs.tar.gz
    dependsOn:
      - hailgenetics_hail_image
      - merge_code
      - make_docs
    clouds:
      - gcp
  - kind: deploy
    name: deploy_grafana
    namespace:
      valueFrom: default_ns.name
    config: grafana/deployment.yaml
    scopes:
      - deploy
      - test
      - dev
    dependsOn:
      - default_ns
      - hail_ubuntu_image
      - create_certs
      - copy_third_party_images
  - kind: deploy
    name: deploy_prometheus
    namespace:
      valueFrom: default_ns.name
    config: prometheus/prometheus.yaml
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - create_certs
      - copy_third_party_images
  - kind: deploy
    name: deploy_auth
    namespace:
      valueFrom: default_ns.name
    config: auth/deployment.yaml
    wait:
      - kind: Service
        name: auth
        for: alive
      - kind: Service
        name: auth-driver
        for: alive
    dependsOn:
      - default_ns
      - create_deploy_config
      - create_session_key
      - auth_database
      - auth_image
      - create_certs
      - create_accounts
  - kind: runImage
    name: create_initial_user
    runIfRequested: true
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

      {% if default_ns.name == "default" %}
      hailctl auth create-user {{ code.username }} {{ code.login_id }} developer billing_manager sysadmin
      {% else %}
      hailctl auth create-user \
        --hail-identity {{ code.hail_identity }} \
        --hail-credentials-secret-name {{ code.username }}-gsa-key \
        {{ code.username }} {{ code.login_id }} developer billing_manager sysadmin
      {% endif %}
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - merge_code
      - deploy_auth
  - kind: runImage
    name: delete_monitoring_tables
    image:
      valueFrom: monitoring_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} monitoring
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - monitoring_image
      - merge_code
      - create_test_database_server_config
  - kind: createDatabase2
    name: monitoring_database
    databaseName: monitoring
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/000-initial.sql
    inputs:
      - from: /repo/monitoring/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: monitoring
    dependsOn:
      - default_ns
      - merge_code
      - delete_monitoring_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: deploy
    name: deploy_monitoring
    namespace:
      valueFrom: default_ns.name
    config: monitoring/deployment.yaml
    wait:
      - kind: Service
        name: monitoring
        for: alive
    dependsOn:
      - default_ns
      - monitoring_image
      - monitoring_database
      - deploy_auth
      - create_certs
      - create_accounts
    clouds:
      - gcp
  - kind: runImage
    name: test_monitoring
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=120 \
              /io/monitoring/test
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    timeout: 300
    inputs:
      - from: /repo/monitoring/test
        to: /io/monitoring/test
    dependsOn:
      - hail_dev_image
      - create_deploy_config
      - create_accounts
      - default_ns
      - create_certs
      - deploy_monitoring
    clouds:
      - gcp
  - kind: runImage
    name: test_auth_copy_paste_login
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex

      # Or else hailctl will try to authenticate as CI
      unset HAIL_IDENTITY_PROVIDER_JSON

      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 10 \
                         --retry-delay 5 \
                         -XPOST)
      hailctl auth copy-paste-login "$COPY_PASTE_TOKEN"

      if hailctl auth copy-paste-login "$COPY_PASTE_TOKEN"
      then
          echo "reusing a token should not work, but did"
          exit 1
      fi

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 3 \
                         --retry-delay 5 \
                         -XPOST)
      python3 -c '
      from hailtop.auth import copy_paste_login;
      copy_paste_login("'$COPY_PASTE_TOKEN'")
      '

      python3 -c '
      from hailtop.auth import copy_paste_login;
      import aiohttp
      try:
          copy_paste_login("'$COPY_PASTE_TOKEN'")
          print("reusing a token should not work, but did")
          sys.exit(1)
      except aiohttp.client_exceptions.ClientResponseError as exc:
          assert(exc.status == 401)
      '
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
    dependsOn:
      - default_ns
      - create_accounts
      - deploy_auth
      - create_deploy_config
      - hailgenetics_hailtop_image
  - kind: runImage
    name: test_auth_copy_paste_login_timeout
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex

      # Or else hailctl will try to authenticate as CI
      unset HAIL_IDENTITY_PROVIDER_JSON

      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 10 \
                         --retry-delay 5 \
                         -XPOST)
      sleep $(( 5 * 60 + 1))
      if hailctl auth copy-paste-login "$COPY_PASTE_TOKEN" --namespace {{ default_ns.name }}
      then
          echo "using an expired token should not work, but did"
          exit 1
      fi

      python3 -c '
      import aiohttp
      from hailtop.auth import copy_paste_login;
      try:
          copy_paste_login("'$COPY_PASTE_TOKEN'")
          print("using an expired token should not work, but did")
          sys.exit(1)
      except aiohttp.client_exceptions.ClientResponseError as exc:
          assert(exc.status == 401)
      '
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
    dependsOn:
      - default_ns
      - create_accounts
      - deploy_auth
      - create_deploy_config
      - hailgenetics_hailtop_image
  - kind: runImage
    name: delete_batch_tables
    image:
      valueFrom: batch_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} batch
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - batch_image
      - merge_code
      - create_test_database_server_config
  - kind: runImage
    name: delete_ci_tables
    image:
      valueFrom: ci_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} ci
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - ci_image
      - merge_code
      - create_test_database_server_config
  - kind: createDatabase2
    name: ci_database
    databaseName: ci
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/000-initial.sql
      - name: invalidated-batches
        script: /io/sql/001-invalidated-batches.sql
      - name: add-frozen-mode
        script: /io/sql/002-add-frozen-mode.sql
        online: true
      - name: active-namespaces
        script: /io/sql/003-active-namespaces.sql
        online: true
      - name: fix-default-namespace-record
        script: /io/sql/004-fix-default-namespace.py
        online: true
      - name: record-failure-notifications
        script: /io/sql/005-record-failure-notifications.sql
        online: true
      - name: add-service-rate-limits
        script: /io/sql/006-add-service-rate-limits.sql
        online: true
      - name: retried-tests
        script: /io/sql/007-retried-tests.sql
        online: true
    inputs:
      - from: /repo/ci/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: ci
    dependsOn:
      - default_ns
      - merge_code
      - delete_ci_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: createDatabase2
    name: batch_database
    databaseName: batch
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/000-initial.sql
      - name: insert-globals
        script: /io/sql/001-insert_globals.py
      - name: create-billing-projects
        script: /io/sql/002-create-billing-projects.sql
      - name: increase-spec-size
        script: /io/sql/003-increase-spec-size.sql
      - name: remove-fk-batches-user-resources
        script: /io/sql/004-remove-fk-batches-user-resources.sql
      - name: change-ready-cores-bigint
        script: /io/sql/005-change-ready-cores-bigint.sql
      - name: add-user-resources-ready-cores-token
        script: /io/sql/006-add-user-resources-ready-cores-token.sql
      - name: add-lock-in-share-mode
        script: /io/sql/007-add-lock-in-share-mode.sql
      - name: add-batch-create-token
        script: /io/sql/008-add-batch-create-token.sql
      - name: improve-cancel
        script: /io/sql/009-improve-cancel.sql
      - name: store-specs-in-gcs
        script: /io/sql/010-store-specs-in-gcs.sql
      - name: add-batches-time-closed
        script: /io/sql/011-add-batches-time-closed.sql
      - name: add-instance-zone
        script: /io/sql/012-add-instance-zone.sql
      - name: add-batches-time-created-index
        script: /io/sql/013-add-batches-time-created-index.sql
      - name: change-test-worker-disk-size
        script: /io/sql/014-change_test_worker_disk_size.py
      - name: add-resources
        script: /io/sql/015-add-resources.sql
      - name: insert-resources
        script: /io/sql/016-insert_resources.py
      - name: insert-standing-worker-globals
        script: /io/sql/017-insert_standing_worker_globals.py
      - name: insert-local-ssd-resource
        script: /io/sql/018-insert_local_ssd_resource.py
      - name: fix-mark-job-complete-on-error
        script: /io/sql/019-fix-mark-job-complete-on-error.sql
      - name: add-worker-pd-ssd-data-disk
        script: /io/sql/020-add-worker-pd-ssd-data-disk.sql
      - name: add-aggregated-batch-resources-tokens
        script: /io/sql/021-add-aggregated-batch-resources-tokens.sql
      - name: change-test-worker-pool-size
        script: /io/sql/022-change_test_worker_pool_size.py
      - name: add-test-dev-to-billing-project-users
        script: /io/sql/023-add-test-dev-to-billing-project-users.sql
      - name: add-status-flag-billing-projects
        script: /io/sql/024-add-status-flag-billing-projects.sql
      - name: add-aggregated-billing-project-resources
        script: /io/sql/025-add-aggregated-billing-project-resources.sql
      - name: add-enable-standing-worker
        script: /io/sql/026-add-enable-standing-worker.py
      - name: add-inst-colls
        script: /io/sql/027-add-inst-colls.sql
      - name: add-creating-job-state
        script: /io/sql/028-add-creating-job-state.sql
      - name: add-job-private-inst-coll
        script: /io/sql/029-add-job-private-inst-coll.sql
      - name: insert-nonpreemptible-resources
        script: /io/sql/030-insert_nonpreemptible_resources.py
      - name: fix-schedule-job
        script: /io/sql/031-fix-schedule-job.sql
      - name: increase-test-and-dev-pool-sizes
        script: /io/sql/032-increase-test-and-dev-pool-sizes.py
      - name: fix-provisioning-bug
        script: /io/sql/033-fix-provisioning-bug.sql
      - name: do-not-lock-entire-batch
        script: /io/sql/034-do-not-lock-entire-batch.sql
      - name: add-fail-fast
        script: /io/sql/035-add-fail-fast.sql
      - name: add-frozen-mode
        script: /io/sql/036-add-frozen-mode.sql
      - name: add-instance-config
        script: /io/sql/037-add-instance-config.sql
      - name: cloud-agnostic
        script: /io/sql/038-cloud-agnostic.sql
      - name: support-azure
        script: /io/sql/039-support-azure.sql
      - name: add-azure-tables
        script: /io/sql/040-add_azure_tables.py
      - name: change-azure-test-highcpu-pool
        script: /io/sql/041-change_azure_test_highcpu_pool.py
      - name: add-latest-product-versions
        script: /io/sql/042-add-latest-product-versions.sql
      - name: add-products-and-versions
        script: /io/sql/043-add_products_and_versions.py
      - name: kill-more-deadlocks
        script: /io/sql/044-kill-more-deadlocks.sql
      - name: kill-more-deadlocks2
        script: /io/sql/045-kill-more-deadlocks2.sql
      - name: no-locks-add-attempt
        script: /io/sql/046-no-locks-add-attempt.sql
      - name: fix-n-cancelled-creating-jobs
        script: /io/sql/047-fix-n-cancelled-creating-jobs.sql
      - name: big-test-instances
        script: /io/sql/048-big-test-instances.py
      - name: kill-mjc-deadlocks
        script: /io/sql/049-kill-mjc-deadlocks.sql
      - name: add-nonpreemptible-pools
        script: /io/sql/050-add-nonpreemptible-pools.sql
      - name: set-test-and-dev-pools-to-8-core-max-2
        script: /io/sql/051-set-test-and-dev-pools-to-8-core-max-2.py
      - name: set-test-and-dev-jpim-to-max-5
        script: /io/sql/052-set-test-and-dev-jpim-to-max-5.py
        online: true
      - name: minimize-deadlock-errors
        script: /io/sql/053-minimize-deadlock-errors.sql
        online: true
      - name: add-resource-ids-sql
        script: /io/sql/054-add-resource-ids.sql
        online: true
      - name: add-resource-ids-python
        script: /io/sql/055-add_resource_ids.py
        online: true
      - name: modify-resource-id-trigger
        script: /io/sql/056-modify-resource-id-trigger.sql
        online: true
      - name: rm-resource-names-agg-resources
        script: /io/sql/057-rm-resource-names-agg-resources.sql
        online: true
      - name: rm-resource-foreign-keys
        script: /io/sql/058-rm-resource-foreign-keys.py
        online: true
      - name: rm-resource-names-agg-resources-pt-2
        script: /io/sql/059-rm-resource-names-agg-resources-pt-2.sql
        online: true
      - name: add-att-resources-format-version-lt-3
        script: /io/sql/060-add-att-resources-format-version-lt-3.sql
        online: true
      - name: insert-attempt-resources-format-version-lt-3
        script: /io/sql/061-insert_attempt_resources_format_version_lt_3.py
        online: true
      - name: revert-attempt-resources-trigger-back-compat
        script: /io/sql/062-revert-attempt-resources-trigger-back-compat.sql
        online: true
      - name: add-agg-billing-by-date
        script: /io/sql/063-add-agg-billing-by-date.sql
        online: true
      - name: populate-agg-billing-by-date
        script: /io/sql/064-populate_agg_billing_by_date.py
        online: true
      - name: revert-temp-agg-by-date-infra
        script: /io/sql/065-revert-temp-agg-by-date-infra.sql
        online: true
      - name: rename-timestamp-to-date
        script: /io/sql/066-rename-timestamp-to-date.sql
        online: true
      - name: add-real-time-billing
        script: /io/sql/067-add-real-time-billing.sql
        online: true
      - name: add-batch-updates
        script: /io/sql/068-add-batch-updates.sql
        online: true
      - name: populate-batch-updates
        script: /io/sql/069-populate_batch_updates.py
        online: true
      - name: cleanup-add-batch-updates
        script: /io/sql/070-cleanup-add-batch-updates.sql
        online: true
      - name: add-jobs-update-id-index
        script: /io/sql/071-add-jobs-update-id-index.sql
        online: true
      - name: no-dev-standing-workers-by-default
        script: /io/sql/072-no_dev_standing_workers_by_default.py
        online: true
      - name: add-commit-updates
        script: /io/sql/073-add-commit-updates.sql
        online: true
      - name: add-job-regions
        script: /io/sql/074-add-job-regions.sql
        online: true
      - name: faster-regions-pool-scheduler
        script: /io/sql/075-faster-regions-pool-scheduler.sql
        online: true
      - name: add-batch-state-index
        script: /io/sql/076-add-batch-state-index.sql
        online: true
      - name: case-sensitive-billing-project
        script: /io/sql/077-case-sensitive-billing-project.sql
        online: true
      - name: driver-behavior
        script: /io/sql/078-driver_behavior.py
        online: true
      - name: cleanup-old-billing-tables
        script: /io/sql/079-cleanup-old-billing-tables.sql
        online: true
      - name: add-deduped-resource-ids-to-resources
        script: /io/sql/080-add-deduped-resource-ids-to-resources.sql
        online: true
      - name: create-resource-dedup-mapping
        script: /io/sql/081-create_resource_dedup_mapping.py
        online: true
      - name: config-pool-min-instances
        script: /io/sql/082-config-pool-min-instances.sql
        online: true
      - name: add-deduped-resource-ids-to-attempt-resources
        script: /io/sql/083-add-deduped-resource-ids-to-attempt-resources.sql
        online: true
      - name: setup-deduped-resources-migration
        script: /io/sql/084-setup-deduped-resources-migration.sql
        online: true
      - name: dedup-attempt-resources
        script: /io/sql/085-dedup_attempt_resources.py
        online: true
      - name: mitigate-bad-attempt-resources-trigger
        script: /io/sql/086-mitigate-bad-attempt-resources-trigger.sql
        online: true
      - name: set-test-and-dev-pools-to-16-core-max-3
        script: /io/sql/087-set-test-and-dev-pools-to-16-core-max-3.py
        online: true
      - name: fix-billing-triggers-deduped-resource-id
        script: /io/sql/088-fix-billing-triggers-deduped-resource-id.sql
        online: true
      - name: set-test-and-dev-pools-to-max-16-standing-16
        script: /io/sql/089-set-test-and-dev-pools-to-max-16-standing-16.py
        online: true
      - name: set-test-min-pool-size-to-1
        script: /io/sql/090-set-test-min-pool-size-to-1.py
        online: true
      - name: dedup-billing-project-users-v2
        script: /io/sql/091-dedup_billing_project_users_v2.py
        online: true
      - name: list-jobs-extra-indices
        script: /io/sql/092-list-jobs-extra-indices.sql
        online: true
      - name: dedup-billing-project-users-by-date
        script: /io/sql/093-dedup_billing_project_users_by_date.py
        online: true
      - name: add-jobs-ready-time
        script: /io/sql/094-add-jobs-ready-time.sql
        online: true
      - name: add-list-batches-index
        script: /io/sql/095-add-list-batches-index.sql
        online: true
      - name: add-feature-flags
        script: /io/sql/096-add-feature-flags.sql
        online: true
      - name: add-oms-agent-flag
        script: /io/sql/097-add-oms-agent-flag.sql
        online: true
      - name: increase-test-max-idle-time
        script: /io/sql/098-increase_test_max_idle_time.py
        online: true
      - name: turn_off_oms_agent_test_dev
        script: /io/sql/099-turn_off_oms_agent_test_dev.py
        online: true
      - name: add-billing-index-token
        script: /io/sql/100-add-billing-index-token.sql
        online: true
      - name: jobs-after-update-simple.sql
        script: /io/sql/101-jobs-after-update-simple.sql
        online: true
      - name: turn-on-billing-compaction
        script: /io/sql/102-turn_on_billing_compaction.py
        online: true
      - name: cleanup-deprecated-functions
        script: /io/sql/103-cleanup-deprecated-functions.sql
        online: true
      - name: set-test-pools-to-known-parameters
        script: /io/sql/104-set-test-pools-to-known-parameters.py
        online: true
      - name: add-gcp-support-logs-specs-and-firewall-fees
        script: /io/sql/105-add-gcp-support-logs-specs-and-firewall-fees.py
        online: true
      - name: add-product-sku
        script: /io/sql/106-add-product-sku.sql
        online: true
      - name: dedup-agg-batch-resources
        script: /io/sql/107-dedup_agg_batch_resources.py
        online: true
      - name: setup-job-groups
        script: /io/sql/108-setup-job-groups.sql
        online: true
      - name: dedup-job-resources
        script: /io/sql/109-dedup_job_resources.py
        online: true
      - name: fix-job-groups-state-enum
        script: /io/sql/110-fix-job-groups-state-enum.sql
        online: true
      - name: populate-job-groups
        script: /io/sql/111-populate_job_groups.py
        online: true
      - name: rename-job-groups-tables
        script: /io/sql/112-rename-job-groups-tables.sql
        online: false  # this must be offline
      - name: remove-v2-billing-writes
        script: /io/sql/113-remove-v2-billing-writes.sql
        online: true
      - name: update-ip-fee-resource
        script: /io/sql/114-update-ip-fee-resource.py
        online: true
      - name: remove-v2-billing-tables
        script: /io/sql/115-remove-v2-billing-tables.sql
        online: true
      - name: finalize-job-groups
        script: /io/sql/116-finalize-job-groups.sql
        online: true
      - name: fix-mark-job-complete-deadlocks
        script: /io/sql/117-fix-mark-job-complete-deadlocks.sql
        online: true
      - name: jobs-add-n-max-attempts
        script: /io/sql/118-jobs-add-n-max-attempts.sql
        online: true
      - name: is-job-cancelled
        script: /io/sql/119-is-job-cancelled.sql
        online: true
      - name: add-dockerhub-proxy-flag
        script: /io/sql/120-add-dockerhub-proxy-flag.sql
        online: true
      - name: fix-provisioning-availability-bug
        script: /io/sql/121-fix-provisioning-availability-bug.sql
        online: true
    inputs:
      - from: /repo/batch/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: batch
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: batch-driver
    dependsOn:
      - default_ns
      - merge_code
      - delete_batch_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: deploy
    name: deploy_batch
    namespace:
      valueFrom: default_ns.name
    config: batch/deployment.yaml
    wait:
      - kind: Service
        name: batch
        for: alive
      - kind: Service
        name: batch-driver
        for: alive
    dependsOn:
      - default_ns
      - deploy_batch_sa
      - create_accounts
      - batch_image
      - batch_worker_image
      - batch_database
      - deploy_auth
      - create_certs
  - kind: runImage
    name: add_developers
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex

      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      {% for user in code.get("developers", []) %}
      {% if user['username'] != 'test-dev' and user['login_id'] is not none %}
      hailctl auth create-user \
        --hail-identity {{ user["hail_identity"] }} \
        --hail-credentials-secret-name {{ user["username"] }}-gsa-key \
        {{ user["username"] }} {{ user["login_id"] }} developer billing_manager sysadmin
      {% endif %}
      {% endfor %}
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    scopes:
      - dev
      - test
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - deploy_batch
  - kind: runImage
    name: upload_spark_jars
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/batch-gsa-key/key.json

      {% if deploy %}
      export JAR_PREFIX={{ global.query_storage_uri }}
      {% else %}
      export JAR_PREFIX={{ global.test_storage_uri }}/{{ default_ns.name }}
      {% endif %}

      bash /io/upload-spark-jars.sh {{ default_ns.name }} $(cat /io/SPARK_VERSION)
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    inputs:
      - from: /repo/hail/scripts/upload-spark-jars.sh
        to: /io/upload-spark-jars.sh
      - from: /repo/hail/env/SPARK_VERSION
        to: /io/SPARK_VERSION
    dependsOn:
      - default_ns
      - merge_code
      - create_test_gsa_keys
      - hailgenetics_hailtop_image
  - kind: runImage
    name: upload_query_jar
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/batch-gsa-key/key.json
      export UPLOAD_RELEASE_JAR=true
      
      {% if deploy %}
      export JAR_PREFIX={{ global.query_storage_uri }}
      {% else %}
      export JAR_PREFIX={{ global.test_storage_uri }}/{{ default_ns.name }}
      {% endif %}
      
      bash /io/upload-qob-jar.sh {{ default_ns.name }} $(cat /io/REVISION) /io/hail.jar
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    inputs:
      - from: /repo/hail/scripts/upload-qob-jar.sh
        to: /io/upload-qob-jar.sh
      - from: /derived/release/hail/out/hail/assembly.dest/out.jar
        to: /io/hail.jar
      - from: /repo/hail/env/REVISION
        to: /io/REVISION
    dependsOn:
      - default_ns
      - deploy_batch
      - hailgenetics_hailtop_image
      - build_hail_jar_and_wheel
      - merge_code
      - create_test_gsa_keys
      - upload_spark_jars
  - kind: runImage
    name: test_hail_python_service_backend_gcp
    scopes:
      - test
      - dev
    numSplits: 15
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.25'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/repo/hail/python

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/"
      export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/"
      export HAIL_GENETICS_VEP_GRCH37_85_IMAGE={{ hailgenetics_vep_grch37_85_image.image }}
      export HAIL_GENETICS_VEP_GRCH38_95_IMAGE={{ hailgenetics_vep_grch38_95_image.image }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export GCS_REQUESTER_PAYS_PROJECT=broad-ctsa

      export HAIL_SHUFFLE_MAX_BRANCH=4
      export HAIL_SHUFFLE_CUTOFF=1000000
      export HAIL_QUERY_BACKEND=batch
      export HAIL_BATCH_REGIONS={{ global.gcp_region }}
      export HAIL_BATCH_BILLING_PROJECT=test
      export HAIL_BATCH_REMOTE_TMPDIR={{ global.test_storage_uri }}

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --ignore=test/hailtop/ \
              --ignore=test/hail/matrixtable/test_file_formats.py \
              --ignore=test/hail/utils/test_hl_hadoop_and_hail_fs.py \
              -m "backend and not vep" \
              --timeout=600 \
              test
    timeout: 5400
    inputs:
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
      - from: /repo/hail/python/pytest.ini
        to: /io/repo/hail/python/pytest.ini
      - from: /repo/hail/python/test
        to: /io/repo/hail/python/test
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - merge_code
      - deploy_batch
      - create_deploy_config
      - create_accounts
      - hail_run_image
      - upload_query_jar
      - upload_test_resources_to_blob_storage
      - build_hail_jar_and_wheel
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_service_backend_gcp_vep
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.25'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/repo/hail/python

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/"
      export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/"
      export HAIL_GENETICS_VEP_GRCH37_85_IMAGE={{ hailgenetics_vep_grch37_85_image.image }}
      export HAIL_GENETICS_VEP_GRCH38_95_IMAGE={{ hailgenetics_vep_grch38_95_image.image }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export GCS_REQUESTER_PAYS_PROJECT=broad-ctsa

      export HAIL_SHUFFLE_MAX_BRANCH=4
      export HAIL_SHUFFLE_CUTOFF=1000000
      export HAIL_QUERY_BACKEND=batch
      export HAIL_BATCH_REGIONS={{ global.gcp_region }}
      export HAIL_BATCH_BILLING_PROJECT=test
      export HAIL_BATCH_REMOTE_TMPDIR={{ global.test_storage_uri }}

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              -m "backend and vep" \
              --timeout=600 \
              test/hail/methods/test_qc.py
    timeout: 5400
    inputs:
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
      - from: /repo/hail/python/pytest.ini
        to: /io/repo/hail/python/pytest.ini
      - from: /repo/hail/python/test
        to: /io/repo/hail/python/test
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - merge_code
      - deploy_batch
      - create_deploy_config
      - create_accounts
      - hail_run_image
      - upload_query_jar
      - upload_test_resources_to_blob_storage
      - build_hail_jar_and_wheel
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_service_backend_gcp_fs
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.25'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/repo/hail/python

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/"
      export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/"
      export HAIL_GENETICS_VEP_GRCH37_85_IMAGE={{ hailgenetics_vep_grch37_85_image.image }}
      export HAIL_GENETICS_VEP_GRCH38_95_IMAGE={{ hailgenetics_vep_grch38_95_image.image }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export GCS_REQUESTER_PAYS_PROJECT=broad-ctsa

      export HAIL_QUERY_BACKEND=batch
      export HAIL_BATCH_REGIONS={{ global.gcp_region }}
      export HAIL_BATCH_BILLING_PROJECT=test
      export HAIL_BATCH_REMOTE_TMPDIR={{ global.test_storage_uri }}

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              -m backend \
              --timeout=600 \
              test/hail/utils/test_hl_hadoop_and_hail_fs.py
    timeout: 900
    inputs:
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
      - from: /repo/hail/python/pytest.ini
        to: /io/repo/hail/python/pytest.ini
      - from: /repo/hail/python/test
        to: /io/repo/hail/python/test
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - merge_code
      - deploy_batch
      - create_deploy_config
      - create_accounts
      - hail_run_image
      - upload_query_jar
      - upload_spark_jars
      - upload_test_resources_to_blob_storage
      - build_hail_jar_and_wheel
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_spark_conf_requester_pays_parsing
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.5'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      export YOU_MAY_OVERWRITE_MY_SPARK_DEFAULTS_CONF_AND_HAILCTL_SETTINGS=1

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --asyncio-mode=auto \
              /io/test_requester_pays_parsing.py
    inputs:
      - from: /derived/release/hail/build/deploy/dist
        to: /io/wheel
      - from: /repo/hail/scripts/test_requester_pays_parsing.py
        to: /io/test_requester_pays_parsing.py
    dependsOn:
      - default_ns
      - merge_code
      - hail_run_image
      - build_hail_jar_and_wheel
  - kind: buildImage2
    name: netcat_ubuntu_image
    publishAs: netcat
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        RUN hail-apt-get-install netcat-openbsd
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: volume_image
    publishAs: volume
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        VOLUME ["/data"]
        WORKDIR "/tmp"
        VOLUME ["relative_volume"]
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: gpu_image
    publishAs: gpu
    resources:
      storage: 10Gi
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        RUN hail-pip-install torch torchvision -f https://download.pytorch.org/whl/cu113/torch_stable.html
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: workdir_image
    publishAs: workdir
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        WORKDIR ["/work"]
    dependsOn:
      - hail_ubuntu_image
  - kind: runImage
    name: test_batch
    scopes:
      - test
      - dev
    numSplits: 5
    image:
      valueFrom: batch_image.image
    script: |
      set -ex

      # Use the test identity's credentials instead of CI's
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_TEST_GSA_KEY_FILE=/test-gsa-key/key.json
      export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json

      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export CI_UTILS_IMAGE={{ ci_utils_image.image }}
      export HAIL_CURL_IMAGE={{ hail_ubuntu_image.image }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_VOLUME_IMAGE={{ volume_image.image }}
      export HAIL_WORKDIR_IMAGE={{ workdir_image.image }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}"
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"
      export HAIL_TOKEN="{{ token }}"
      export HAIL_CLOUD="{{ global.cloud }}"
      export HAIL_PRODUCTION_DOMAIN="{{ global.domain }}"
      export HAIL_GPU_IMAGE="{{ gpu_image.image }}"
      export HAIL_BATCH_REMOTE_TMPDIR="{{ global.test_storage_uri }}/test_batch/{{ token }}/"

      hail-pip-install -r /io/dev-requirements.txt

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              -k "not test_scale and not test_invariants" \
              --timeout=360 \
              /io/test/test_batch.py
    inputs:
      - from: /repo/batch/test
        to: /io/test
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
    port: 5000
    timeout: 1500
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - deploy_batch
      - batch_image
      - ci_utils_image
      - hailgenetics_hail_image
      - hailgenetics_hailtop_image
      - netcat_ubuntu_image
      - volume_image
      - workdir_image
      - hail_ubuntu_image
      - gpu_image
  - kind: runImage
    name: test_batch_job_private_machines
    scopes:
      - test
      - dev
    numSplits: 1
    image:
      valueFrom: batch_image.image
    script: |
      set -ex

      # Use the test identity's credentials instead of CI's
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_TEST_GSA_KEY_FILE=/test-gsa-key/key.json
      export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json

      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export CI_UTILS_IMAGE={{ ci_utils_image.image }}
      export HAIL_CURL_IMAGE={{ hail_ubuntu_image.image }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_VOLUME_IMAGE={{ volume_image.image }}
      export HAIL_WORKDIR_IMAGE={{ workdir_image.image }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}"
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"
      export HAIL_TOKEN="{{ token }}"
      export HAIL_CLOUD="{{ global.cloud }}"
      export HAIL_PRODUCTION_DOMAIN="{{ global.domain }}"
      export HAIL_BATCH_REMOTE_TMPDIR="{{ global.test_storage_uri }}/test_batch/{{ token }}/"

      hail-pip-install -r /io/dev-requirements.txt

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              --timeout=660 \
              /io/test/test_batch_rare_instances.py
    inputs:
      - from: /repo/batch/test
        to: /io/test
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
    port: 5000
    timeout: 1500
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - deploy_batch
      - batch_image
      - ci_utils_image
      - hailgenetics_hail_image
      - hailgenetics_hailtop_image
      - netcat_ubuntu_image
      - volume_image
      - workdir_image
      - hail_ubuntu_image
  - kind: runImage
    name: batch_worker_image_regression_tests
    scopes:
      - test
      - dev
    numSplits: 1
    image:
      valueFrom: batch_image.image
    script: |
      set -ex

      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export HAIL_TEST_GSA_KEY_FILE=/test-gsa-key/key.json
      export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TOKEN="{{ token }}"
      export HAIL_CLOUD="{{ global.cloud }}"
      export HAIL_PRODUCTION_DOMAIN="{{ global.domain }}"
      export HAIL_BATCH_REMOTE_TMPDIR="{{ global.test_storage_uri }}/batch_worker_image_regression_tests/{{ token }}/"

      hail-pip-install -r /io/dev-requirements.txt

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              --timeout=660 \
              /io/test/test_worker_image_security.py
    inputs:
      - from: /repo/batch/test
        to: /io/test
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
    port: 5000
    timeout: 1800
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - deploy_batch
      - batch_image
  - kind: runImage
    name: test_auth
    scopes:
      - test
      - dev
    numSplits: 1
    image:
      valueFrom: auth_image.image
    script: |
      set -ex

      hail-pip-install -r /io/dev-requirements.txt

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              --timeout=360 \
              --ignore=/io/test/test_system_permissions.py \
              /io/test/
    inputs:
      - from: /repo/auth/test
        to: /io/test
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
    timeout: 1500
    dependsOn:
      - auth_image

  - kind: runImage
    name: delete_test_billing_projects
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: batch_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json
      export HAIL_TOKEN="{{ test_batch.token }}"
      cd /io/test
      python3 -c '
      import billing_projects
      import asyncio
      asyncio.get_event_loop().run_until_complete(billing_projects.delete_all_test_billing_projects())
      '
    inputs:
      - from: /repo/batch/test
        to: /io/test
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    alwaysRun: true
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - batch_image
      - deploy_batch
    after:
      - test_batch
  - kind: runImage
    name: create_ci_test_repo
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -e
      TOKEN=$(cat /secret/ci-secrets/user1)
      REPO_NAME="ci-test-{{token}}"

      echo creating $REPO_NAME...
      curl -XPOST \
        -i \
        -fsSL \
        https://api.github.com/orgs/hail-ci-test/repos \
        -H "Authorization: token ${TOKEN}" \
        -d "{ \"name\" : \"$REPO_NAME\" }"

      # checkout new ci repo
      # Repeated clones should happen in fresh directories so we mv to the final
      # destination on successful clone
      set +x
      clone() {
        tmp_dir=$(mktemp -d) && git clone https://$1@github.com/hail-ci-test/$2.git $tmp_dir && mv $tmp_dir $2
      }
      typeset -fx clone
      retry clone $TOKEN $REPO_NAME
      set -x
      cd $REPO_NAME

      mkdir -p ./ci/test ./hail/
      cp /io/repo/ci/test/resources/build.yaml ./
      cp -R /io/repo/ci/* ./ci/
      cp /io/repo/tls/Dockerfile ./ci/test/resources/Dockerfile.certs
      cp /io/repo/tls/create_certs.py ./ci/test/resources/
      cp /io/repo/tls/create_test_db_config.sh ./ci/test/resources/
      cp /io/repo/pylintrc ./
      cp /io/repo/check-sql.sh ./
      cp /io/repo/pyproject.toml ./
      cp -R /io/repo/docker ./
      cp -R /io/repo/gear ./
      cp -R /io/repo/hail/python ./hail/
      cp /io/repo/hail/Makefile ./hail/
      cp /io/repo/hail/env_var.mk ./hail/
      cp -R /io/repo/web_common ./
      cp /io/repo/hail/python/setup-hailtop.py ./hail/python/

      git config user.name ci
      git config user.email ci@hail.is
      git add * && git commit -m "setup repo"
      retry git push
    secrets:
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
    scopes:
      - test
      - dev
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
      - from: /repo/tls
        to: /io/repo/tls
      - from: /repo/pylintrc
        to: /io/repo/pylintrc
      - from: /repo/check-sql.sh
        to: /io/repo/check-sql.sh
      - from: /repo/pyproject.toml
        to: /io/repo/pyproject.toml
      - from: /repo/docker
        to: /io/repo/docker
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/hail/python
        to: /io/repo/hail/python
      - from: /repo/hail/Makefile
        to: /io/repo/hail/Makefile
      - from: /repo/hail/env_var.mk
        to: /io/repo/hail/env_var.mk
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - default_ns
      - ci_utils_image
      - merge_code
  - kind: runImage
    name: create_ci_config
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      kubectl -n {{ default_ns.name }} create secret generic ci-config \
          --from-literal=github_context="ci-test" \
          --from-literal=storage_uri="{{ global.test_storage_uri }}" \
          --from-literal=deploy_steps="[]" \
          --from-literal=watched_branches="[[\"hail-ci-test/ci-test-{{create_ci_test_repo.token}}:main\", true, true]]" \
          --save-config --dry-run=client -o yaml \
        | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - ci_utils_image
      - create_ci_test_repo
  - kind: deploy
    name: deploy_ci
    namespace:
      valueFrom: default_ns.name
    config: ci/deployment.yaml
    wait:
      - kind: Service
        name: ci
        for: alive
    dependsOn:
      - default_ns
      - ci_image
      - ci_utils_image
      - create_accounts
      - ci_database
      - deploy_auth
      - deploy_batch
      - create_ci_test_repo
      - create_ci_config
      - deploy_ci_agent
      - create_certs
      - hail_buildkit_image
  - kind: runImage
    name: test_system_permissions
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: auth_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json

      hail-pip-install -r /io/dev-requirements.txt

      python3 -m pytest \
              --color=yes \
              --log-cli-level=INFO -s -r A -vv --instafail \
              --durations=50 --timeout=600 \
              /io/test/test_system_permissions.py
    inputs:
      - from: /repo/auth/test
        to: /io/test
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
    timeout: 900
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - deploy_auth
      - deploy_batch
      - deploy_ci
      - deploy_monitoring
      - auth_image
  - kind: runImage
    name: test_ci_unit
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_image.image
    script: |
      set -ex
      hail-pip-install -r /io/dev-requirements.txt
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              /io/ci/unit-test
    timeout: 1200
    inputs:
      - from: /repo/ci
        to: /io/ci
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
      - from: /repo/build.yaml
        to: /io/build.yaml
    scopes:
      - test
      - dev
    dependsOn:
      - merge_code
      - ci_image
  - kind: runImage
    name: test_ci
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      export ORGANIZATION=hail-ci-test
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export REPO_NAME=ci-test-"{{ create_ci_test_repo.token }}"
      export NAMESPACE="{{ default_ns.name }}"
      pip install /io/ci
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              /io/ci/test
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    timeout: 5400
    inputs:
      - from: /repo/ci
        to: /io/ci
    scopes:
      - test
      - dev
    dependsOn:
      - merge_code
      - create_deploy_config
      - create_accounts
      - default_ns
      - create_certs
      - deploy_ci
      - hail_dev_image
      - create_ci_test_repo
  - kind: runImage
    name: test_hailtop_python
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    numSplits: 5
    image:
      valueFrom: hail_dev_image.image
    script: |
      cd /io
      set -ex

      tar xzf test.tar.gz
      python3 -m pip install --no-dependencies /io/debug-wheel/hail-*-py3-none-any.whl

      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"
      export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}"
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}

      {% if global.cloud == 'gcp' %}
      export GCS_REQUESTER_PAYS_PROJECT={{ global.gcp_project }}
      {% endif %}

      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/test_hailtop_python/{{ token }}/

      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=360 \
              --ignore=test/hailtop/inter_cloud \
              test/hailtop
    inputs:
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /derived/debug/hail/build/deploy/dist
        to: /io/debug-wheel
    timeout: 1200
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - hailgenetics_hail_image
      - hailgenetics_hailtop_image
      - create_deploy_config
      - create_accounts
      - default_ns
      - hail_dev_image
      - build_hail_debug_jar_and_wheel
      - deploy_batch
  - kind: runImage
    name: test_hailctl_batch
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex

      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"

      {% if global.cloud == "gcp" %}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      {% else %}
      echo "unknown cloud {{ global.cloud }}"
      exit 1
      {% endif %}


      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/hailctl-test/{{ token }}

      mkdir -p foo
      echo "bar" > foo/baz.txt

      cat >simple_hail.py <<EOF
      import hail as hl

      with open('foo/baz.txt') as f:
          print(f.read())

      hl.init(app_name='test-hailctl-batch-submit-query')
      hl.utils.range_table(10).collect()
      EOF

      # https://stackoverflow.com/questions/3524978/logging-functions-in-bash-and-stdout
      exec 3>&1

      function log { echo "$@" 1>&3; }

      function submit {
        local status=$(hailctl batch submit --quiet --output json --wait "$@")
        log "$status"
        log $(hailctl batch log -o json $(jq -jr '.batch_id' <<< "$status") 1)  # print the job log
        test $(jq -jr '.state' <<< "$status") = "Success"
      }

      submit -v simple_hail.py:/ -v foo/:/foo/ --workdir / --name=test-hailctl-batch-submit python3 simple_hail.py

      cat >hail_with_args.py <<EOF
      import hail as hl
      import sys

      with open('foo/baz.txt') as f:
          print(f.read())

      hl.init(app_name='test-hailctl-batch-submit-query')
      assert hl.utils.range_table(int(sys.argv[1]))._force_count() == 100
      EOF

      submit -v hail_with_args.py:/ -v foo/:/foo/ --workdir / --name=test-hailctl-batch-submit python3 hail_with_args.py 100

      cat >file.sh <<EOF
      set -ex

      cat foo/baz.txt
      echo "Hello World!"
      EOF

      submit --shell /bin/sh --name=test-hailctl-batch-submit -v file.sh:/ -v foo/:/foo/ -o json --image busybox:latest sh file.sh

      cat >file-with-args.sh <<EOF
      [ "$#" -eq 2 ]

      cat foo/baz.txt
      echo "Hello World! $1 $2"
      EOF

      submit --name=test-hailctl-batch-submit -v file-with-args.sh:/ -v foo/:/foo/ -o json --image ubuntu:latest sh file-with-args.sh abc 123
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - hailgenetics_hail_image
      - upload_query_jar
      - upload_spark_jars
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - deploy_batch
  - kind: runImage
    name: test_hailctl_unit
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      cd /io/repo
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=120 \
              hail/python/test/hailtop/hailctl/batch/test_jobs_unit.py
    inputs:
      - from: /repo/hail/python
        to: /io/repo/hail/python
    scopes:
      - test
      - dev
    dependsOn:
      - merge_code
      - hail_dev_image
  - kind: runImage
    name: test_batch_docs
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/test_batch_docs/{{ token }}/
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --doctest-modules \
              --doctest-glob='*.rst' \
              --doctest-continue-on-failure \
              --ignore=docs/cookbook/files/ \
              --timeout=120 \
              --ignore=docs/conf.py \
              /io/hailtop/batch
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - hail_dev_image
      - merge_code
      - deploy_batch
    timeout: 1200
    inputs:
      - from: /repo/hail/python/hailtop
        to: /io/hailtop
  - kind: runImage
    name: cleanup_ci_test_repo
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_ubuntu_image.image
    script: |
      set -e
      TOKEN=$(cat /secret/ci-secrets/user1)
      echo deleting ci-test-{{ create_ci_test_repo.token }}...
      curl -XDELETE \
        -ifsSL \
        https://api.github.com/repos/hail-ci-test/ci-test-{{ create_ci_test_repo.token }} \
        -H "Authorization: token ${TOKEN}"
    secrets:
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
    alwaysRun: true
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - hail_ubuntu_image
      - create_ci_test_repo
    after:
      - deploy_ci
      - test_ci
  - kind: runImage
    name: test_dataproc-37
    image:
      valueFrom: ci_utils_image.image
    resources:
      preemptible: False
    script: |
      set -ex

      cd /io/repo

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      gcloud config set project hail-vdc
      gcloud config set dataproc/region us-central1

      cd hail
      make test-dataproc-37 HAIL_BUILD_MODE=Release \
          -o $(ls build/deploy/dist/hail-*-py3-none-any.whl)
    dependsOn:
      - ci_utils_image
      - default_ns
      - build_hail_jar_and_wheel
      - test_install_in_isolation_python3_12
      - upload_temporary_hailctl_artifacts
    inputs:
      - from: /repo
        to: /io/repo
      - from: /derived/release/hail/build/deploy/dist
        to: /io/repo/hail/build/deploy/dist
    secrets:
      - name: test-dataproc-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dataproc-service-account-key
    scopes:
      - deploy
      - dev
    forceIfLabeled:
      - run-dataproc-tests
    onlyIfRelease: true
    clouds:
      - gcp
  - kind: runImage
    name: test_dataproc-38
    image:
      valueFrom: ci_utils_image.image
    resources:
      preemptible: False
    script: |
      set -ex

      cd /io/repo

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      gcloud config set project hail-vdc
      gcloud config set dataproc/region us-central1

      cd hail
      make test-dataproc-38 HAIL_BUILD_MODE=Release \
          -o $(ls build/deploy/dist/hail-*-py3-none-any.whl)
    dependsOn:
      - ci_utils_image
      - default_ns
      - build_hail_jar_and_wheel
      - test_install_in_isolation_python3_12
      - upload_temporary_hailctl_artifacts
    inputs:
      - from: /repo
        to: /io/repo
      - from: /derived/release/hail/build/deploy/dist
        to: /io/repo/hail/build/deploy/dist
    secrets:
      - name: test-dataproc-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dataproc-service-account-key
    scopes:
      - deploy
      - dev
    forceIfLabeled:
      - run-dataproc-tests
    onlyIfRelease: true
    clouds:
      - gcp
  - kind: runImage
    name: mirror_hailgenetics_images
    image: quay.io/skopeo/stable:v1.13.2
    script: |
      set -ex

      REGISTRY={{ global.docker_prefix.split('/')[0] }}

      set +x
      {% if global.cloud == "gcp" %}
      cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
      {% else %}
      echo "unknown cloud {{ global.cloud }}"
      exit 1
      {% endif %}
      set -x

      cd /io/docker/hailgenetics
      export HAIL_PIP_VERSION=$(cat /io/HAIL_PIP_VERSION)
      export DOCKER_PREFIX={{ global.docker_prefix }}
      {% if global.dockerhub_prefix %}
      export DOCKERHUB_PREFIX={{ global.dockerhub_prefix }}
      {% endif %}
      bash mirror_images.sh
    inputs:
      - from: /repo/hail/env/HAIL_PIP_VERSION
        to: /io/HAIL_PIP_VERSION
      - from: /repo/docker/hailgenetics/mirror_images.sh
        to: /io/docker/hailgenetics/mirror_images.sh
      - from: /repo/docker/copy_image.sh
        to: /io/docker/copy_image.sh
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - merge_code
  - kind: buildImage2
    name: website_image
    dockerFile: /io/repo/website/Dockerfile
    contextPath: /io/repo
    publishAs: website
    inputs:
      - from: /repo/website
        to: /io/repo/website
      - from: /docs.tar.gz
        to: /io/repo/docs.tar.gz
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_ubuntu_image
      - get_pip_versioned_docs
    clouds:
      - gcp
    resources:
      storage: 20Gi
      cpu: "2"
      memory: standard
  - kind: deploy
    name: deploy_website
    namespace:
      valueFrom: default_ns.name
    config: website/deployment.yaml
    wait:
      - kind: Service
        name: www
        for: alive
    dependsOn:
      - default_ns
      - website_image
      - create_certs
      - create_session_key
      - create_deploy_config
    clouds:
      - gcp
  - kind: runImage
    name: test_website
    scopes:
      - test
      - dev
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      hailctl curl {{ default_ns.name }} www / \
              -vvv \
              -fsSL \
              --retry 3 \
              --retry-delay 5
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - create_accounts
      - hailgenetics_hailtop_image
      - deploy_website
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_scala_fs
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
      storage: 6Gi
    script: |
      set -ex

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export HAIL_FS_TEST_CLOUD_RESOURCES_URI={{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/fs
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}

      cd /io/repo/hail

      tar -C /io -xzf /io/mill-cache.tgz

      export HAIL_BUILD_MODE='CI'
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      set +e
      sh mill --no-daemon -j2 hail.test.test-fs
      exit_code=$?
      set -e
      if [[ $exit_code -eq 2 ]]
      then
          echo "some tests were skipped, but exiting success anyway"
          exit 0
      else
          exit $exit_code
      fi
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/.git
        to: /io/repo/.git
      - from: /derived/debug/hail/out
        to: /io/repo/hail/out
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    timeout: 1200
    dependsOn:
      - default_ns
      - create_certs
      - create_accounts
      - hail_run_image
      - build_debug_hail_test
      - build_hail_test_artifacts
      - upload_test_resources_to_blob_storage
      - download_mill_and_maven_deps
  - kind: runImage
    name: test_hail_services_java
    scopes:
      - test
      - dev
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
      storage: 6Gi
    script: |
      set -ex

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

      cd /io/repo/hail

      tar -C /io -xzf /io/mill-cache.tgz

      export HAIL_BUILD_MODE='CI'
      export COURSIER_CACHE='/io/cache/coursier/v1'
      export COURSIER_ARCHIVE_CACHE='/io/cache/coursier/arc'
      export MILL_DOWNLOAD_PATH=/io/cache/mill
      sh mill --no-daemon -j2 hail.test.test-services
    inputs:
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/.git
        to: /io/repo/.git
      - from: /derived/debug/hail/out
        to: /io/repo/hail/out
      - from: /mill-cache.tgz
        to: /io/mill-cache.tgz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    timeout: 1200
    dependsOn:
      - default_ns
      - create_certs
      - create_accounts
      - hail_run_image
      - build_debug_hail_test
      - build_hail_test_artifacts
      - upload_query_jar
      - upload_spark_jars
      - deploy_batch
      - download_mill_and_maven_deps
  - kind: runImage
    name: release
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      cd /io

      gcloud auth activate-service-account --key-file=/ci-deploy-0-1--hail-is-hail/ci-deploy-0-1--hail-is-hail.json

      gcloud auth -q configure-docker {{ global.docker_prefix.split('/')[0] }}
      cat /docker-hub-hailgenetics/password | skopeo login --username hailgenetics --password-stdin docker.io

      cp /pypi-credentials/pypirc $HOME/.pypirc
      printf 'Authorization: token ' > github-oauth
      cat /hail-ci-0-1-github-oauth-token/oauth-token >>github-oauth
      printf '#!/bin/bash\necho ' > git-askpass
      cat /hail-ci-0-1-github-oauth-token/oauth-token >>git-askpass
      chmod 755 git-askpass
      export GIT_ASKPASS=/io/git-askpass

      cd /io/repo/hail
      export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION)

      # #14452 - must build wheel with release `hailtop/hailctl/deploy.yaml`
      # use `-o` to prevent `mill` from rebuilding the "SHADOW_JAR"
      make upload-artifacts HAIL_BUILD_MODE=Release DEPLOY_REMOTE=origin \
        -o out/hail/assembly.dest/out.jar

      echo Setting arguments and invoking release.sh.

      HAIL_VERSION=$(cat env/HAIL_VERSION) \
      WHEEL="build/deploy/dist/hail-${HAIL_PIP_VERSION}-py3-none-any.whl" \
      GIT_VERSION=$(cat env/REVISION) \
      REMOTE=origin \
      GITHUB_OAUTH_HEADER_FILE=/io/github-oauth \
      HAIL_GENETICS_HAIL_IMAGE=docker://{{ hailgenetics_hail_image.image }} \
      HAIL_GENETICS_HAIL_IMAGE_PY_3_12=docker://{{ hailgenetics_hail_image.image }} \
      HAIL_GENETICS_HAIL_IMAGE_PY_3_13=docker://{{ hailgenetics_hail_image_python_3_13.image }} \
      HAIL_GENETICS_HAIL_IMAGE_PY_3_14=docker://{{ hailgenetics_hail_image_python_3_14.image }} \
      HAIL_GENETICS_HAILTOP_IMAGE=docker://{{ hailgenetics_hailtop_image.image }} \
      HAIL_GENETICS_VEP_GRCH37_85_IMAGE=docker://{{ hailgenetics_vep_grch37_85_image.image }} \
      HAIL_GENETICS_VEP_GRCH38_95_IMAGE=docker://{{ hailgenetics_vep_grch38_95_image.image }} \
      WEBSITE_TAR=/io/www.tar.gz \
      bash scripts/release.sh
    inputs:
      - from: /repo
        to: /io/repo
      - from: /derived/release/hail/out/hail/assembly.dest/out.jar
        to: /io/repo/hail/out/hail/assembly.dest/out.jar
      - from: /www.tar.gz
        to: /io/www.tar.gz
    secrets:
      - name: pypi-credentials
        namespace:
          valueFrom: default_ns.name
        mountPath: /pypi-credentials
      - name: ci-deploy-0-1--hail-is-hail
        namespace:
          valueFrom: default_ns.name
        mountPath: /ci-deploy-0-1--hail-is-hail
      - name: docker-hub-hailgenetics
        namespace:
          valueFrom: default_ns.name
        mountPath: /docker-hub-hailgenetics
      - name: hail-ci-0-1-github-oauth-token
        namespace:
          valueFrom: default_ns.name
        mountPath: /hail-ci-0-1-github-oauth-token
    onlyIfRelease: true
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - ci_utils_image
      - merge_code
      - hailgenetics_hail_image
      - hailgenetics_hail_image_python_3_13
      - hailgenetics_hail_image_python_3_14
      - hailgenetics_hailtop_image
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
      - build_hail_jar_and_wheel
      - test_hail_java
      - test_hail_python
      - test_hailtop_python
      - test_hail_python_local_backend
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_gcp_vep
      - test_hail_python_service_backend_gcp_fs
      - test_hail_scala_fs
      - test_hail_services_java
      - test_hail_spark_conf_requester_pays_parsing
      - test_dataproc-37
      - test_dataproc-38
      - make_docs
    clouds:
      - gcp
  - kind: runImage
    name: start_hail_benchmark
    onlyIfRelease: true
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
    dependsOn:
      - default_ns
      - hail_dev_image
      - release
    image:
      valueFrom: hail_dev_image.image
    inputs:
      - from: /repo/hail/python
        to: /io/repo/hail/python
      - from: /repo/hail/env
        to: /io/repo/hail/env
      - from: /repo/hail/scripts/benchmark_in_batch.py
        to: /io/repo/hail/scripts/benchmark_in_batch.py
    script: |
      set -ex

      pushd /io/repo/hail

      BUCKET=hail-benchmarks-2
      HAIL_VERSION=$(cat env/HAIL_VERSION)
      RESULTS_PREFIX="gs://${BUCKET}/{{ scope }}/${HAIL_VERSION}"

      export HAIL_BATCH_REGIONS={{ global.gcp_region }}
      export HAIL_BATCH_BILLING_PROJECT=benchmark
      export HAIL_BATCH_REMOTE_TMPDIR={{ ci_storage_uri }}/{{ token }}

      PYTHONPATH="$PWD/python" python3 ./scripts/benchmark_in_batch.py submit \
          {{ hail_dev_image.image }} \
          $RESULTS_PREFIX \
          $HAIL_VERSION

  - kind: runImage
    name: cancel_all_running_test_batches
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json

      cat >cancel_all_running_test_batches.py <<'EOF'
      from hailtop.batch_client.aioclient import BatchClient
      import asyncio
      import traceback

      async def cancel_all(query):
          bc = await BatchClient.create('test')
          async for b in bc.list_batches(query):
              status = await b.last_known_status()
              print(status)
              try:
                  await b.cancel()
              except:
                  traceback.print_exc()

      asyncio.get_event_loop().run_until_complete(cancel_all('user:test running'))
      EOF

      python3 cancel_all_running_test_batches.py
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    alwaysRun: true
    timeout: 300
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - hailgenetics_hailtop_image
      - deploy_batch
    after:
      - test_batch
      - test_batch_job_private_machines
      - test_ci
      - test_hailtop_python
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_gcp_vep
      - test_hail_python_service_backend_gcp_fs
      - test_hail_services_java
      - test_batch_docs
      - test_hailctl_batch
  - kind: runImage
    name: test_batch_invariants
    image:
      valueFrom: hail_dev_image.image
    script: |
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      python3 -m pytest \
              --color=yes \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              --timeout=120 \
              /io/test/test_invariants.py
    inputs:
      - from: /repo/batch/test
        to: /io/test
    timeout: 300
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    scopes:
      - test
      - dev
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - hail_dev_image
      - deploy_batch
      - cancel_all_running_test_batches
  - kind: runImage
    name: delete_gcp_batch_instances
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    alwaysRun: true
    script: |
      set -ex
      gcloud -q auth activate-service-account --key-file=/batch-gsa-key/key.json
      set +e
      kubectl -n {{ default_ns.name }} scale deployment batch-driver --replicas=0
      gcloud -q compute instances list \
          --filter 'tags.items=batch2-agent AND labels.namespace={{ default_ns.name }}' \
          --format="table[no-heading](zone.basename(), name)" \
          --project {{ global.gcp_project }} \
        | xargs -P8 -n2 -r sh -c 'gcloud -q compute instances delete --zone "$1" --project {{ global.gcp_project }} "$2" || true' argv0
      gcloud -q compute disks list \
          --filter 'labels.batch=1 AND labels.namespace={{ default_ns.name }}' \
          --format="table[no-heading](zone.basename(), name)" \
          --project {{ global.gcp_project }} \
        | xargs -P8 -n2 -r sh -c 'gcloud -q compute disks delete --zone "$1" --project {{ global.gcp_project }} "$2" || true' argv0
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    scopes:
      - dev
      - test
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    clouds:
      - gcp
    dependsOn:
      - default_ns
      - ci_utils_image
      - cancel_all_running_test_batches
    after:
      - test_batch_invariants
      - test_batch
      - test_ci
      - test_hailtop_python
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_gcp_vep
      - test_hail_python_service_backend_gcp_fs
  - kind: runImage
    name: setup_dev_namespace_autoscaledown
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      {% for user in code.get("developers", []) %}
      {% if user['username'] != 'test-dev' %}

      if kubectl get namespace {{ user["username"] }} 2>/dev/null
      then
      cat > the.yaml <<'EOF'
      apiVersion: batch/v1
      kind: CronJob
      metadata:
        name: dev-namespace-scaledown-{{ user["username"] }}
        namespace: {{ user["username"] }}
      spec:
        schedule: "0 0 * * *"  # Every day at midnight UTC
        concurrencyPolicy: Forbid
        successfulJobsHistoryLimit: 1
        failedJobsHistoryLimit: 1
        jobTemplate:
          spec:
            template:
              spec:
                serviceAccountName: admin
                containers:
                - name: dev-namespace-scaledown-{{ user["username"] }}
                  image: bitnami/kubectl:latest
                  command:
                  - /bin/sh
                  - -c
                  - set -ex ; kubectl scale deployments --all -n {{ user["username"] }} --replicas=0 && kubectl scale statefulsets --all -n {{ user["username"] }} --replicas=0
                restartPolicy: OnFailure
      EOF

      kubectl apply -f the.yaml
      echo "For {{ user["username"] }}, applied:"
      echo "---"
      cat the.yaml
      else
      echo "Skipping {{ user["username"] }} because they have no namespace."
      fi

      {% endif %}
      {% endfor %}
    scopes:
      - deploy
    serviceAccount:
      name: ci-agent
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - ci_utils_image
      - default_ns
  - kind: runImage
    name: test_gcp_ar_cleanup_policies
    scopes:
      - test
      - dev
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      cd /io/repo/
      new_policy=$(mktemp)
      python3 devbin/generate_gcp_ar_cleanup_policy.py > $new_policy
      diff $new_policy infra/gcp-broad/gcp-ar-cleanup-policy.txt || {
        echo '>>> AR cleanup policy has changed, new policy is <<<'
        cat $new_policy
        echo '--------------------------------------'
        echo "Please regenerate the policy file by running:"
        echo "> python3 devbin/generate_gcp_ar_cleanup_policy.py > infra/gcp-broad/gcp-ar-cleanup-policy.txt"
        exit 1
      }
    inputs:
      - from: /repo/build.yaml
        to: /io/repo/build.yaml
      - from: /repo/devbin
        to: /io/repo/devbin
      - from: /repo/infra
        to: /io/repo/infra
      - from: /repo/docker
        to: /io/repo/docker
      - from: /repo/ci/test/resources
        to: /io/repo/ci/test/resources
    clouds:
      - gcp
    dependsOn:
      - ci_utils_image
      - default_ns
  - kind: runImage
    name: set_gcp_ar_cleanup_policies
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    scopes:
      - deploy
    script: |
      gcloud artifacts repositories set-cleanup-policies hail \
        --project={{ global.gcp_project }} \
        --location=us \
        --policy=/io/repo/infra/gcp-broad/gcp-ar-cleanup-policy.txt \
        --no-dry-run
    inputs:
      - from: /repo/infra
        to: /io/repo/infra
    clouds:
      - gcp
    dependsOn:
      - test_gcp_ar_cleanup_policies
      - ci_utils_image