diff --git a/.github/workflows/cd-deploy.yaml b/.github/workflows/cd-deploy.yaml index 3f1eb6d69..ca0367f8d 100644 --- a/.github/workflows/cd-deploy.yaml +++ b/.github/workflows/cd-deploy.yaml @@ -69,6 +69,9 @@ jobs: # Check if staff helm chart exists helm status bt-staff &>/dev/null && staff_status=true || staff_status=false + # Delete deployment if selector labels changed (immutable field) + kubectl delete deployment bt-staff-frontend -n bt --ignore-not-found=true + # Upgrade staff helm chart, or install if not exists helm upgrade bt-staff oci://registry-1.docker.io/octoberkeleytime/bt-staff \ --install \ diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml index efd745d8f..095d8e625 100644 --- a/.github/workflows/cd-dev.yaml +++ b/.github/workflows/cd-dev.yaml @@ -83,6 +83,7 @@ jobs: environment: development name: bt-dev-app-${{ needs.compute-sha.outputs.sha_short }} version: 0.1.0-dev-${{ needs.compute-sha.outputs.sha_short }} + deploy_staff: false values: | env: dev ttl: ${{ inputs.ttl }} diff --git a/.github/workflows/datapuller-trigger.yaml b/.github/workflows/datapuller-trigger.yaml new file mode 100644 index 000000000..9abd510e6 --- /dev/null +++ b/.github/workflows/datapuller-trigger.yaml @@ -0,0 +1,64 @@ +name: "Datapuller: Manual Trigger" +run-name: "Datapuller: ${{ github.event.inputs.job }} triggered by ${{ github.actor }}" + +on: + workflow_dispatch: + inputs: + job: + description: "Select which datapuller job to run" + required: true + type: choice + options: + - terms-all + - terms-nearby + - courses + - sections-active + - sections-l5y + - classes-active + - classes-l5y + - grades-recent + - grades-l5y + - enrollments + - enrollment-timeframe + +jobs: + trigger: + name: Trigger and Stream Logs + runs-on: ubuntu-latest + + steps: + - name: SSH, Trigger Job, and Stream Logs + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USERNAME }} + key: ${{ secrets.SSH_KEY }} + command_timeout: 25m + script: | + set -e + + # define variables + CRONJOB="bt-prod-app-datapuller-${{ github.event.inputs.job }}" + JOB_NAME="${CRONJOB}-manual-$(date +%s)" + NAMESPACE="bt" + + # set timeout based on job type + case "${{ github.event.inputs.job }}" in grades-l5y|sections-l5y|classes-l5y) + TIMEOUT=1200s ;; # 20 min + *) + TIMEOUT=600s ;; # 10 min + esac + + echo "=== Creating job ${JOB_NAME} from cronjob ${CRONJOB} ===" + kubectl create job "${JOB_NAME}" --from="cronjob/${CRONJOB}" -n "${NAMESPACE}" + + echo "=== Waiting for pod to start ===" + kubectl wait --for=condition=ready pod -l job-name="${JOB_NAME}" -n "${NAMESPACE}" --timeout=120s || true + + echo "=== Streaming logs ===" + kubectl logs -n "${NAMESPACE}" -l job-name="${JOB_NAME}" --follow || true + + echo "=== Waiting for job to complete ===" + kubectl wait --for=condition=complete job/"${JOB_NAME}" -n "${NAMESPACE}" --timeout="${TIMEOUT}" + + echo "=== Job completed successfully ===" \ No newline at end of file diff --git a/apps/backend/package.json b/apps/backend/package.json index 3e01b1ea9..0bcafd18f 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -44,6 +44,16 @@ "@graphql-tools/schema": "^10.0.25", "@graphql-tools/utils": "^10.9.1", "@keyv/redis": "^5.1.2", + "@kubernetes/client-node": "^1.4.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/auto-instrumentations-node": "^0.53.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.56.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.56.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.56.0", + "@opentelemetry/resources": "^1.29.0", + "@opentelemetry/sdk-logs": "^0.56.0", + "@opentelemetry/sdk-metrics": "1.29.0", + "@opentelemetry/sdk-node": "^0.56.0", "@repo/common": "*", "@repo/gql-typedefs": "*", "@repo/shared": "*", @@ -65,8 +75,9 @@ "papaparse": "^5.5.3", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", + "pino": "^9.6.0", "redis": "^5.8.3", - "uuid": "^13.0.0", - "undici": "^7.14.0" + "undici": "^7.14.0", + "uuid": "^13.0.0" } } diff --git a/git-rebase-todo b/git-rebase-todo new file mode 100644 index 000000000..de8c1cf08 --- /dev/null +++ b/git-rebase-todo @@ -0,0 +1,42 @@ +drop df5cf8460 # feat: Implement OpenTelemetry-based observability stack +pick 46aabe3c0 # datapuller button test +pick ea932f783 # added status for data puller +pick 6bef7b42f # fix: resolve circular data reference in useDatapullerJobStatus +pick 1343d049a # package updates +pick bf5aa310a # package changes +pick 27900b265 # changed to stable selectors +pick 8ca60d940 # revert: disable deploy_staff in dev workflow +pick ecb2c3d22 # staff dashboard test in dev +pick a5c2072bb # staff dashboard deployment in dev +pick 9d8a1e6d6 # don't deploy staff dashboard + +# Rebase 57fb92c94..9d8a1e6d6 onto 57fb92c94 (11 commands) +# +# Commands: +# p, pick = use commit +# r, reword = use commit, but edit the commit message +# e, edit = use commit, but stop for amending +# s, squash = use commit, but meld into previous commit +# f, fixup [-C | -c] = like "squash" but keep only the previous +# commit's log message, unless -C is used, in which case +# keep only this commit's message; -c is same as -C but +# opens the editor +# x, exec = run command (the rest of the line) using shell +# b, break = stop here (continue rebase later with 'git rebase --continue') +# d, drop = remove commit +# l, label