From 92782178604304d6e0ede04d9a149cd484c496af Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Thu, 18 Sep 2025 14:58:34 -0400 Subject: [PATCH 01/19] cancel ci on pre-commit failure Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 49c3d593..68433c58 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -40,6 +40,19 @@ steps: - exit_status: -10 # Agent was lost limit: 5 + - label: ":pre-commit: Pre-commit checks" + key: pre-commit-checks + agents: + queue: cpu_queue_premerge + command: | + pip install pre-commit + pre-commit install + pre-commit run --all-files --hook-stage manual + if [ $? -ne 0 ]; then + echo "--- :rotating_light: pre-commit checks failed! Canceling the build. :rotating_light:" + buildkite-agent build cancel + fi + - block: Run Neuron Test depends_on: ~ key: run-neuron-test From 800923909d939b878d103d1f0f8a95dc122dee34 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Thu, 18 Sep 2025 15:21:52 -0400 Subject: [PATCH 02/19] fix issue Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 68433c58..62c434e0 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -49,7 +49,7 @@ steps: pre-commit install pre-commit run --all-files --hook-stage manual if [ $? -ne 0 ]; then - echo "--- :rotating_light: pre-commit checks failed! Canceling the build. :rotating_light:" + echo "pre-commit checks failed. Canceling the build." buildkite-agent build cancel fi From f9ab67b24b496888678263da96895d2c254251d2 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Thu, 18 Sep 2025 15:26:36 -0400 Subject: [PATCH 03/19] try yaml list Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 62c434e0..1dd337ce 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -44,14 +44,10 @@ steps: key: pre-commit-checks agents: queue: cpu_queue_premerge - command: | - pip install pre-commit - pre-commit install - pre-commit run --all-files --hook-stage manual - if [ $? -ne 0 ]; then - echo "pre-commit checks failed. Canceling the build." - buildkite-agent build cancel - fi + commands: + - pip install pre-commit + - pre-commit install + - pre-commit run --all-files --hook-stage manual || (echo "pre-commit checks failed. Canceling the build." && buildkite-agent build cancel) - block: Run Neuron Test depends_on: ~ From 5e847c9bb44339368298fb40ee85e6f6e7fe9718 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Thu, 18 Sep 2025 15:37:40 -0400 Subject: [PATCH 04/19] add bin to path Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 1dd337ce..66926c0b 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -46,6 +46,7 @@ steps: queue: cpu_queue_premerge commands: - pip install pre-commit + - export PATH="$HOME/.local/bin:$PATH" - pre-commit install - pre-commit run --all-files --hook-stage manual || (echo "pre-commit checks failed. Canceling the build." && buildkite-agent build cancel) From ab9fa1096413638295389409a720406c274e56c3 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 10:14:37 -0400 Subject: [PATCH 05/19] bk agent pipeline cancel Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 66926c0b..adeee131 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -48,7 +48,7 @@ steps: - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" - pre-commit install - - pre-commit run --all-files --hook-stage manual || (echo "pre-commit checks failed. Canceling the build." && buildkite-agent build cancel) + - pre-commit run --all-files --hook-stage manual || (echo "pre-commit checks failed. Canceling the build." && buildkite-agent pipeline cancel) - block: Run Neuron Test depends_on: ~ From 46698453891e6544f68033221d2451c844a5227b Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 10:15:07 -0400 Subject: [PATCH 06/19] remove pre-commit install Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index adeee131..dc1c7c34 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -47,7 +47,6 @@ steps: commands: - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" - - pre-commit install - pre-commit run --all-files --hook-stage manual || (echo "pre-commit checks failed. Canceling the build." && buildkite-agent pipeline cancel) - block: Run Neuron Test From e5cd4471760bde99afd66a8397c1424f7f8c5b37 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 10:29:49 -0400 Subject: [PATCH 07/19] cancel on build failing Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index dc1c7c34..e21d73aa 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -42,12 +42,14 @@ steps: - label: ":pre-commit: Pre-commit checks" key: pre-commit-checks + cancel_on_build_failing: true agents: queue: cpu_queue_premerge commands: + - exit 1 - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" - - pre-commit run --all-files --hook-stage manual || (echo "pre-commit checks failed. Canceling the build." && buildkite-agent pipeline cancel) + - pre-commit run --all-files --hook-stage manual - block: Run Neuron Test depends_on: ~ From e0de60357403984969a38945bb0e4d62406933ec Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 10:53:37 -0400 Subject: [PATCH 08/19] use api call Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index e21d73aa..b1105541 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -46,6 +46,7 @@ steps: agents: queue: cpu_queue_premerge commands: + - curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" From 0dc88b7ac47a6d7092bf964ab33e0190056f64d2 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 11:04:10 -0400 Subject: [PATCH 09/19] add escape character Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index b1105541..2b75234b 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -46,7 +46,7 @@ steps: agents: queue: cpu_queue_premerge commands: - - curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" + - curl -H \"Authorization\: Bearer $BUILDKITE_API_TOKEN\" -X PUT \"https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel\" - exit 1 - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" From 7799bf5aeff1e050b36082d2c119c8fef4aa91ef Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 11:14:49 -0400 Subject: [PATCH 10/19] use as is Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 2b75234b..ba8bfb26 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -46,7 +46,8 @@ steps: agents: queue: cpu_queue_premerge commands: - - curl -H \"Authorization\: Bearer $BUILDKITE_API_TOKEN\" -X PUT \"https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel\" + - curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \ + -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" From 260300b451a3a4d13b2f1ce1f52a8896ac41b180 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 11:29:30 -0400 Subject: [PATCH 11/19] use basd Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index ba8bfb26..c071c31b 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -46,7 +46,9 @@ steps: agents: queue: cpu_queue_premerge commands: - - curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \ + - | + #!/bin/bash + curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit From 341f7eb0cd95fc576eb0341f38c509e4b2380de9 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 11:36:09 -0400 Subject: [PATCH 12/19] use diff env var Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index c071c31b..22da15de 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -48,7 +48,7 @@ steps: commands: - | #!/bin/bash - curl -H "Authorization: Bearer $BUILDKITE_TOKEN" \ + curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit From c32bc67d29d154fe336d5de142d539ab80690ce2 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 12:41:58 -0400 Subject: [PATCH 13/19] use BUILDKITE_AGENT_TOKEN Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 22da15de..df4b7e14 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -48,7 +48,7 @@ steps: commands: - | #!/bin/bash - curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + curl -H "Authorization: Bearer $BUILDKITE_AGENT_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit From d422f6b93d87ac9fef792ab97ab85c10f86e3b5d Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 12:47:07 -0400 Subject: [PATCH 14/19] read env vars Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index df4b7e14..2616c968 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -48,7 +48,7 @@ steps: commands: - | #!/bin/bash - curl -H "Authorization: Bearer $BUILDKITE_AGENT_TOKEN" \ + curl -H "Authorization: Bearer $BUILDKITE_AGENT_JOB_API_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit From 3095178e8b23d84b9d03b389fecf22a27d181dc4 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 12:53:00 -0400 Subject: [PATCH 15/19] read env Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 2616c968..df91d5ee 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -48,6 +48,7 @@ steps: commands: - | #!/bin/bash + env curl -H "Authorization: Bearer $BUILDKITE_AGENT_JOB_API_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 From 80a8cd660b0b9c8cc854fad2be98dbecb24dc21f Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 12:56:46 -0400 Subject: [PATCH 16/19] use BUILDKITE_AGENT_ACCESS_TOKEN Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index df91d5ee..71dc50b9 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -49,7 +49,7 @@ steps: - | #!/bin/bash env - curl -H "Authorization: Bearer $BUILDKITE_AGENT_JOB_API_TOKEN" \ + curl -H "Authorization: Bearer $BUILDKITE_AGENT_ACCESS_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - pip install pre-commit From 78dd33f7ed962df71ed4db49ed55a6e8f9049139 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 14:37:57 -0400 Subject: [PATCH 17/19] cancel if pre-commit fails Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 71dc50b9..a99dd167 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -46,15 +46,17 @@ steps: agents: queue: cpu_queue_premerge commands: - - | - #!/bin/bash - env - curl -H "Authorization: Bearer $BUILDKITE_AGENT_ACCESS_TOKEN" \ - -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - - exit 1 - pip install pre-commit - export PATH="$HOME/.local/bin:$PATH" - pre-commit run --all-files --hook-stage manual + - | + #!/bin/bash + if [[ $? -ne 0 ]]; then + echo "Pre-commit checks failed. Canceling the build." + curl -H "Authorization: Bearer $BUILDKITE_AGENT_ACCESS_TOKEN" \ + -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" + exit 1 + fi - block: Run Neuron Test depends_on: ~ From 80feba5110ed3398dcf18e8aaa71ed6a042520c3 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 14:44:10 -0400 Subject: [PATCH 18/19] fix Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index a99dd167..1ec6cbb0 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -45,18 +45,16 @@ steps: cancel_on_build_failing: true agents: queue: cpu_queue_premerge - commands: - - pip install pre-commit - - export PATH="$HOME/.local/bin:$PATH" - - pre-commit run --all-files --hook-stage manual - - | - #!/bin/bash - if [[ $? -ne 0 ]]; then - echo "Pre-commit checks failed. Canceling the build." - curl -H "Authorization: Bearer $BUILDKITE_AGENT_ACCESS_TOKEN" \ - -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" - exit 1 - fi + command: | + pip install pre-commit + export PATH="$HOME/.local/bin:$PATH" + pre-commit run --all-files --hook-stage manual + if [ $? -ne 0 ]; then + echo "Pre-commit checks failed. Canceling the build." + curl -H "Authorization: Bearer $BUILDKITE_AGENT_ACCESS_TOKEN" \ + -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel" + exit 1 + fi - block: Run Neuron Test depends_on: ~ From 298a1a68611f00d860cd46e420e569a13974beb0 Mon Sep 17 00:00:00 2001 From: Tahmid Muttaki Date: Fri, 19 Sep 2025 14:48:26 -0400 Subject: [PATCH 19/19] use exit code Signed-off-by: Tahmid Muttaki --- buildkite/test-template-fastcheck.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildkite/test-template-fastcheck.j2 b/buildkite/test-template-fastcheck.j2 index 1ec6cbb0..4d9301ad 100644 --- a/buildkite/test-template-fastcheck.j2 +++ b/buildkite/test-template-fastcheck.j2 @@ -48,8 +48,7 @@ steps: command: | pip install pre-commit export PATH="$HOME/.local/bin:$PATH" - pre-commit run --all-files --hook-stage manual - if [ $? -ne 0 ]; then + if ! pre-commit run --all-files --hook-stage manual; then echo "Pre-commit checks failed. Canceling the build." curl -H "Authorization: Bearer $BUILDKITE_AGENT_ACCESS_TOKEN" \ -X PUT "https://api.buildkite.com/v2/organizations/$BUILDKITE_ORGANIZATION_SLUG/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER/cancel"