From 03b69522462373bb3e89f006d8ab2f4ddd14f2c3 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 13:08:04 +1000
Subject: [PATCH 01/31] feat: Upload concurrency

---
 hooks/pre-exit | 31 +++++++++++++++++++++++--------
 plugin.yml     |  2 ++
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index a30296e..f3613cd 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -6,6 +6,7 @@ FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"
 TIMEOUT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT:-30}"
 BASE_PATH="${BUILDKITE_PLUGIN_TEST_COLLECTOR_BASE_PATH:-.}"
 ANNOTATE="${BUILDKITE_PLUGIN_TEST_COLLECTOR_ANNOTATION_LINK:-false}"
+UPLOAD_CONCURRENCY="${BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY:-10}"
 REPORT_URLS_FILE=$(mktemp)
 CURL_RESP_FILE="${CURL_RESP_FILE:-response.json}"
 DEBUG="false"
@@ -160,16 +161,30 @@ find_and_upload() {
       exit "${BUILDKITE_PLUGIN_TEST_COLLECTOR_MISSING_ERROR:-1}"
     fi
   else
+    uploads_in_progress=0
     # needs to be part of else for bash4.3 compatibility
     for file in "${matching_files[@]}"; do
-      echo "Uploading '$file'..."
-      if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
-        echo "Error uploading, will continue"
-      fi
-      if [[ "$ANNOTATE" != "false" ]]; then
-        save-report-url "${CURL_RESP_FILE}"
-      fi
+      while [[ $uploads_in_progress -ge $UPLOAD_CONCURRENCY ]]; do
+        sleep 1
+      done
+
+      ((uploads_in_progress++))
+
+      (
+        echo "Uploading '$file'..."
+        if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
+          echo "Error uploading, will continue"
+        fi
+        if [[ "$ANNOTATE" != "false" ]]; then
+          save-report-url "${CURL_RESP_FILE}"
+        fi
+        
+        ((uploads_in_progress--))
+      ) &
     done
+
+    # Wait for all uploads to finish
+    wait
   fi
 }
 
@@ -194,4 +209,4 @@ else
 fi
 if [ "$ANNOTATE" != "false" ]; then
   annotation-link "${REPORT_URLS_FILE}"
-fi
\ No newline at end of file
+fi
diff --git a/plugin.yml b/plugin.yml
index 46aa67e..ee572c5 100644
--- a/plugin.yml
+++ b/plugin.yml
@@ -35,6 +35,8 @@ configuration:
       type: integer
     annotation-link:
       type: boolean
+    concurrent-uploads:
+      type: integer
   required:
     - files
     - format

From a5f54538779eb434374019970a517e349bc95fd0 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 13:37:18 +1000
Subject: [PATCH 02/31] fix: Add debug logging

---
 hooks/pre-exit | 3 ++-
 plugin.yml     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index f3613cd..c854972 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+set -euox pipefail
 
 TOKEN_ENV_NAME="${BUILDKITE_PLUGIN_TEST_COLLECTOR_API_TOKEN_ENV_NAME:-BUILDKITE_ANALYTICS_TOKEN}"
 FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"
@@ -162,6 +162,7 @@ find_and_upload() {
     fi
   else
     uploads_in_progress=0
+    echo "Starting file uploads"
     # needs to be part of else for bash4.3 compatibility
     for file in "${matching_files[@]}"; do
       while [[ $uploads_in_progress -ge $UPLOAD_CONCURRENCY ]]; do
diff --git a/plugin.yml b/plugin.yml
index ee572c5..4ff1855 100644
--- a/plugin.yml
+++ b/plugin.yml
@@ -35,7 +35,7 @@ configuration:
       type: integer
     annotation-link:
       type: boolean
-    concurrent-uploads:
+    upload-concurrency:
       type: integer
   required:
     - files

From 565ea3bbe8e78506f2f701f3579b5a34f551cb83 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 13:59:52 +1000
Subject: [PATCH 03/31] fix: Use command grouping so output is captured

---
 hooks/pre-exit | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index c854972..fc0aa9e 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -171,7 +171,7 @@ find_and_upload() {
 
       ((uploads_in_progress++))
 
-      (
+      {
         echo "Uploading '$file'..."
         if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
           echo "Error uploading, will continue"
@@ -181,7 +181,7 @@ find_and_upload() {
         fi
         
         ((uploads_in_progress--))
-      ) &
+       } &
     done
 
     # Wait for all uploads to finish

From c8aea0b6c1ca2fd7225b43f79dbd69de5d515b93 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 14:00:16 +1000
Subject: [PATCH 04/31] fix: intentation

---
 hooks/pre-exit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index fc0aa9e..21865fa 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -181,7 +181,7 @@ find_and_upload() {
         fi
         
         ((uploads_in_progress--))
-       } &
+      } &
     done
 
     # Wait for all uploads to finish

From 5cab3053feddaaac9c388e85b4b41a5d801c9d30 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 14:18:46 +1000
Subject: [PATCH 05/31] chore: Debug line

---
 hooks/pre-exit | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 21865fa..48b87ba 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -171,6 +171,8 @@ find_and_upload() {
 
       ((uploads_in_progress++))
 
+      echo "DEBUG"
+
       {
         echo "Uploading '$file'..."
         if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then

From bde27fc45b70ff197706fc41632747a3ad55cf09 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 14:20:25 +1000
Subject: [PATCH 06/31] chore: Remove tracing

---
 hooks/pre-exit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 48b87ba..3733469 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euox pipefail
+set -euo pipefail
 
 TOKEN_ENV_NAME="${BUILDKITE_PLUGIN_TEST_COLLECTOR_API_TOKEN_ENV_NAME:-BUILDKITE_ANALYTICS_TOKEN}"
 FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"

From 1f8e73946c1ca6c83df077e7a167ae13d9a4fd4f Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 14:45:13 +1000
Subject: [PATCH 07/31] chore: add tracing, remove subshell

---
 hooks/pre-exit | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 3733469..a0a197e 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+set -euox pipefail
 
 TOKEN_ENV_NAME="${BUILDKITE_PLUGIN_TEST_COLLECTOR_API_TOKEN_ENV_NAME:-BUILDKITE_ANALYTICS_TOKEN}"
 FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"
@@ -169,21 +169,19 @@ find_and_upload() {
         sleep 1
       done
 
-      ((uploads_in_progress++))
+      #((uploads_in_progress++))
 
       echo "DEBUG"
 
-      {
-        echo "Uploading '$file'..."
-        if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
-          echo "Error uploading, will continue"
-        fi
-        if [[ "$ANNOTATE" != "false" ]]; then
-          save-report-url "${CURL_RESP_FILE}"
-        fi
-        
-        ((uploads_in_progress--))
-      } &
+      echo "Uploading '$file'..."
+      if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
+        echo "Error uploading, will continue"
+      fi
+      if [[ "$ANNOTATE" != "false" ]]; then
+        save-report-url "${CURL_RESP_FILE}"
+      fi
+      
+      #((uploads_in_progress--))
     done
 
     # Wait for all uploads to finish

From 52de3bf28a23782dc95c667d698b9785eca4eee1 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 14:54:20 +1000
Subject: [PATCH 08/31] chore: change increment operation

---
 hooks/pre-exit | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index a0a197e..4b4efbf 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -169,10 +169,10 @@ find_and_upload() {
         sleep 1
       done
 
-      #((uploads_in_progress++))
+      uploads_in_progress=$((uploads_in_progress + 1))
 
       echo "DEBUG"
-
+      
       echo "Uploading '$file'..."
       if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
         echo "Error uploading, will continue"
@@ -181,7 +181,7 @@ find_and_upload() {
         save-report-url "${CURL_RESP_FILE}"
       fi
       
-      #((uploads_in_progress--))
+      uploads_in_progress=$((uploads_in_progress - 1))
     done
 
     # Wait for all uploads to finish

From 032a0c3e430bf43a22912181bd25a9c5f15017e8 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 14:57:59 +1000
Subject: [PATCH 09/31] chore: Re-add subshell

---
 hooks/pre-exit | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 4b4efbf..a7b9e88 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -169,19 +169,21 @@ find_and_upload() {
         sleep 1
       done
 
+      # Increment using this notation as double-parentheses will result in an exit code other than 0
       uploads_in_progress=$((uploads_in_progress + 1))
 
-      echo "DEBUG"
-      
-      echo "Uploading '$file'..."
-      if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
-        echo "Error uploading, will continue"
-      fi
-      if [[ "$ANNOTATE" != "false" ]]; then
-        save-report-url "${CURL_RESP_FILE}"
-      fi
-      
-      uploads_in_progress=$((uploads_in_progress - 1))
+      # Spawn a subcommand group to allow parallel uploads
+      {
+        echo "Uploading '$file'..."
+        if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
+          echo "Error uploading, will continue"
+        fi
+        if [[ "$ANNOTATE" != "false" ]]; then
+          save-report-url "${CURL_RESP_FILE}"
+        fi
+        
+        uploads_in_progress=$((uploads_in_progress - 1))
+      } &
     done
 
     # Wait for all uploads to finish

From 0d4eaef6d1b9fa1c9812958e78a766009f97ad94 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 15:07:55 +1000
Subject: [PATCH 10/31] chore: Remove tracing

---
 hooks/pre-exit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index a7b9e88..8789963 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euox pipefail
+set -euo pipefail
 
 TOKEN_ENV_NAME="${BUILDKITE_PLUGIN_TEST_COLLECTOR_API_TOKEN_ENV_NAME:-BUILDKITE_ANALYTICS_TOKEN}"
 FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"

From 3ea88ad8096ae1ac5d9953f28fd8105813fe7f7f Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 15:34:56 +1000
Subject: [PATCH 11/31] chore: More debug logs

---
 hooks/pre-exit | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 8789963..17eef5e 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -162,10 +162,11 @@ find_and_upload() {
     fi
   else
     uploads_in_progress=0
-    echo "Starting file uploads"
+    echo "Uploading '${#matching_files[@]}'∂ files matching '${FILES_PATTERN}'"
     # needs to be part of else for bash4.3 compatibility
     for file in "${matching_files[@]}"; do
       while [[ $uploads_in_progress -ge $UPLOAD_CONCURRENCY ]]; do
+        echo "Waiting for uploads to finish..."
         sleep 1
       done
 
@@ -183,6 +184,7 @@ find_and_upload() {
         fi
         
         uploads_in_progress=$((uploads_in_progress - 1))
+        echo "Finished uploading '$file'"
       } &
     done
 

From 56add337fa2992cfeaaaad5034123a5d28a891f8 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 15:49:05 +1000
Subject: [PATCH 12/31] fix: Store pids instead of using counters

---
 hooks/pre-exit | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 17eef5e..3b226c1 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -161,18 +161,15 @@ find_and_upload() {
       exit "${BUILDKITE_PLUGIN_TEST_COLLECTOR_MISSING_ERROR:-1}"
     fi
   else
-    uploads_in_progress=0
+    declare -a uploads_in_progress=()
     echo "Uploading '${#matching_files[@]}'∂ files matching '${FILES_PATTERN}'"
     # needs to be part of else for bash4.3 compatibility
     for file in "${matching_files[@]}"; do
-      while [[ $uploads_in_progress -ge $UPLOAD_CONCURRENCY ]]; do
+      while [[ "${#uploads_in_progress[@]}" -ge $UPLOAD_CONCURRENCY ]]; do
         echo "Waiting for uploads to finish..."
         sleep 1
       done
 
-      # Increment using this notation as double-parentheses will result in an exit code other than 0
-      uploads_in_progress=$((uploads_in_progress + 1))
-
       # Spawn a subcommand group to allow parallel uploads
       {
         echo "Uploading '$file'..."
@@ -182,10 +179,11 @@ find_and_upload() {
         if [[ "$ANNOTATE" != "false" ]]; then
           save-report-url "${CURL_RESP_FILE}"
         fi
-        
-        uploads_in_progress=$((uploads_in_progress - 1))
+
         echo "Finished uploading '$file'"
       } &
+
+      uploads_in_progress+=($!)
     done
 
     # Wait for all uploads to finish

From 0509e90b4ac05dca37440d8a93ffe90ba43a257e Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 15:49:30 +1000
Subject: [PATCH 13/31] fix: accidental char

---
 hooks/pre-exit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 3b226c1..39f822b 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -162,7 +162,7 @@ find_and_upload() {
     fi
   else
     declare -a uploads_in_progress=()
-    echo "Uploading '${#matching_files[@]}'∂ files matching '${FILES_PATTERN}'"
+    echo "Uploading '${#matching_files[@]}' files matching '${FILES_PATTERN}'"
     # needs to be part of else for bash4.3 compatibility
     for file in "${matching_files[@]}"; do
       while [[ "${#uploads_in_progress[@]}" -ge $UPLOAD_CONCURRENCY ]]; do

From 5936cb86cc0317d5d34a918f0304e8d4a1f4a9f3 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 16:08:12 +1000
Subject: [PATCH 14/31] fix: Remove pids no longer running

---
 hooks/pre-exit | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 39f822b..7b32c46 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -167,6 +167,11 @@ find_and_upload() {
     for file in "${matching_files[@]}"; do
       while [[ "${#uploads_in_progress[@]}" -ge $UPLOAD_CONCURRENCY ]]; do
         echo "Waiting for uploads to finish..."
+        for index in "${!uploads_in_progress[@]}"; do
+          if ! ps -p "${uploads_in_progress[index]}" > /dev/null; then
+            unset 'uploads_in_progress[index]'
+          fi
+        done
         sleep 1
       done
 
@@ -187,7 +192,7 @@ find_and_upload() {
     done
 
     # Wait for all uploads to finish
-    wait
+    wait "${uploads_in_progress[@]}"
   fi
 }
 

From 0090dfefe024e1d063474cf02963486945057ce3 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 16:31:15 +1000
Subject: [PATCH 15/31] chore: Change default concurrency back to 1

---
 hooks/pre-exit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 7b32c46..539d8ca 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -6,7 +6,7 @@ FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"
 TIMEOUT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT:-30}"
 BASE_PATH="${BUILDKITE_PLUGIN_TEST_COLLECTOR_BASE_PATH:-.}"
 ANNOTATE="${BUILDKITE_PLUGIN_TEST_COLLECTOR_ANNOTATION_LINK:-false}"
-UPLOAD_CONCURRENCY="${BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY:-10}"
+UPLOAD_CONCURRENCY="${BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY:-1}"
 REPORT_URLS_FILE=$(mktemp)
 CURL_RESP_FILE="${CURL_RESP_FILE:-response.json}"
 DEBUG="false"

From 980cc62076ef7df25dc07a60edf1e7e63317a7e1 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 17:06:43 +1000
Subject: [PATCH 16/31] docs: Add docs & tests

---
 README.md                   |  6 ++++++
 hooks/pre-exit              | 16 ++++++++++++++--
 tests/pre-exit-success.bats | 22 ++++++++++++++++++++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 430965b..f8f2edd 100644
--- a/README.md
+++ b/README.md
@@ -90,6 +90,12 @@ Adds an annotation to the build run with a link to the uploaded report.
 
 Default value: `false`
 
+#### `upload-concurrency`(number)
+
+The number of concurrent file uploads to perform to the Buildkite analytics API.
+
+Default value: `1`
+
 ## Examples
 
 ### Upload a JUnit file
diff --git a/hooks/pre-exit b/hooks/pre-exit
index 539d8ca..c24aa66 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -163,16 +163,27 @@ find_and_upload() {
   else
     declare -a uploads_in_progress=()
     echo "Uploading '${#matching_files[@]}' files matching '${FILES_PATTERN}'"
+    
     # needs to be part of else for bash4.3 compatibility
     for file in "${matching_files[@]}"; do
+      iterations_waited=0
       while [[ "${#uploads_in_progress[@]}" -ge $UPLOAD_CONCURRENCY ]]; do
-        echo "Waiting for uploads to finish..."
+        iterations_waited=$((iterations_waited + 1))
+        if [[ "${DEBUG}" == "true" ]]; then
+          echo "Waiting for uploads to finish..."
+        fi
+
+        sleep 1
+
         for index in "${!uploads_in_progress[@]}"; do
           if ! ps -p "${uploads_in_progress[index]}" > /dev/null; then
             unset 'uploads_in_progress[index]'
+          elif [[ "$iterations_waited" -gt 10 ]]; then
+            echo "Upload '${uploads_in_progress[index]}' has been running for more than 10 seconds, killing it"
+            kill "${uploads_in_progress[index]}"
+            unset 'uploads_in_progress[index]'
           fi
         done
-        sleep 1
       done
 
       # Spawn a subcommand group to allow parallel uploads
@@ -188,6 +199,7 @@ find_and_upload() {
         echo "Finished uploading '$file'"
       } &
 
+      # Store the PID of the upload
       uploads_in_progress+=($!)
     done
 
diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index b4c8890..1f7d814 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -57,6 +57,28 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "curl success 3"
 }
 
+@test "Uploads multiple files concurrently" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
+
+  stub curl \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 2'" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 3'"
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub curl
+
+  assert_success
+  assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
+  assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
+  assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
+  assert_output --partial "curl success 1"
+  assert_output --partial "curl success 2"
+  assert_output --partial "curl success 3"
+}
+
 @test "Single file pattern through array" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES_0='**/*/junit-1.xml'
   unset BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES

From 26ce593bc8f517e8c770f72bbeb7766a9db8fcd6 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 17:12:28 +1000
Subject: [PATCH 17/31] chore: Test tests

---
 tests/pre-exit-success.bats | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 1f7d814..f544d16 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -75,8 +75,8 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
   assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
   assert_output --partial "curl success 1"
-  assert_output --partial "curl success 2"
-  assert_output --partial "curl success 3"
+  assert_output --partial "curl success 1"
+  assert_output --partial "curl success 1"
 }
 
 @test "Single file pattern through array" {

From 9e58efcf77d83ecb0e15ae0d85754cd47f887982 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 17:17:46 +1000
Subject: [PATCH 18/31] chore: Test tests

---
 tests/pre-exit-success.bats | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index f544d16..902f23f 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -63,8 +63,8 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
 
   stub curl \
     "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'" \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 2'" \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 3'"
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'"
 
   run "$PWD/hooks/pre-exit"
 

From 712af760ec5d375b8133877f206b474f6be33c3e Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 17:19:23 +1000
Subject: [PATCH 19/31] chore: Test tests

---
 tests/pre-exit-success.bats | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 902f23f..0b54177 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -62,8 +62,6 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
 
   stub curl \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'" \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'" \
     "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'"
 
   run "$PWD/hooks/pre-exit"

From e6d475d329151b8836dec7f9fe0451529411be47 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 1 Jun 2023 17:31:17 +1000
Subject: [PATCH 20/31] chore: decrease verbosity

---
 hooks/pre-exit | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index c24aa66..3d8ff36 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -189,14 +189,18 @@ find_and_upload() {
       # Spawn a subcommand group to allow parallel uploads
       {
         echo "Uploading '$file'..."
+        
         if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
           echo "Error uploading, will continue"
         fi
+
         if [[ "$ANNOTATE" != "false" ]]; then
           save-report-url "${CURL_RESP_FILE}"
         fi
 
-        echo "Finished uploading '$file'"
+        if [[ "${DEBUG}" == "true" ]]; then
+          echo "Finished uploading '$file'"
+        fi
       } &
 
       # Store the PID of the upload

From 47302c3bd990848d75d7ca4cf3eb3f53d44d2d0f Mon Sep 17 00:00:00 2001
From: js-murph <7096301+js-murph@users.noreply.github.com>
Date: Fri, 16 Jun 2023 14:46:57 +1000
Subject: [PATCH 21/31] Update tests/pre-exit-success.bats
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Matías Bellone <toote@users.noreply.github.com>
---
 tests/pre-exit-success.bats | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 0b54177..676c596 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -62,7 +62,7 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
 
   stub curl \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'"
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo curl success \${10};"
 
   run "$PWD/hooks/pre-exit"
 

From d4b4e219a72741c13f11df217e9359b79aa89349 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Mon, 19 Jun 2023 14:28:45 +1000
Subject: [PATCH 22/31] fix: Concurrency tests

---
 tests/pre-exit-success.bats | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 676c596..eb333c0 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -72,9 +72,8 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
   assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
   assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
-  assert_output --partial "curl success 1"
-  assert_output --partial "curl success 1"
-  assert_output --partial "curl success 1"
+  
+  assert_equal "$(echo "$output" | grep -c "curl success")" "3"
 }
 
 @test "Single file pattern through array" {

From 0e15880b46d3748850961b6678d1f7d46f4f4eaf Mon Sep 17 00:00:00 2001
From: js-murph <7096301+js-murph@users.noreply.github.com>
Date: Mon, 19 Jun 2023 14:51:17 +1000
Subject: [PATCH 23/31] Update tests/pre-exit-success.bats
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Matías Bellone <toote@users.noreply.github.com>
---
 tests/pre-exit-success.bats | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index eb333c0..cbd18ac 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -57,7 +57,8 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "curl success 3"
 }
 
-@test "Uploads multiple files concurrently" {
+@test "Uploads multiple files concurrently does not break basic functionality" {
+   # would love to test functionality but can not do so due to limitations on bats-mock :(
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
 

From 14fce484cb9570a2d148d89d8b9a9f551828dd30 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Mon, 19 Jun 2023 14:54:10 +1000
Subject: [PATCH 24/31] fix: BSD compatability

---
 hooks/pre-exit | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 3d8ff36..6dd296b 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -176,7 +176,8 @@ find_and_upload() {
         sleep 1
 
         for index in "${!uploads_in_progress[@]}"; do
-          if ! ps -p "${uploads_in_progress[index]}" > /dev/null; then
+          # Note: kill -0 does not kill the pid, it provides a *nix compatible way to test the pid is responding.
+          if ! kill -0 "${uploads_in_progress[index]}" > /dev/null; then
             unset 'uploads_in_progress[index]'
           elif [[ "$iterations_waited" -gt 10 ]]; then
             echo "Upload '${uploads_in_progress[index]}' has been running for more than 10 seconds, killing it"

From 57b729752e8331480732ce2426aef675deb98e57 Mon Sep 17 00:00:00 2001
From: js-murph <7096301+js-murph@users.noreply.github.com>
Date: Mon, 19 Jun 2023 15:15:49 +1000
Subject: [PATCH 25/31] fix: Use timeout var instead of magic numbers for
 concurrency

* fix: Use timeout var instead of magic numbers for concurrency
---
 hooks/pre-exit | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 6dd296b..723f7f7 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -179,8 +179,8 @@ find_and_upload() {
           # Note: kill -0 does not kill the pid, it provides a *nix compatible way to test the pid is responding.
           if ! kill -0 "${uploads_in_progress[index]}" > /dev/null; then
             unset 'uploads_in_progress[index]'
-          elif [[ "$iterations_waited" -gt 10 ]]; then
-            echo "Upload '${uploads_in_progress[index]}' has been running for more than 10 seconds, killing it"
+          elif [[ "$iterations_waited" -gt $TIMEOUT ]]; then
+            echo "Upload '${uploads_in_progress[index]}' has been running for more than '${TIMEOUT}' seconds, killing it"
             kill "${uploads_in_progress[index]}"
             unset 'uploads_in_progress[index]'
           fi

From 139f14ea70d1fccba05d1b59587d0110ab5ee0d8 Mon Sep 17 00:00:00 2001
From: js-murph <7096301+js-murph@users.noreply.github.com>
Date: Tue, 20 Jun 2023 12:19:22 +1000
Subject: [PATCH 26/31] fix: Add missing concurrency tests (#4)

* fix: Add missing concurrency tests
---
 hooks/pre-exit              |  3 ++-
 tests/pre-exit-success.bats | 45 +++++++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 723f7f7..160b127 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -9,6 +9,7 @@ ANNOTATE="${BUILDKITE_PLUGIN_TEST_COLLECTOR_ANNOTATION_LINK:-false}"
 UPLOAD_CONCURRENCY="${BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY:-1}"
 REPORT_URLS_FILE=$(mktemp)
 CURL_RESP_FILE="${CURL_RESP_FILE:-response.json}"
+TEST_LOGGING="${BUILDKITE_PLUGIN_TEST_LOGS:-false}"
 DEBUG="false"
 
 if [[ "${BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG:-}" =~ ^(true|on|1|always)$ ]]; then
@@ -169,7 +170,7 @@ find_and_upload() {
       iterations_waited=0
       while [[ "${#uploads_in_progress[@]}" -ge $UPLOAD_CONCURRENCY ]]; do
         iterations_waited=$((iterations_waited + 1))
-        if [[ "${DEBUG}" == "true" ]]; then
+        if [[ "${DEBUG}" == "true" || "${TEST_LOGGING}" == "true" ]]; then
           echo "Waiting for uploads to finish..."
         fi
 
diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index cbd18ac..ec02813 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -63,7 +63,7 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
 
   stub curl \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo curl success \${10};"
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo \"curl success \${10}\""
 
   run "$PWD/hooks/pre-exit"
 
@@ -73,7 +73,27 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
   assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
   assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
-  
+  assert_equal "$(echo "$output" | grep -c "curl success")" "3"
+}
+
+@test "Concurrency waits when the queue is full" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
+  export BUILDKITE_PLUGIN_TEST_LOGS='true'
+
+  stub curl \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : sleep 3; echo \"curl success \${10}\"" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo \"curl success \${10}\""
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub curl
+
+  assert_success
+  assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
+  assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
+  assert_output --partial "Waiting for uploads to finish..."
+  assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
   assert_equal "$(echo "$output" | grep -c "curl success")" "3"
 }
 
@@ -254,3 +274,24 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
   assert_output --partial "Error uploading, will continue"
 }
+
+@test "Concurrency gracefully handles failure" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
+  export BUILDKITE_PLUGIN_TEST_LOGS='true'
+
+  stub curl \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : exit 10" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub curl
+
+  assert_success
+  assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
+  assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
+  assert_equal "$(echo "$output" | grep -c "Error uploading, will continue")" "2"
+  assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
+  assert_equal "$(echo "$output" | grep -c "curl success")" "1"
+}

From f8ca0cf10bda4ff09d42811c59c6aa40159a7ddf Mon Sep 17 00:00:00 2001
From: js-murph <7096301+js-murph@users.noreply.github.com>
Date: Tue, 20 Jun 2023 16:21:35 +1000
Subject: [PATCH 27/31] fix: Various test fixes and new timeout test (#5)

fix: Various test fixes and new timeout test
---
 hooks/pre-exit              |  3 +--
 tests/pre-exit-success.bats | 35 +++++++++++++++++++++++++++++------
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 160b127..723f7f7 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -9,7 +9,6 @@ ANNOTATE="${BUILDKITE_PLUGIN_TEST_COLLECTOR_ANNOTATION_LINK:-false}"
 UPLOAD_CONCURRENCY="${BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY:-1}"
 REPORT_URLS_FILE=$(mktemp)
 CURL_RESP_FILE="${CURL_RESP_FILE:-response.json}"
-TEST_LOGGING="${BUILDKITE_PLUGIN_TEST_LOGS:-false}"
 DEBUG="false"
 
 if [[ "${BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG:-}" =~ ^(true|on|1|always)$ ]]; then
@@ -170,7 +169,7 @@ find_and_upload() {
       iterations_waited=0
       while [[ "${#uploads_in_progress[@]}" -ge $UPLOAD_CONCURRENCY ]]; do
         iterations_waited=$((iterations_waited + 1))
-        if [[ "${DEBUG}" == "true" || "${TEST_LOGGING}" == "true" ]]; then
+        if [[ "${DEBUG}" == "true" ]]; then
           echo "Waiting for uploads to finish..."
         fi
 
diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index ec02813..cf7bfc2 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -62,8 +62,7 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
 
-  stub curl \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo \"curl success \${10}\""
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo \"curl success \${10}\""
 
   run "$PWD/hooks/pre-exit"
 
@@ -79,11 +78,11 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
 @test "Concurrency waits when the queue is full" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
-  export BUILDKITE_PLUGIN_TEST_LOGS='true'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG='true'
 
   stub curl \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : sleep 3; echo \"curl success \${10}\"" \
-    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo \"curl success \${10}\""
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : sleep 3; echo \"curl success \${10}\"" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success \${10}\""
 
   run "$PWD/hooks/pre-exit"
 
@@ -189,6 +188,31 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "curl success"
 }
 
+@test "Concurrency gracefully handles command-group timeout" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT='3'
+
+  stub kill "\*: echo 'killed command-group'"
+
+  stub curl \
+    "-X POST --silent --show-error --max-time 3 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : sleep 60" \
+    "-X POST --silent --show-error --max-time 3 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub kill
+  unstub curl
+  
+
+  assert_success
+  assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
+  assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
+  assert_equal "$(echo "$output" | grep -c "has been running for more than")" "2"
+  assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
+  assert_equal "$(echo "$output" | grep -c "curl success")" "1"
+}
+
 @test "Git available sends plugin version" {
   stub git "rev-parse --short HEAD : echo 'some-commit-id'"
   stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success with \${30}\""
@@ -278,7 +302,6 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
 @test "Concurrency gracefully handles failure" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
-  export BUILDKITE_PLUGIN_TEST_LOGS='true'
 
   stub curl \
     "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : exit 10" \

From a85a3aec937b38480accc4857c27b7573c9c00e3 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Fri, 23 Jun 2023 11:33:02 +1000
Subject: [PATCH 28/31] chore: Remove non-functional timeout test

---
 tests/pre-exit-success.bats | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index cf7bfc2..9268107 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -188,31 +188,6 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "curl success"
 }
 
-@test "Concurrency gracefully handles command-group timeout" {
-  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
-  export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
-  export BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT='3'
-
-  stub kill "\*: echo 'killed command-group'"
-
-  stub curl \
-    "-X POST --silent --show-error --max-time 3 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : sleep 60" \
-    "-X POST --silent --show-error --max-time 3 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
-
-  run "$PWD/hooks/pre-exit"
-
-  unstub kill
-  unstub curl
-  
-
-  assert_success
-  assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
-  assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
-  assert_equal "$(echo "$output" | grep -c "has been running for more than")" "2"
-  assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
-  assert_equal "$(echo "$output" | grep -c "curl success")" "1"
-}
-
 @test "Git available sends plugin version" {
   stub git "rev-parse --short HEAD : echo 'some-commit-id'"
   stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success with \${30}\""

From b9aded8f481fde157ad398c9a42dc0d3a6b32b87 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Thu, 29 Jun 2023 16:21:47 +1000
Subject: [PATCH 29/31] fix: Re-add working concurrency timeout test

---
 tests/pre-exit-success.bats | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 9268107..9a6d6f1 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -188,6 +188,25 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_output --partial "curl success"
 }
 
+@test "Concurrency gracefully handles command-group timeout" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='2'
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT='3'
+
+  stub curl "if [ \${10} != 'data=@\"./tests/fixtures/junit-3.xml\"' ]; then echo sleeping for \${10}; sleep 10 & wait \$!; else echo curl success \${10}; fi"
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub curl
+  
+  assert_success
+  assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
+  assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
+  assert_output --partial "Error uploading, will continue"
+  assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
+  assert_equal "$(echo "$output" | grep -c "curl success")" "1"
+}
+
 @test "Git available sends plugin version" {
   stub git "rev-parse --short HEAD : echo 'some-commit-id'"
   stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success with \${30}\""

From d1f196b886c816409656adee929ea54e3a405004 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Fri, 30 Jun 2023 10:49:23 +1000
Subject: [PATCH 30/31] fix: timeout test consistency and response race
 condition

---
 hooks/pre-exit              | 17 ++++++++++++-----
 tests/pre-exit-success.bats |  2 +-
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/hooks/pre-exit b/hooks/pre-exit
index 723f7f7..8235804 100755
--- a/hooks/pre-exit
+++ b/hooks/pre-exit
@@ -8,7 +8,7 @@ BASE_PATH="${BUILDKITE_PLUGIN_TEST_COLLECTOR_BASE_PATH:-.}"
 ANNOTATE="${BUILDKITE_PLUGIN_TEST_COLLECTOR_ANNOTATION_LINK:-false}"
 UPLOAD_CONCURRENCY="${BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY:-1}"
 REPORT_URLS_FILE=$(mktemp)
-CURL_RESP_FILE="${CURL_RESP_FILE:-response.json}"
+CURL_RESP_FILE="${CURL_RESP_FILE:-}"
 DEBUG="false"
 
 if [[ "${BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG:-}" =~ ^(true|on|1|always)$ ]]; then
@@ -95,6 +95,7 @@ save-report-url() {
 # Upload failures should not fail the build, and should have a sensible timeout,
 # so that Test Analytics availability doesn't affect build reliability.
 upload() {
+  local response_file="$4"
   local file="$3"
   local format="$2"
 
@@ -125,7 +126,7 @@ upload() {
   fi
 
   if [[ "$ANNOTATE" != "false" ]]; then
-    curl_args+=("-o" "${CURL_RESP_FILE}")
+    curl_args+=("-o" "${response_file}")
   fi
 
   curl_args+=("${BUILDKITE_PLUGIN_TEST_COLLECTOR_API_URL:-https://analytics-api.buildkite.com/v1/uploads}")
@@ -153,7 +154,7 @@ find_and_upload() {
   matching_files=()
   while IFS=$'' read -r matching_file ; do
     matching_files+=("$matching_file")
-  done < <("${FIND_CMD[@]}" "${BASE_PATH}" -path "${BASE_PATH}/${FILES_PATTERN}")
+  done < <("${FIND_CMD[@]}" "${BASE_PATH}" -path "${BASE_PATH}/${FILES_PATTERN}" | sort)
 
   if [[ "${#matching_files[@]}" -eq "0" ]]; then
     echo "No files found matching '${FILES_PATTERN}'"
@@ -190,13 +191,19 @@ find_and_upload() {
       # Spawn a subcommand group to allow parallel uploads
       {
         echo "Uploading '$file'..."
+
+        if [[ -n "${CURL_RESP_FILE}" ]]; then
+          response_file="${CURL_RESP_FILE}"
+        else
+          response_file="$(mktemp -t 'response.XXXXXX')"
+        fi
         
-        if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
+        if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}" "${response_file}"; then
           echo "Error uploading, will continue"
         fi
 
         if [[ "$ANNOTATE" != "false" ]]; then
-          save-report-url "${CURL_RESP_FILE}"
+          save-report-url "${response_file}"
         fi
 
         if [[ "${DEBUG}" == "true" ]]; then
diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 9a6d6f1..58fd8a0 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -202,7 +202,7 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   assert_success
   assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
   assert_output --partial "Uploading './tests/fixtures/junit-2.xml'..."
-  assert_output --partial "Error uploading, will continue"
+  assert_equal "$(echo "$output" | grep -c "has been running for more than")" "2"
   assert_output --partial "Uploading './tests/fixtures/junit-3.xml'..."
   assert_equal "$(echo "$output" | grep -c "curl success")" "1"
 }

From dbb571653dd0817c93420c46b07889d825008b88 Mon Sep 17 00:00:00 2001
From: John Murphy <johnm@squareup.com>
Date: Mon, 3 Jul 2023 09:53:35 +1000
Subject: [PATCH 31/31] fix: skip multiple files concurrently

---
 tests/pre-exit-success.bats | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
index 58fd8a0..7863904 100644
--- a/tests/pre-exit-success.bats
+++ b/tests/pre-exit-success.bats
@@ -62,6 +62,8 @@ COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \*
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_UPLOAD_CONCURRENCY='3'
 
+  skip "bats-mock does not currently support concurrency, so we can't test this reliably"
+
   stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo \"curl success \${10}\""
 
   run "$PWD/hooks/pre-exit"