From 81065a8b9297ff48e0ac0a63cc08f0ff2b490377 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 14 Mar 2025 19:58:09 -0700 Subject: [PATCH 1/4] add new CI scripts and configs --- .github/workflows/main.yml | 26 ++++++++++++++++++++ scripts/safe_run.sh | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100755 scripts/safe_run.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a1c265f3fb..914753f11e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -177,3 +177,29 @@ jobs: name: yellow-ai-vs-ai-proto-logs path: | /tmp/tbots/yellow/proto_* + + common-commands: + strategy: + matrix: + commands: + - bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main --run_blue --run_diagnostics --interface lo --keyboard_estop --ci_mode + - bazel run --run_under="xvfb-run" //software/ai/hl/stp/tactic/goalie:goalie_tactic_test -- --enable_thunderscope + + name: Sanity Check on Common Commands + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Environment Setup + run: | + "${GITHUB_WORKSPACE}"/environment_setup/setup_software.sh + + - name: Build Thunderscope + run: | + cd src + bazel build //software/thunderscope:thunderscope_main + + - name: Check Common Commands + run: | + cd src + ../scripts/safe_run.sh ${{ matrix.commands }} diff --git a/scripts/safe_run.sh b/scripts/safe_run.sh new file mode 100755 index 0000000000..fe455f4786 --- /dev/null +++ b/scripts/safe_run.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Timeout in seconds +# When the time is up and no error was shown, this test will pass +TIME_LIMIT=60 # 1 minute + +# Match Python traceback +ERROR_PATTERN="Traceback (most recent call last):" + +# Temporary log file +LOG_FILE=$(mktemp) + +# Run the command and record the log +"$@" &> "$LOG_FILE" & +CMD_PID=$! + +# Time the process +SECONDS=0 +while kill -0 $CMD_PID 2>/dev/null; do + # Check if time is up + if [ $SECONDS -ge $TIME_LIMIT ]; then + echo "Time limit reached, stopping process: $CMD_PID" + kill $CMD_PID + wait $CMD_PID + exit 0 # Upon time out and no error, returns 0 status code + fi + + # Check if the log contains Traceback + if grep -q "$ERROR_PATTERN" "$LOG_FILE"; then + echo $LOG_FILE + echo "[Error detected] Potential error found in command output!" + kill $CMD_PID + wait $CMD_PID + exit 1 + fi + + sleep 1 # Run this loop once per second +done + +# Get the exit code of the process +wait $CMD_PID +EXIT_CODE=$? + +# Clean up log file +rm -f "$LOG_FILE" + +# Exit with the command status code +exit $EXIT_CODE + + From 06df1b7d71876c7f9f7f802ed08bffbc8fe60437 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 14 Mar 2025 20:46:48 -0700 Subject: [PATCH 2/4] revise timeout and commands --- .github/workflows/main.yml | 4 +++- scripts/safe_run.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 914753f11e..55505cb974 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -180,9 +180,11 @@ jobs: common-commands: strategy: + # TODO: set fail-fast to true in production. + fail-fast: false matrix: commands: - - bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main --run_blue --run_diagnostics --interface lo --keyboard_estop --ci_mode + - bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main -- --run_blue --run_diagnostics --interface lo --keyboard_estop --ci_mode - bazel run --run_under="xvfb-run" //software/ai/hl/stp/tactic/goalie:goalie_tactic_test -- --enable_thunderscope name: Sanity Check on Common Commands diff --git a/scripts/safe_run.sh b/scripts/safe_run.sh index fe455f4786..8bf857cace 100755 --- a/scripts/safe_run.sh +++ b/scripts/safe_run.sh @@ -2,7 +2,7 @@ # Timeout in seconds # When the time is up and no error was shown, this test will pass -TIME_LIMIT=60 # 1 minute +TIME_LIMIT=120 # 2 minutes # Match Python traceback ERROR_PATTERN="Traceback (most recent call last):" From 05c6cd956d89272f13432877cb06f6f3f471350e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 15 Mar 2025 13:22:51 -0700 Subject: [PATCH 3/4] print log upon error --- .github/workflows/main.yml | 1 + scripts/safe_run.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55505cb974..24f2764101 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -205,3 +205,4 @@ jobs: run: | cd src ../scripts/safe_run.sh ${{ matrix.commands }} + diff --git a/scripts/safe_run.sh b/scripts/safe_run.sh index 8bf857cace..6678f38ef9 100755 --- a/scripts/safe_run.sh +++ b/scripts/safe_run.sh @@ -27,7 +27,7 @@ while kill -0 $CMD_PID 2>/dev/null; do # Check if the log contains Traceback if grep -q "$ERROR_PATTERN" "$LOG_FILE"; then - echo $LOG_FILE + cat $LOG_FILE echo "[Error detected] Potential error found in command output!" kill $CMD_PID wait $CMD_PID From 7eded6d6323323ed7d381e5f04fc0180fb0aef03 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 15 Mar 2025 15:24:38 -0700 Subject: [PATCH 4/4] add thunderscope command and proto replay --- .github/workflows/main.yml | 9 +++++---- scripts/safe_run.sh | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24f2764101..8144f8b267 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -184,8 +184,9 @@ jobs: fail-fast: false matrix: commands: - - bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main -- --run_blue --run_diagnostics --interface lo --keyboard_estop --ci_mode - - bazel run --run_under="xvfb-run" //software/ai/hl/stp/tactic/goalie:goalie_tactic_test -- --enable_thunderscope + - ../scripts/safe_run.sh bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main -- --run_blue --run_diagnostics --interface lo --keyboard_estop --ci_mode + - ../scripts/safe_run.sh bazel run --run_under="xvfb-run" //software/ai/hl/stp/tactic/goalie:goalie_tactic_test -- --enable_thunderscope + - ../scripts/safe_run.sh bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main -- --enable_autoref --ci_mode && ../scripts/safe_run.sh bazel run --run_under="xvfb-run" //software/thunderscope:thunderscope_main -- blue_log="$(find /tmp/tbots/blue -maxdepth 1 -type d -name 'proto_*' -printf '/tmp/tbots/blue/%f\n' 2>/dev/null | head -n1)" name: Sanity Check on Common Commands runs-on: ubuntu-20.04 @@ -200,9 +201,9 @@ jobs: run: | cd src bazel build //software/thunderscope:thunderscope_main - + - name: Check Common Commands run: | cd src - ../scripts/safe_run.sh ${{ matrix.commands }} + ${{ matrix.commands }} diff --git a/scripts/safe_run.sh b/scripts/safe_run.sh index 6678f38ef9..3be54c7521 100755 --- a/scripts/safe_run.sh +++ b/scripts/safe_run.sh @@ -14,22 +14,23 @@ LOG_FILE=$(mktemp) "$@" &> "$LOG_FILE" & CMD_PID=$! +echo "Process Running in Wrapper with Timeout $TIME_LIMIT ..." + # Time the process SECONDS=0 while kill -0 $CMD_PID 2>/dev/null; do # Check if time is up if [ $SECONDS -ge $TIME_LIMIT ]; then echo "Time limit reached, stopping process: $CMD_PID" - kill $CMD_PID + kill -SIGINT $CMD_PID wait $CMD_PID exit 0 # Upon time out and no error, returns 0 status code fi # Check if the log contains Traceback if grep -q "$ERROR_PATTERN" "$LOG_FILE"; then - cat $LOG_FILE echo "[Error detected] Potential error found in command output!" - kill $CMD_PID + kill -SIGINT $CMD_PID wait $CMD_PID exit 1 fi @@ -37,6 +38,7 @@ while kill -0 $CMD_PID 2>/dev/null; do sleep 1 # Run this loop once per second done +cat $LOG_FILE # Get the exit code of the process wait $CMD_PID EXIT_CODE=$?