Add progress markers, unbuffered output, and log replay

- Print "Starting/Finished test X on GPU Y" to trace during execution
- Capture per-test output to .log files (unbuffered with python -u)
- Replay all logs sequentially into trace after parallel section
- Trace reads like sequential execution; per-test .log files in artifacts
- OOM errors captured via unbuffered writes before process death

Author: sudhakarsingh27

qa/L0_pytorch_unittest/test.sh

@@ -56,13 +56,16 @@ function run_test() {
 
     if [ "$NUM_GPUS" -le 1 ]; then
         # Single GPU: run synchronously (identical to original behavior)
-        eval "${env_prefix} python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/${xml_name} ${test_path}" \
+        eval "${env_prefix} python3 -u -m pytest --tb=auto --junitxml=$XML_LOG_DIR/${xml_name} ${test_path}" \
             || test_fail "$fail_label"
     else
         # Multi GPU: run in background on assigned GPU
+        echo ">>> Starting: ${fail_label} on GPU ${GPU_LIST[$gpu_id]}"
         (
-            eval "CUDA_VISIBLE_DEVICES=${GPU_LIST[$gpu_id]} ${env_prefix} python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/${xml_name} ${test_path}" \
+            eval "CUDA_VISIBLE_DEVICES=${GPU_LIST[$gpu_id]} ${env_prefix} python3 -u -m pytest --tb=auto --junitxml=$XML_LOG_DIR/${xml_name} ${test_path}" \
+                > "$XML_LOG_DIR/${xml_name%.xml}.log" 2>&1 \
                 || test_fail "$fail_label"
+            echo ">>> Finished: ${fail_label} on GPU ${GPU_LIST[$gpu_id]}"
         ) &
     fi
 
@@ -121,6 +124,24 @@ if [ "$NUM_GPUS" -gt 1 ]; then
     wait
 fi
 
+# ── Replay per-test logs into trace ──────────────────────────────────────────
+
+if [ "$NUM_GPUS" -gt 1 ]; then
+    echo ""
+    echo "=== Per-test output (replayed from parallel execution) ==="
+    for logfile in "$XML_LOG_DIR"/*.log; do
+        if [ -f "$logfile" ]; then
+            echo ""
+            echo "────────────────────────────────────────────────────────"
+            echo ">>> $(basename "$logfile" .log)"
+            echo "────────────────────────────────────────────────────────"
+            cat "$logfile"
+        fi
+    done
+    echo ""
+    echo "=== End of per-test output ==="
+fi
+
 # ── Report results ──────────────────────────────────────────────────────────
 
 if [ -s "$FAIL_DIR/failures" ]; then