update ut_result_check.sh

Author: zxd1997066

.github/scripts/ut_result_check.sh

@@ -1,10 +1,59 @@
 #!/bin/bash
 ut_suite="${1:-op_regression}"   # op_regression / op_extended / op_ut / torch_xpu
 
-if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' ]]; then
-    grep -E "^FAILED|have failures" "${ut_suite}"_test.log | awk '{print $2}' > ./"${ut_suite}"_failed.log
+# usage
+# compare_and_filter_logs <UT'log> <Known_issue.log> [output.log]
+
+compare_and_filter_logs() {
+    local file_UT="$1"
+    local file_known_issue="$2"
+    local output_file="${3:-${file_UT%.*}_filtered.log}"
+    local filtered_content="${file_UT%.*}_removed.log"
+
+    if [[ $# -lt 2 ]]; then
+        echo "[ERROR] Need 2 files to compare"
+        return 1
+    fi
+
+    # Check whether UT's failed log contains the case of the known issue'log
+    echo "Checking whether $file_UT contains $file_known_issue"
+    if grep -qxFf "$file_known_issue" "$file_UT"; then
+        echo "$file_UT contains $file_known_issue"
+    else
+        echo "$file_UT does not contain $file_known_issue"
+        return 1
+    fi
+
+    # Filter the same content from file_UT as file_known_issue
+    echo "Filtering $file_known_issue for $file_UT"
+    grep -vFxf "$file_known_issue" "$file_UT" > "$output_file"
+
+    # Keep the filtered UT cases
+    grep -noFf "$file_known_issue" "$file_UT" > "$filtered_content"
+    echo "Filtered cases file: $filtered_content"
+    if [[ -s "$filtered_content" ]]; then
+        echo -e "\n\033[1;31m[Filtered Cases]\033[0m"
+        awk -F':' '{
+            line_number = $1
+            $1 = ""
+            gsub(/^ /, "", $0)
+            printf "\033[33m%3d\033[0m: %s\n", line_number, $0
+        }' "$filtered_content"
+    else
+        echo -e "\n\033[1;32mNo Filtered Cases\033[0m"
+    fi
+}
+
+if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' ]]; then
+    grep -E "FAILED" "${ut_suite}"_test.log | awk '{print $1}' | grep -v "FAILED" > ./"${ut_suite}"_failed.log
+    grep -E "have failures" "${ut_suite}"_test.log | awk '{print $1}' >> ./"${ut_suite}"_failed.log
     grep "PASSED" "${ut_suite}"_test.log | awk '{print $1}' > ./"${ut_suite}"_passed.log
-    num_failed=$(wc -l < "./${ut_suite}_failed.log")
+    compare_and_filter_logs "${ut_suite}"_failed.log Known_issue.log
+    if [[ -f "${ut_suite}_failed_filtered.log" ]]; then
+      num_failed=$(wc -l < "./${ut_suite}_failed_filtered.log")
+    else
+      num_failed=$(wc -l < "./${ut_suite}_failed.log")
+    fi
     num_passed=$(wc -l < "./${ut_suite}_passed.log")
     echo -e "========================================================================="
     echo -e "Show Failed cases in ${ut_suite}"
@@ -18,10 +67,23 @@ if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1'
     fi
 fi
 if [[ "${ut_suite}" == 'op_ut' ]]; then
-    grep -E "^FAILED|have failures" op_ut_with_skip_test.log | awk '{print $2}' > ./"${ut_suite}"_with_skip_test_failed.log
-    grep -E "^FAILED|have failures" op_ut_with_only_test.log | awk '{print $2}' > ./"${ut_suite}"_with_only_test_failed.log
-    num_failed_with_skip=$(wc -l < "./${ut_suite}_with_skip_test_failed.log")
-    num_failed_with_only=$(wc -l < "./${ut_suite}_with_only_test_failed.log")
+    grep -E "FAILED" op_ut_with_skip_test.log | awk '{print $1}' | grep -v "FAILED" > ./"${ut_suite}"_with_skip_test_failed.log
+    grep -E "have failures" op_ut_with_skip_test.log | awk '{print $1}' >> ./"${ut_suite}"_with_skip_test_failed.log
+    grep -E "FAILED" op_ut_with_only_test.log | awk '{print $1}' | grep -v "FAILED" > ./"${ut_suite}"_with_only_test_failed.log
+    grep -E "have failures" op_ut_with_only_test.log | awk '{print $1}' >> ./"${ut_suite}"_with_only_test_failed.log
+    compare_and_filter_logs "${ut_suite}"_with_skip_test_failed.log Known_issue.log
+    if [[ -f "${ut_suite}_with_skip_test_failed_filtered.log" ]]; then
+      num_failed_with_skip=$(wc -l < "./${ut_suite}_with_skip_test_failed_filtered.log")
+    else
+      num_failed_with_skip=$(wc -l < "./${ut_suite}_with_skip_test_failed.log")
+    fi
+    compare_and_filter_logs "${ut_suite}"_with_only_test_failed.log Known_issue.log
+    if [[ -f "${ut_suite}_with_only_test_failed_filtered.log" ]]; then
+      num_failed_with_only=$(wc -l < "./${ut_suite}_with_only_test_failed_filtered.log")
+    else
+      num_failed_with_only=$(wc -l < "./${ut_suite}_with_only_test_failed.log")
+    fi
+    num_failed_with_only=$(wc -l < "./${ut_suite}_with_only_test_failed_filtered.log")
     echo -e "========================================================================="
     echo -e "Show Failed cases in ${ut_suite} with skip"
     echo -e "========================================================================="
@@ -73,13 +135,19 @@ if [[ "${ut_suite}" == 'torch_xpu' ]]; then
     fi
 fi
 if [[ "${ut_suite}" == 'xpu_distributed' || "${ut_suite}" == 'pytorch_distributed' ]]; then
-    grep -E "^FAILED|have failures" "${ut_suite}"_test.log | awk '{print $2}' > ./"${ut_suite}"_test_failed.log
-    num_failed_distributed=$(wc -l < "./${ut_suite}_test_failed.log")
+    grep -E "^FAILED" xpu_distributed_test.log | awk '{print $2}' > ./"${ut_suite}"_test_failed.log
+    grep -E "have failures" xpu_distributed_test.log | awk '{print $1}' >> ./"${ut_suite}"_test_failed.log
+    compare_and_filter_logs "${ut_suite}"_test_failed.log Known_issue.log
+    if [[ -f "${ut_suite}_test_failed_filtered.log" ]]; then
+      num_failed_xpu_distributed=$(wc -l < "./${ut_suite}_test_failed_filtered.log")
+    else
+      num_failed_xpu_distributed=$(wc -l < "./${ut_suite}_test_failed.log")
+    fi
     echo -e "========================================================================="
-    echo -e "Show Failed cases in ${ut_suite}"
+    echo -e "Show Failed cases in ${ut_suite} xpu distributed"
     echo -e "========================================================================="
     cat "./${ut_suite}_test_failed.log"
-    ((num_failed=num_failed_distributed))
+    ((num_failed=num_failed_xpu_distributed))
     if [[ $num_failed -gt 0 ]]; then
       echo -e "[ERROR] UT ${ut_suite} test Fail"
       exit 1