Improve CI/CD

.ci/boot-linux-prepare.sh

@@ -16,10 +16,10 @@ which mkfs.ext4 > /dev/null 2>&1 || which $(brew --prefix e2fsprogs)/sbin/mkfs.e
         echo "Error: mkfs.ext4 not found"
         exit 1
     }
-which 7z > /dev/null 2>&1 || {
-    echo "Error: 7z not found"
-    exit 1
-}
+# Optional tooling used later in the test suite
+if ! command -v debugfs > /dev/null 2>&1 && ! command -v 7z > /dev/null 2>&1; then
+    print_warning "Neither debugfs nor 7z is available; virtio-blk verification will be skipped."
+fi
 
 ACTION=$1
 

.ci/boot-linux.sh

@@ -6,13 +6,54 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 check_platform
 
-function cleanup
+cleanup()
 {
     sleep 1
     pkill -9 rv32emu
 }
 
-function ASSERT
+check_image_for_file()
+{
+    local image_path=$1
+    local file_path=$2
+    local tool_available=0
+    local debugfs_cmd
+
+    debugfs_cmd=$(command -v debugfs 2> /dev/null || true)
+    if [ -z "${debugfs_cmd}" ] && [ -x /sbin/debugfs ]; then
+        debugfs_cmd=/sbin/debugfs
+    fi
+
+    if [ -n "${debugfs_cmd}" ]; then
+        tool_available=1
+        if "${debugfs_cmd}" -R "stat ${file_path}" "${image_path}" > /dev/null 2>&1; then
+            return 0
+        fi
+    fi
+
+    if command -v 7z > /dev/null 2>&1; then
+        tool_available=1
+        if 7z l "${image_path}" | grep -q "${file_path}"; then
+            return 0
+        fi
+    fi
+
+    if [ -n "${debugfs_cmd}" ]; then
+        tool_available=1
+        if sudo "${debugfs_cmd}" -R "stat ${file_path}" "${image_path}" > /dev/null 2>&1; then
+            return 0
+        fi
+    fi
+
+    if [ "${tool_available}" -eq 0 ]; then
+        print_warning "Skipping verification of ${file_path} in ${image_path}: neither debugfs nor 7z is available."
+        return 0
+    fi
+
+    return 1
+}
+
+ASSERT()
 {
     $*
     local RES=$?
@@ -130,9 +171,12 @@ for i in "${!TEST_OPTIONS[@]}"; do
 
     OPTS="${OPTS_BASE}"
     # No need to add option when running base test
-    if [[ ! "${TEST_OPTIONS[$i]}" =~ "base" ]]; then
-        OPTS+="${TEST_OPTIONS[$i]}"
-    fi
+    case "${TEST_OPTIONS[$i]}" in
+        *base*) ;;
+        *)
+            OPTS+="${TEST_OPTIONS[$i]}"
+            ;;
+    esac
     RUN_LINUX="build/rv32emu ${OPTS}"
 
     ASSERT expect <<- DONE
@@ -145,13 +189,20 @@ for i in "${!TEST_OPTIONS[@]}"; do
     cleanup
 
     printf "\nBoot Linux Test: [ ${MESSAGES[$ret]}${COLOR_N} ]\n"
-    if [[ "${TEST_OPTIONS[$i]}" =~ vblk ]]; then
-        # read-only test first, so the emu.txt definitely does not exist, skipping the check
-        if [[ ! "${TEST_OPTIONS[$i]}" =~ readonly ]]; then
-            7z l ${VBLK_IMG} | grep emu.txt > /dev/null 2>&1 || ret=4
-        fi
-        printf "Virtio-blk Test: [ ${MESSAGES[$ret]}${COLOR_N} ]\n"
-    fi
+    case "${TEST_OPTIONS[$i]}" in
+        *vblk*)
+            # read-only test first, so the emu.txt definitely does not exist, skipping the check
+            case "${TEST_OPTIONS[$i]}" in
+                *readonly*) ;;
+                *)
+                    if ! check_image_for_file "${VBLK_IMG}" emu.txt; then
+                        ret=4
+                    fi
+                    ;;
+            esac
+            printf "Virtio-blk Test: [ ${MESSAGES[$ret]}${COLOR_N} ]\n"
+            ;;
+    esac
 done
 
 exit ${ret}

.ci/check-format.sh

@@ -4,31 +4,58 @@
 
 set -u -o pipefail
 
-set -x
-
 REPO_ROOT="$(git rev-parse --show-toplevel)"
 
-C_SOURCES=$(find "${REPO_ROOT}" | egrep "\.(c|cxx|cpp|h|hpp)$")
-for file in ${C_SOURCES}; do
-    clang-format-18 ${file} > expected-format
-    diff -u -p --label="${file}" --label="expected coding style" ${file} expected-format
-done
-C_MISMATCH_LINE_CNT=$(clang-format-18 --output-replacements-xml ${C_SOURCES} | egrep -c "</replacement>")
-
-SH_SOURCES=$(find "${REPO_ROOT}" | egrep "\.sh$")
-for file in ${SH_SOURCES}; do
-    shfmt -d "${file}"
-done
-SH_MISMATCH_FILE_CNT=$(shfmt -l ${SH_SOURCES})
-
-PY_SOURCES=$(find "${REPO_ROOT}" | egrep "\.py$")
-for file in ${PY_SOURCES}; do
-    echo "Checking Python file: ${file}"
-    black --diff "${file}"
-done
-PY_MISMATCH_FILE_CNT=0
-if [ -n "${PY_SOURCES}" ]; then
-    PY_MISMATCH_FILE_CNT=$(echo "$(black --check ${PY_SOURCES} 2>&1)" | grep -c "^would reformat ")
+# Use git ls-files to exclude submodules and untracked files
+C_SOURCES=()
+while IFS= read -r file; do
+    [ -n "$file" ] && C_SOURCES+=("$file")
+done < <(git ls-files -- '*.c' '*.cxx' '*.cpp' '*.h' '*.hpp')
+
+if [ ${#C_SOURCES[@]} -gt 0 ]; then
+    if command -v clang-format-18 > /dev/null 2>&1; then
+        echo "Checking C/C++ files..."
+        clang-format-18 -n --Werror "${C_SOURCES[@]}"
+        C_FORMAT_EXIT=$?
+    else
+        echo "Skipping C/C++ format check: clang-format-18 not found" >&2
+        C_FORMAT_EXIT=0
+    fi
+else
+    C_FORMAT_EXIT=0
+fi
+
+SH_SOURCES=()
+while IFS= read -r file; do
+    [ -n "$file" ] && SH_SOURCES+=("$file")
+done < <(git ls-files -- '*.sh')
+
+if [ ${#SH_SOURCES[@]} -gt 0 ]; then
+    echo "Checking shell scripts..."
+    MISMATCHED_SH=$(shfmt -l "${SH_SOURCES[@]}")
+    if [ -n "$MISMATCHED_SH" ]; then
+        echo "The following shell scripts are not formatted correctly:"
+        printf '%s\n' "$MISMATCHED_SH"
+        shfmt -d "${SH_SOURCES[@]}"
+        SH_FORMAT_EXIT=1
+    else
+        SH_FORMAT_EXIT=0
+    fi
+else
+    SH_FORMAT_EXIT=0
+fi
+
+PY_SOURCES=()
+while IFS= read -r file; do
+    [ -n "$file" ] && PY_SOURCES+=("$file")
+done < <(git ls-files -- '*.py')
+
+if [ ${#PY_SOURCES[@]} -gt 0 ]; then
+    echo "Checking Python files..."
+    black --check --diff "${PY_SOURCES[@]}"
+    PY_FORMAT_EXIT=$?
+else
+    PY_FORMAT_EXIT=0
 fi
 
-exit $((C_MISMATCH_LINE_CNT + SH_MISMATCH_FILE_CNT + PY_MISMATCH_FILE_CNT))
+exit $((C_FORMAT_EXIT + SH_FORMAT_EXIT + PY_FORMAT_EXIT))

.ci/common.sh

@@ -1,3 +1,8 @@
+# Bash strict mode (enabled only when executed directly, not sourced)
+if ! (return 0 2> /dev/null); then
+    set -euo pipefail
+fi
+
 # Expect host is Linux/x86_64, Linux/aarch64, macOS/arm64
 
 MACHINE_TYPE=$(uname -m)
@@ -15,12 +20,67 @@ check_platform()
     esac
 }
 
-if [[ "${OS_TYPE}" == "Linux" ]]; then
+if [ "${OS_TYPE}" = "Linux" ]; then
     PARALLEL=-j$(nproc)
 else
     PARALLEL=-j$(sysctl -n hw.logicalcpu)
 fi
 
+# Color output helpers
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+print_success()
+{
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+print_error()
+{
+    echo -e "${RED}[ERROR]${NC} $1" >&2
+}
+
+print_warning()
+{
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+# Assertion function for tests
+# Usage: ASSERT <condition> <error_message>
+ASSERT()
+{
+    local condition=$1
+    shift
+    local message="$*"
+
+    if ! eval "${condition}"; then
+        print_error "Assertion failed: ${message}"
+        print_error "Condition: ${condition}"
+        return 1
+    fi
+}
+
+# Cleanup function registry
+CLEANUP_FUNCS=()
+
+register_cleanup()
+{
+    CLEANUP_FUNCS+=("$1")
+}
+
+cleanup()
+{
+    local func
+    for func in "${CLEANUP_FUNCS[@]-}"; do
+        [ -n "${func}" ] || continue
+        eval "${func}" || true
+    done
+}
+
+trap cleanup EXIT
+
 # Universal download utility with curl/wget compatibility
 # Provides consistent interface regardless of which tool is available
 
@@ -49,7 +109,7 @@ download_to_stdout()
     local url="$1"
     case "$DOWNLOAD_TOOL" in
         curl)
-            curl -fsSL "$url"
+            curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL "$url"
             ;;
         wget)
             wget -qO- "$url"
@@ -66,7 +126,7 @@ download_to_file()
     local output="$2"
     case "$DOWNLOAD_TOOL" in
         curl)
-            curl -fsSL -o "$output" "$url"
+            curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL -o "$output" "$url"
             ;;
         wget)
             wget -q -O "$output" "$url"
@@ -88,7 +148,7 @@ download_with_headers()
             for header in "$@"; do
                 headers+=(-H "$header")
             done
-            curl -fsSL "${headers[@]}" "$url"
+            curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL "${headers[@]}" "$url"
             ;;
         wget)
             for header in "$@"; do
@@ -107,7 +167,7 @@ download_silent()
     local url="$1"
     case "$DOWNLOAD_TOOL" in
         curl)
-            curl -fsSL "$url"
+            curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL "$url"
             ;;
         wget)
             wget -qO- "$url"
@@ -124,7 +184,7 @@ download_with_progress()
     local output="$2"
     case "$DOWNLOAD_TOOL" in
         curl)
-            curl -fL -# -o "$output" "$url"
+            curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -L -# -o "$output" "$url"
             ;;
         wget)
             wget -O "$output" "$url"
@@ -141,7 +201,7 @@ check_url()
     local url="$1"
     case "$DOWNLOAD_TOOL" in
         curl)
-            curl -fsSL --head "$url" > /dev/null 2>&1
+            curl -fS --retry 5 --retry-delay 2 --retry-max-time 60 -sL --head "$url" > /dev/null 2>&1
             ;;
         wget)
             wget --spider -q "$url" 2> /dev/null

.ci/gdbstub-test.sh

@@ -9,8 +9,8 @@ prefixes=("${CROSS_COMPILE}" "riscv32-unknown-elf-" "riscv-none-elf-")
 for prefix in "${prefixes[@]}"; do
     utility=${prefix}gdb
     set +e # temporarily disable exit on error
-    command -v "${utility}" &> /dev/null
-    if [[ $? == 0 ]]; then
+    command -v "${utility}" > /dev/null 2>&1
+    if [ $? = 0 ]; then
         GDB=${utility}
     fi
     set -e

.ci/riscv-toolchain-install.sh

@@ -9,11 +9,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 check_platform
 mkdir -p toolchain
 
-if [[ "$#" == "0" ]] || [[ "$1" != "riscv-collab" ]]; then
+if [ "$#" = "0" ] || [ "$1" != "riscv-collab" ]; then
     GCC_VER=15.2.0-1
     TOOLCHAIN_REPO=https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack
 
-    if [[ "${OS_TYPE}" == "Linux" ]]; then
+    if [ "${OS_TYPE}" = "Linux" ]; then
         case "${MACHINE_TYPE}" in
             "x86_64")
                 TOOLCHAIN_URL=${TOOLCHAIN_REPO}/releases/download/v${GCC_VER}/xpack-riscv-none-elf-gcc-${GCC_VER}-linux-x64.tar.gz

.github/workflows/main.yml

@@ -345,6 +345,10 @@ jobs:
              .ci/riscv-toolchain-install.sh
              echo "${{ github.workspace }}/toolchain/bin" >> $GITHUB_PATH
              echo "$(brew --prefix llvm@18)/bin" >> $GITHUB_PATH
+     - name: Set up Python 3.12
+       uses: actions/setup-python@v6
+       with:
+         python-version: '3.12'
      - name: Install compiler
        id: install_cc
        uses: rlalik/setup-cpp-compiler@master