656 combined replay stream in chrono-player #181
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "ChronoLog: Distributed Pipeline" | |
| on: | |
| pull_request: | |
| branches: [develop, main] | |
| workflow_dispatch: | |
| concurrency: | |
| group: distributed-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io | |
| jobs: | |
| chronolog-base: | |
| uses: ./.github/workflows/chronolog-base.yml | |
| permissions: | |
| contents: read | |
| packages: write | |
| distributed-deployment: | |
| runs-on: ubuntu-24.04 | |
| needs: chronolog-base | |
| permissions: | |
| contents: read | |
| packages: write | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull base Docker image | |
| run: | | |
| set -e | |
| IMAGE_NAME="${{ needs.chronolog-base.outputs.image-name }}" | |
| echo "Pulling base image: ${IMAGE_NAME}" | |
| docker pull "${IMAGE_NAME}" | |
| echo "✅ Base image pulled" | |
| - name: Set up distributed deployment | |
| run: | | |
| set -e | |
| NUM_CONTAINERS=7 | |
| NUM_KEEPERS=2 | |
| NUM_GRAPHERS=2 | |
| NUM_PLAYERS=2 | |
| # Launch all containers directly from the base image; ChronoLog is | |
| # built and installed inside c1 and propagates to the other | |
| # containers via the shared_home volume. | |
| BASE_IMAGE="${{ needs.chronolog-base.outputs.image-name }}" | |
| cat > dynamic-compose.yaml <<EOF | |
| x-common: &x-common | |
| image: ${BASE_IMAGE} | |
| init: true | |
| networks: | |
| - chronolog_net | |
| cap_add: | |
| - SYS_ADMIN | |
| - SYS_PTRACE | |
| security_opt: | |
| - seccomp:unconfined | |
| - apparmor:unconfined | |
| privileged: false | |
| mem_limit: 4g | |
| mem_reservation: 2g | |
| cpus: 2 | |
| shm_size: 512m | |
| command: > | |
| bash -c "sleep infinity" | |
| services: | |
| EOF | |
| for i in $(seq 1 $NUM_CONTAINERS); do | |
| cat >> dynamic-compose.yaml << EOF | |
| c$i: | |
| <<: *x-common | |
| hostname: c$i | |
| container_name: chronolog-c$i | |
| volumes: | |
| - shared_home:/home/grc-iit | |
| - ${{ github.workspace }}:/workspace | |
| EOF | |
| if [ $i -gt 1 ]; then | |
| cat >> dynamic-compose.yaml <<EOF | |
| depends_on: | |
| - c1 | |
| EOF | |
| fi | |
| done | |
| cat >> dynamic-compose.yaml <<EOF | |
| networks: | |
| chronolog_net: | |
| volumes: | |
| shared_home: | |
| EOF | |
| echo "✅ Generated dynamic-compose.yaml" | |
| cat dynamic-compose.yaml | |
| - name: Launch docker-compose containers | |
| id: launch | |
| run: | | |
| _start=$(date +%s) | |
| docker compose -f dynamic-compose.yaml up -d | |
| echo "✅ Containers launched" | |
| docker ps | |
| echo "duration=$(( $(date +%s) - _start ))" >> "$GITHUB_OUTPUT" | |
| - name: Wait for containers to be ready | |
| run: | | |
| echo "Waiting for containers to be ready..." | |
| sleep 10 | |
| for i in $(seq 1 7); do | |
| echo "Checking container c$i..." | |
| docker exec chronolog-c$i bash -c "echo 'Container c$i is ready'" || echo "Container c$i not ready yet" | |
| done | |
| - name: Create grc-iit user and install SSH in containers | |
| run: | | |
| set -e | |
| NUM_CONTAINERS=7 | |
| for i in $(seq 1 $NUM_CONTAINERS); do | |
| # --user root: the base image now sets USER=grc-iit (via docker | |
| # commit --change in chronolog-base.yml), so the default exec | |
| # user is grc-iit. apt-get / useradd / service ssh start all | |
| # require root. | |
| docker exec --user root chronolog-c$i bash -c " | |
| apt-get update && apt-get install -y openssh-server sudo pssh dnsutils jq chrpath || true | |
| if ! id -u grc-iit >/dev/null 2>&1; then | |
| useradd -m -s /bin/bash grc-iit | |
| echo 'grc-iit ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers | |
| mkdir -p /home/grc-iit | |
| chown -R grc-iit:grc-iit /home/grc-iit | |
| fi | |
| service ssh start || /etc/init.d/ssh start || true | |
| " | |
| done | |
| echo "✅ User grc-iit created and SSH installed in all containers" | |
| - name: Prepare SSH keys and known hosts | |
| run: | | |
| set -e | |
| NUM_CONTAINERS=7 | |
| docker exec --user grc-iit chronolog-c1 bash -c "mkdir -p /home/grc-iit/.ssh && ssh-keygen -t rsa -b 4096 -f /home/grc-iit/.ssh/id_rsa -N '' || true" | |
| docker exec --user grc-iit chronolog-c1 bash -c "cat /home/grc-iit/.ssh/id_rsa.pub > /home/grc-iit/.ssh/authorized_keys" | |
| docker exec --user grc-iit chronolog-c1 bash -c "for i in \$(seq 1 $NUM_CONTAINERS); do ssh-keyscan -t rsa,ed25519 c\$i >> /home/grc-iit/.ssh/known_hosts 2>/dev/null; done" | |
| docker exec --user root chronolog-c1 bash -c "chown -R grc-iit:grc-iit /home/grc-iit/.ssh && chmod 700 /home/grc-iit/.ssh && chmod 600 /home/grc-iit/.ssh/id_rsa && chmod 644 /home/grc-iit/.ssh/id_rsa.pub && chmod 600 /home/grc-iit/.ssh/authorized_keys" | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo 'export USER=grc-iit' >> ~/.bashrc" | |
| echo "✅ SSH keys configured (shared across all containers via shared_home volume)" | |
| - name: Copy workspace to containers | |
| run: | | |
| set -e | |
| docker exec --user root chronolog-c1 bash -c "mkdir -p /home/grc-iit/chronolog-repo && cp -r /workspace/* /home/grc-iit/chronolog-repo/ 2>/dev/null || true && chown -R grc-iit:grc-iit /home/grc-iit/chronolog-repo" | |
| echo "✅ Workspace copied to containers" | |
| - name: Fix /home/grc-iit ownership for grc-iit user | |
| run: | | |
| set -e | |
| # The base image's spack install wrote DB/lock files as root, but | |
| # the build step runs as grc-iit. Reclaim ownership of the shared | |
| # volume so grc-iit can write to the spack DB and install prefix. | |
| docker exec --user root chronolog-c1 bash -c "chown -R grc-iit:grc-iit /home/grc-iit" | |
| echo "✅ /home/grc-iit chowned to grc-iit in c1 (propagates via shared_home)" | |
| - name: Build and install ChronoLog (in c1, shared via volume) | |
| id: build-install | |
| run: | | |
| set -e | |
| _start=$(date +%s) | |
| echo '[1/5] Building and installing ChronoLog in c1 (propagates to c2..c7 via shared_home)...' | |
| docker exec --user grc-iit chronolog-c1 bash -c " | |
| set -e | |
| cd /home/grc-iit/chronolog-repo | |
| source /home/grc-iit/spack/share/spack/setup-env.sh | |
| spack env activate -p . | |
| if [ ! -d .spack-env/view ] || [ ! -d .spack-env/view/bin ] || [ ! -d .spack-env/view/lib ]; then | |
| echo 'Reconcretizing Spack environment...' | |
| spack concretize --force | |
| fi | |
| INSTALL_DIR=\"/home/grc-iit/chronolog-install\" | |
| mkdir -p \"\${INSTALL_DIR}\" | |
| ./tools/deploy/local_single_user_deploy.sh -b -I \"\${INSTALL_DIR}\" | |
| ./tools/deploy/local_single_user_deploy.sh -i -I \"\${INSTALL_DIR}\" | |
| " | |
| echo '✅ ChronoLog built and installed' | |
| echo "duration=$(( $(date +%s) - _start ))" >> "$GITHUB_OUTPUT" | |
| - name: Prepare hosts files | |
| run: | | |
| set -e | |
| NUM_CONTAINERS=7 | |
| NUM_KEEPERS=2 | |
| NUM_GRAPHERS=2 | |
| NUM_PLAYERS=2 | |
| WORK_DIR="/home/grc-iit/chronolog-install/chronolog" | |
| docker exec --user root chronolog-c1 bash -c "mkdir -p ${WORK_DIR}/conf && chown -R grc-iit:grc-iit ${WORK_DIR}" | |
| docker exec --user grc-iit chronolog-c1 bash -c "rm -rf ${WORK_DIR}/conf/hosts_*" | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c1 > ${WORK_DIR}/conf/hosts_visor" | |
| for i in $(seq 2 $(($NUM_KEEPERS + 1))); do | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c$i >> ${WORK_DIR}/conf/hosts_keeper" | |
| done | |
| for i in $(seq $(($NUM_KEEPERS + 2)) $(($NUM_KEEPERS + $NUM_GRAPHERS + 1))); do | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c$i >> ${WORK_DIR}/conf/hosts_grapher" | |
| done | |
| for i in $(seq $(($NUM_KEEPERS + $NUM_GRAPHERS + 2)) $(($NUM_KEEPERS + $NUM_GRAPHERS + $NUM_PLAYERS + 1))); do | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c$i >> ${WORK_DIR}/conf/hosts_player" | |
| done | |
| for i in $(seq 1 $NUM_CONTAINERS); do | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c$i >> ${WORK_DIR}/conf/hosts_clients" | |
| done | |
| for i in $(seq 1 $NUM_CONTAINERS); do | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c$i >> ${WORK_DIR}/conf/hosts_all" | |
| done | |
| # Client hosts file for the MPI perf test: first 4 containers | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c1 > ${WORK_DIR}/conf/hosts_client" | |
| for i in $(seq 2 4); do | |
| docker exec --user grc-iit chronolog-c1 bash -c "echo c$i >> ${WORK_DIR}/conf/hosts_client" | |
| done | |
| echo "✅ Hosts files prepared" | |
| docker exec chronolog-c1 bash -c "ls -la ${WORK_DIR}/conf/" | |
| - name: Verify Installation | |
| run: | | |
| set -e | |
| docker exec --user grc-iit chronolog-c1 bash -c " | |
| set -e | |
| WORK_DIR=\"/home/grc-iit/chronolog-install/chronolog\" | |
| REPO_DIR=\"/home/grc-iit/chronolog-repo\" | |
| cd \${REPO_DIR} | |
| source /home/grc-iit/spack/share/spack/setup-env.sh | |
| spack env activate -d . 2>/dev/null || spack env activate -p . 2>/dev/null || true | |
| echo '[2/5] Verifying installation...' | |
| if [ -d \"\${WORK_DIR}/bin\" ]; then | |
| echo '✅ bin directory found' | |
| ls -la \"\${WORK_DIR}/bin\" | |
| else | |
| echo '❌ Install failed — bin directory not found' | |
| exit 1 | |
| fi | |
| main_binaries=(chrono-visor chrono-keeper chrono-grapher chrono-player) | |
| tool_binaries=(tools/cli/chrono-client-cli tools/benchmark/chrono-bench) | |
| missing=0 | |
| for exe in \"\${main_binaries[@]}\"; do | |
| if [ ! -x \"\${WORK_DIR}/bin/\$exe\" ]; then | |
| echo \"❌ \$exe is missing or not executable!\" | |
| missing=1 | |
| else | |
| echo \"✅ \$exe is present and executable\" | |
| fi | |
| done | |
| for exe in \"\${tool_binaries[@]}\"; do | |
| if [ ! -x \"\${WORK_DIR}/\$exe\" ]; then | |
| echo \"❌ \$exe is missing or not executable!\" | |
| missing=1 | |
| else | |
| echo \"✅ \$exe is present and executable\" | |
| fi | |
| done | |
| if [ \"\$missing\" = 1 ]; then | |
| echo '❌ Installation verification failed' | |
| exit 1 | |
| fi | |
| echo '✅ Installation verification passed' | |
| " | |
| - name: Deploy ChronoLog | |
| id: deploy | |
| run: | | |
| set -e | |
| _start=$(date +%s) | |
| docker exec --user grc-iit chronolog-c1 bash -c " | |
| set -e | |
| WORK_DIR=\"/home/grc-iit/chronolog-install/chronolog\" | |
| REPO_DIR=\"/home/grc-iit/chronolog-repo\" | |
| cd \${REPO_DIR} | |
| source /home/grc-iit/spack/share/spack/setup-env.sh | |
| spack env activate -d . 2>/dev/null || spack env activate -p . 2>/dev/null || true | |
| echo '[3/5] Deploying ChronoLog...' | |
| if [ -x ./tools/deploy/single_user_deploy.sh ]; then | |
| echo '🚀 Starting ChronoLog deployment...' | |
| ./tools/deploy/single_user_deploy.sh -d -w \"\${WORK_DIR}\" | |
| else | |
| echo '❌ Deploy script not found or not executable' | |
| exit 1 | |
| fi | |
| echo '✅ ChronoLog deployment started' | |
| " | |
| echo "duration=$(( $(date +%s) - _start ))" >> "$GITHUB_OUTPUT" | |
| - name: Verify Deployment | |
| run: | | |
| set -e | |
| WORK_DIR="/home/grc-iit/chronolog-install/chronolog" | |
| echo '[4/5] Verifying deployment...' | |
| echo 'Waiting for services to start...' | |
| sleep 15 | |
| echo '📦 Listing ChronoLog processes across all containers...' | |
| PROCESS_LIST="/tmp/chrono_processes.txt" | |
| rm -f "${PROCESS_LIST}" | |
| PSSH_OUT_DIR="/tmp/pssh_processes" | |
| docker exec --user grc-iit chronolog-c1 bash -c " | |
| rm -rf ${PSSH_OUT_DIR} | |
| mkdir -p ${PSSH_OUT_DIR} | |
| parallel-ssh -h ${WORK_DIR}/conf/hosts_all -o ${PSSH_OUT_DIR} -t 30 \"ps -eo pid,args 2>/dev/null | grep -E 'chrono-visor|chrono-keeper|chrono-grapher|chrono-player' | grep -v grep || true\" | |
| for host_file in ${PSSH_OUT_DIR}/*; do | |
| host=\$(basename \"\$host_file\") | |
| if [ -f \"\$host_file\" ] && [ -s \"\$host_file\" ]; then | |
| sed \"s/^/\$host /\" \"\$host_file\" | |
| fi | |
| done | |
| rm -rf ${PSSH_OUT_DIR} | |
| " > "${PROCESS_LIST}" | |
| echo "Collected process list:" | |
| cat "${PROCESS_LIST}" || echo "No processes found" | |
| echo '🔍 Checking required components (count and host distribution)...' | |
| missing=0 | |
| WORK_DIR="/home/grc-iit/chronolog-install/chronolog" | |
| EXPECTED_VISOR_HOSTS=$(docker exec chronolog-c1 bash -c "cat ${WORK_DIR}/conf/hosts_visor 2>/dev/null" | tr '\n' ' ' | xargs) | |
| EXPECTED_KEEPER_HOSTS=$(docker exec chronolog-c1 bash -c "cat ${WORK_DIR}/conf/hosts_keeper 2>/dev/null" | tr '\n' ' ' | xargs) | |
| EXPECTED_GRAPHER_HOSTS=$(docker exec chronolog-c1 bash -c "cat ${WORK_DIR}/conf/hosts_grapher 2>/dev/null" | tr '\n' ' ' | xargs) | |
| EXPECTED_PLAYER_HOSTS=$(docker exec chronolog-c1 bash -c "cat ${WORK_DIR}/conf/hosts_player 2>/dev/null" | tr '\n' ' ' | xargs) | |
| echo "Expected host assignments:" | |
| echo " chrono-visor: ${EXPECTED_VISOR_HOSTS}" | |
| echo " chrono-keeper: ${EXPECTED_KEEPER_HOSTS}" | |
| echo " chrono-grapher: ${EXPECTED_GRAPHER_HOSTS}" | |
| echo " chrono-player: ${EXPECTED_PLAYER_HOSTS}" | |
| check_process_distribution() { | |
| local proc_name=$1 | |
| local expected_hosts=$2 | |
| local expected_count=$3 | |
| if [ ! -f "${PROCESS_LIST}" ] || [ ! -s "${PROCESS_LIST}" ]; then | |
| count=0 | |
| else | |
| count=$(grep -c "$proc_name" "${PROCESS_LIST}" 2>/dev/null || echo "0") | |
| count=$(echo "$count" | tr -d '[:space:]' | head -1) | |
| if [ -z "$count" ] || ! [[ "$count" =~ ^[0-9]+$ ]]; then | |
| count=0 | |
| fi | |
| fi | |
| if [ "$count" -ne "$expected_count" ]; then | |
| echo "❌ Wrong count for $proc_name: found $count, expected $expected_count" | |
| return 1 | |
| fi | |
| local missing_hosts="" | |
| local extra_hosts="" | |
| for host in $expected_hosts; do | |
| if ! grep -q "^${host}.*${proc_name}" "${PROCESS_LIST}" 2>/dev/null; then | |
| missing_hosts="${missing_hosts} ${host}" | |
| fi | |
| done | |
| local found_hosts=$(grep "${proc_name}" "${PROCESS_LIST}" 2>/dev/null | awk '{print $1}' | sort -u | tr '\n' ' ' | xargs) | |
| local expected_hosts_pattern=" ${expected_hosts} " | |
| for found_host in $found_hosts; do | |
| if [[ ! "${expected_hosts_pattern}" =~ " ${found_host} " ]]; then | |
| extra_hosts="${extra_hosts} ${found_host}" | |
| fi | |
| done | |
| if [ -n "$missing_hosts" ]; then | |
| echo "❌ $proc_name missing on expected hosts:${missing_hosts}" | |
| return 1 | |
| fi | |
| if [ -n "$extra_hosts" ]; then | |
| echo "❌ $proc_name found on unexpected hosts:${extra_hosts}" | |
| return 1 | |
| fi | |
| echo "✅ $proc_name: found $count processes on correct hosts (${expected_hosts})" | |
| return 0 | |
| } | |
| if ! check_process_distribution "chrono-visor" "$EXPECTED_VISOR_HOSTS" 1; then | |
| missing=1 | |
| fi | |
| if ! check_process_distribution "chrono-keeper" "$EXPECTED_KEEPER_HOSTS" 2; then | |
| missing=1 | |
| fi | |
| if ! check_process_distribution "chrono-grapher" "$EXPECTED_GRAPHER_HOSTS" 2; then | |
| missing=1 | |
| fi | |
| if ! check_process_distribution "chrono-player" "$EXPECTED_PLAYER_HOSTS" 2; then | |
| missing=1 | |
| fi | |
| if [ "$missing" != "0" ]; then | |
| echo '' | |
| echo '❌ Deployment verification failed' | |
| echo '' | |
| echo 'Debugging information:' | |
| echo 'Checking processes on each container individually:' | |
| for i in $(seq 1 7); do | |
| echo "--- Container c$i ---" | |
| docker exec chronolog-c$i ps aux | grep -E 'chrono|visor' | grep -v grep || echo "No chrono processes found" | |
| done | |
| echo '' | |
| echo 'Checking deployment logs if available:' | |
| docker exec chronolog-c1 bash -c "ls -la /home/grc-iit/chronolog-install/chronolog/monitor/ 2>/dev/null || echo 'No monitor directory'" || true | |
| exit 1 | |
| fi | |
| rm -f "${PROCESS_LIST}" | |
| echo '✅ Deployment verification passed (count and host distribution correct)' | |
| - name: Run Client Performance Test | |
| id: perf-test | |
| run: | | |
| set -e | |
| _start=$(date +%s) | |
| docker exec --user grc-iit chronolog-c1 bash -c " | |
| set -e | |
| WORK_DIR=\"/home/grc-iit/chronolog-install/chronolog\" | |
| REPO_DIR=\"/home/grc-iit/chronolog-repo\" | |
| cd \${REPO_DIR} | |
| source /home/grc-iit/spack/share/spack/setup-env.sh | |
| echo '📦 Setting up spack environment...' | |
| if spack env activate -d . 2>/dev/null; then | |
| echo '✅ Spack environment activated' | |
| else | |
| echo '⚠️ Could not activate spack environment with -d flag, trying -p flag...' | |
| spack env activate -p . 2>/dev/null || true | |
| fi | |
| if [ ! -d \"\${REPO_DIR}/.spack-env/view\" ]; then | |
| echo '📦 Spack view not found, running spack concretize...' | |
| spack concretize --force 2>&1 || echo '⚠️ spack concretize returned non-zero' | |
| fi | |
| echo '[5/5] Running client performance test...' | |
| MPIEXEC_BIN=\"\" | |
| if [ -x \"\${REPO_DIR}/.spack-env/view/bin/mpiexec\" ]; then | |
| MPIEXEC_BIN=\"\${REPO_DIR}/.spack-env/view/bin/mpiexec\" | |
| echo \"Found mpiexec in spack view\" | |
| elif command -v mpiexec &> /dev/null; then | |
| MPIEXEC_BIN=\$(which mpiexec) | |
| echo \"Found mpiexec in PATH\" | |
| else | |
| echo \"Searching for mpiexec in spack opt directory...\" | |
| MPIEXEC_BIN=\$(find /home/grc-iit/spack/opt/spack -name mpiexec \\( -type f -o -type l \\) -executable 2>/dev/null | head -1) | |
| if [ -z \"\${MPIEXEC_BIN}\" ]; then | |
| MPIEXEC_BIN=\$(find /home/grc-iit/spack/opt/spack -name mpiexec 2>/dev/null | head -1) | |
| fi | |
| fi | |
| if [ -z \"\${MPIEXEC_BIN}\" ] || [ ! -x \"\${MPIEXEC_BIN}\" ]; then | |
| echo '❌ mpiexec not found' | |
| exit 1 | |
| fi | |
| echo \"✅ Using mpiexec at: \${MPIEXEC_BIN}\" | |
| export LD_LIBRARY_PATH=\${WORK_DIR}/lib:\$LD_LIBRARY_PATH | |
| cd \${WORK_DIR} | |
| if timeout 300 \${MPIEXEC_BIN} -l -n 4 -f conf/hosts_client \ | |
| tools/benchmark/chrono-bench \ | |
| --config conf/default-chrono-client-conf.json \ | |
| -a 4096 -b 4096 -s 4096 -n 1024 -t 1 -h 1 -p -y 2>&1 | tee /tmp/mpi_test_output.log; then | |
| echo '✅ Client performance test completed successfully' | |
| else | |
| TEST_EXIT_CODE=\$? | |
| if [ \$TEST_EXIT_CODE -eq 124 ]; then | |
| echo '❌ Client performance test timed out after 5 minutes' | |
| else | |
| echo \"❌ Client performance test failed with exit code \${TEST_EXIT_CODE}\" | |
| fi | |
| if [ -f /tmp/mpi_test_output.log ]; then | |
| tail -50 /tmp/mpi_test_output.log || true | |
| fi | |
| exit \${TEST_EXIT_CODE} | |
| fi | |
| " | |
| echo "duration=$(( $(date +%s) - _start ))" >> "$GITHUB_OUTPUT" | |
| # Capture the perf-test output for the job summary (tail, to cap size). | |
| { | |
| echo "output<<PERF_EOF" | |
| docker exec --user grc-iit chronolog-c1 tail -80 /tmp/mpi_test_output.log 2>/dev/null || echo "(no output captured)" | |
| echo "PERF_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Stop ChronoLog | |
| if: always() | |
| run: | | |
| docker exec --user grc-iit chronolog-c1 bash -c " | |
| WORK_DIR=\"/home/grc-iit/chronolog-install/chronolog\" | |
| cd /home/grc-iit/chronolog-repo 2>/dev/null || true | |
| source /home/grc-iit/spack/share/spack/setup-env.sh 2>/dev/null || true | |
| spack env activate -d . 2>/dev/null || spack env activate -p . 2>/dev/null || true | |
| echo 'Stopping ChronoLog...' | |
| if [ -x ./tools/deploy/single_user_deploy.sh ]; then | |
| ./tools/deploy/single_user_deploy.sh -s -w \"\${WORK_DIR}\" 2>/dev/null || true | |
| fi | |
| " || echo "⚠️ Stop command had errors but continuing cleanup" | |
| - name: Clean up containers | |
| if: always() | |
| run: | | |
| docker compose -f dynamic-compose.yaml down -v | |
| echo "✅ Containers cleaned up" | |
| - name: Job summary | |
| if: always() | |
| run: | | |
| fmt_duration() { | |
| local s=${1:-0} | |
| if [ "$s" -ge 60 ]; then | |
| printf "%dm %02ds" $(( s / 60 )) $(( s % 60 )) | |
| else | |
| printf "%ds" "$s" | |
| fi | |
| } | |
| LAUNCH_D=${{ steps.launch.outputs.duration || 0 }} | |
| BUILD_D=${{ steps.build-install.outputs.duration || 0 }} | |
| DEPLOY_D=${{ steps.deploy.outputs.duration || 0 }} | |
| PERF_D=${{ steps.perf-test.outputs.duration || 0 }} | |
| PERF_OUTCOME="${{ steps.perf-test.outcome }}" | |
| PERF_OUTPUT=$(cat <<'PERF_SUMMARY_EOF' | |
| ${{ steps.perf-test.outputs.output }} | |
| PERF_SUMMARY_EOF | |
| ) | |
| BRANCH="${{ github.head_ref || github.ref_name }}" | |
| case "$PERF_OUTCOME" in | |
| success) PERF_ICON="✅" ;; | |
| failure) PERF_ICON="❌" ;; | |
| *) PERF_ICON="⚠️" ;; | |
| esac | |
| { | |
| echo "## 🌐 ChronoLog Distributed Pipeline" | |
| echo "" | |
| echo "Branch: \`${BRANCH}\` · Commit: \`${GITHUB_SHA:0:7}\`" | |
| echo "" | |
| echo "### 🧩 Topology" | |
| echo "" | |
| echo "7 containers launched from the base image (\`c1\`..\`c7\`):" | |
| echo "" | |
| echo "| Role | Hosts | Count |" | |
| echo "|------|-------|-------|" | |
| echo "| chrono-visor | c1 | 1 |" | |
| echo "| chrono-keeper | c2, c3 | 2 |" | |
| echo "| chrono-grapher | c4, c5 | 2 |" | |
| echo "| chrono-player | c6, c7 | 2 |" | |
| echo "" | |
| echo "### ⏱️ Stage timings" | |
| echo "" | |
| echo "| Stage | Duration |" | |
| echo "|-------|----------|" | |
| echo "| Launch compose containers | $(fmt_duration "$LAUNCH_D") |" | |
| echo "| Build + install ChronoLog | $(fmt_duration "$BUILD_D") |" | |
| echo "| Deploy ChronoLog (distributed) | $(fmt_duration "$DEPLOY_D") |" | |
| echo "| Client performance test | $(fmt_duration "$PERF_D") |" | |
| echo "" | |
| echo "### 🚀 Client performance test" | |
| echo "" | |
| echo "${PERF_ICON} Outcome: \`${PERF_OUTCOME}\` (4 MPI ranks, 1024 events × 4096 B)" | |
| echo "" | |
| if [ -n "$(echo "$PERF_OUTPUT" | tr -d '[:space:]')" ]; then | |
| echo "<details><summary>Last 80 lines of MPI output</summary>" | |
| echo "" | |
| echo '```' | |
| echo "$PERF_OUTPUT" | |
| echo '```' | |
| echo "" | |
| echo "</details>" | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" |