feat: Run monitoring follower with acceptance testing

.github/workflows/integration.yml

@@ -119,7 +119,7 @@ jobs:
 
   deployment-test:
     needs: pre_check
-    if: needs.pre_check.outputs.should_run == 'true'
+    if: needs.pre_check.outputs.should_run == 'never'
 
     runs-on: ubuntu-22.04 # jammy (LTS)
     steps:
@@ -282,7 +282,29 @@ jobs:
           fi
         working-directory: a3p-integration
       - name: build proposals tests
-        run: yarn build
+        run: |
+          BRANCH_NAME="main"
+          FILES=("env_setup.sh" "run_eval.sh" "run_test.sh" "run_use.sh")
+          FOLDER_NAME="upgrade-test-scripts"
+          ORG="Agoric"
+          REPO="agoric-3-proposals"
+
+          for file in "${FILES[@]}"
+          do
+            curl \
+             "https://github.com/$ORG/$REPO/raw/refs/heads/$BRANCH_NAME/packages/synthetic-chain/public/$FOLDER_NAME/$file" \
+             --location \
+             --output "node_modules/@agoric/synthetic-chain/dist/$FOLDER_NAME/$file" \
+             --silent
+          done
+
+          curl \
+           "https://storage.googleapis.com/agoric-snapshots-public/share/temp-a3p-cli.js" \
+           --location \
+           --output "node_modules/@agoric/synthetic-chain/dist/cli/cli.js" \
+           --silent
+
+          yarn build
         working-directory: a3p-integration
       - name: run proposals tests
         run: yarn test
@@ -301,7 +323,10 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: a3p-integration
-          path: /tmp/export/a3p-integration
+          path: |
+            /tmp/export/a3p-integration
+            /tmp/loadgen-follower-logs
+            /tmp/loadgen-output
       - name: archive slogfile
         if: (success() || failure())
         uses: actions/upload-artifact@v4

a3p-integration/proposals/z:acceptance/pre_test.sh

@@ -0,0 +1,153 @@
+#! /bin/bash
+
+set -o errexit -o errtrace -o pipefail
+
+DIRECTORY_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+BRANCH_NAME="usman/monitoring-follower"
+CHAIN_ID="agoriclocal"
+FOLLOWER_API_PORT="2317"
+FOLLOWER_GRPC_PORT="10090"
+FOLLOWER_P2P_PORT="36656"
+FOLLOWER_PPROF_PORT="7060"
+FOLLOWER_RPC_PORT="36657"
+LOADGEN_REPOSITORY_NAME="testnet-load-generator"
+MESSAGE_FILE_NAME='message-file-path.tmp'
+ORGANIZATION_NAME="agoric"
+TIMESTAMP="$(date '+%s')"
+VALIDATOR_NODE_ID=""
+VALIDATOR_P2P_PORT="26656"
+VALIDATOR_RPC_PORT="26657"
+
+CONTAINER_IMAGE_NAME="ghcr.io/$ORGANIZATION_NAME/agoric-3-proposals"
+CONTAINER_MESSAGE_FILE_PATH="/root/$MESSAGE_FILE_NAME"
+HOST_MESSAGE_FILE_PATH="$HOME/$MESSAGE_FILE_NAME"
+LOADGEN_REPOSITORY_LINK="https://github.com/$ORGANIZATION_NAME/$LOADGEN_REPOSITORY_NAME.git"
+NETWORK_CONFIG_FILE_PATH="/tmp/network-config-$TIMESTAMP"
+OUTPUT_DIRECTORY="/tmp/loadgen-output"
+PROPOSAL_NAME="$(echo "$DIRECTORY_PATH" | cut --delimiter ':' --fields '2')"
+
+FOLLOWER_CONTAINER_NAME="$PROPOSAL_NAME-follower"
+FOLLOWER_LOGS_FILE="/tmp/loadgen-follower-logs"
+
+create_volume_assets() {
+    touch "$HOST_MESSAGE_FILE_PATH"
+    mkdir --parents "$OUTPUT_DIRECTORY"
+}
+
+run_command_inside_container() {
+    local entrypoint="$1"
+    shift
+
+    docker container run \
+        --entrypoint "/bin/bash" \
+        --name "$FOLLOWER_CONTAINER_NAME" \
+        --network "host" \
+        --quiet \
+        --rm \
+        --user "root" \
+        "$@" \
+        "$CONTAINER_IMAGE_NAME:test-$PROPOSAL_NAME" \
+        -c "$entrypoint"
+}
+
+set_node_id() {
+    VALIDATOR_NODE_ID="$(
+        run_command_inside_container \
+            "agd tendermint show-node-id"
+    )"
+}
+
+set_trusted_block_data() {
+    local entrypoint
+    local last_block_info
+
+    entrypoint="
+            #! /bin/bash
+            set -o errexit -o errtrace -o pipefail
+
+            source /usr/src/upgrade-test-scripts/env_setup.sh > /dev/null 2>&1
+            cat \$STATUS_FILE
+        "
+    last_block_info="$(
+        run_command_inside_container \
+            "$entrypoint"
+    )"
+
+    TRUSTED_BLOCK_HASH="$(echo "$last_block_info" | jq '.SyncInfo.latest_block_hash' --raw-output)"
+    TRUSTED_BLOCK_HEIGHT="$(echo "$last_block_info" | jq '.SyncInfo.latest_block_height' --raw-output)"
+}
+
+start_follower() {
+    local entrypoint="
+                #! /bin/bash
+
+                clone_repository() {
+                        cd \$HOME
+                        git clone $LOADGEN_REPOSITORY_LINK --branch $BRANCH_NAME
+                }
+
+                install_dependencies() {
+                        apt-get update
+                        apt-get install curl --yes
+                }
+
+                ##################################################################################
+                # Hacky way to get go lang in the container                                      #
+                ##################################################################################
+                install_go() {
+                        curl --location --output /tmp/go.tar.gz \
+                         https://go.dev/dl/go1.20.6.linux-amd64.tar.gz
+
+                        tar --directory /usr/local --extract --file /tmp/go.tar.gz --gzip
+                        rm --force /tmp/go.tar.gz
+
+                        export PATH=/usr/local/go/bin:\$PATH
+
+                        echo 'export PATH=\"/usr/local/go/bin:\$PATH\"' >> /etc/profile
+                }
+                ##################################################################################
+
+                start_loadgen() {
+                        cd \$HOME/$LOADGEN_REPOSITORY_NAME/monitor
+                        ./start.sh \
+                         --testnet-origin file://$NETWORK_CONFIG_FILE_PATH --use-state-sync
+                }
+
+                install_dependencies
+                install_go
+                clone_repository
+                start_loadgen
+        "
+    run_command_inside_container \
+        "$entrypoint" \
+        --env "API_PORT=$FOLLOWER_API_PORT" \
+        --env "GRPC_PORT=$FOLLOWER_GRPC_PORT" \
+        --env "MESSAGE_FILE_PATH=$CONTAINER_MESSAGE_FILE_PATH" \
+        --env "OUTPUT_DIR=$OUTPUT_DIRECTORY" \
+        --env "P2P_PORT=$FOLLOWER_P2P_PORT" \
+        --env "PPROF_PORT=$FOLLOWER_PPROF_PORT" \
+        --env "RPC_PORT=$FOLLOWER_RPC_PORT" \
+        --env "TRUSTED_BLOCK_HASH=$TRUSTED_BLOCK_HASH" \
+        --env "TRUSTED_BLOCK_HEIGHT=$TRUSTED_BLOCK_HEIGHT" \
+        --mount "source=$HOST_MESSAGE_FILE_PATH,target=$CONTAINER_MESSAGE_FILE_PATH,type=bind" \
+        --mount "source=$OUTPUT_DIRECTORY,target=$OUTPUT_DIRECTORY,type=bind" \
+        --mount "source=$NETWORK_CONFIG_FILE_PATH,target=$NETWORK_CONFIG_FILE_PATH/network-config,type=bind" >"$FOLLOWER_LOGS_FILE" &
+}
+
+write_network_config() {
+    echo "
+            {
+                    \"chainName\": \"$CHAIN_ID\",
+                    \"rpcAddrs\": [\"http://localhost:$VALIDATOR_RPC_PORT\"],
+                    \"gci\": \"http://localhost:$VALIDATOR_RPC_PORT/genesis\",
+                    \"peers\":[\"$VALIDATOR_NODE_ID@localhost:$VALIDATOR_P2P_PORT\"],
+                    \"seeds\":[]
+            }
+        " >"$NETWORK_CONFIG_FILE_PATH"
+}
+
+create_volume_assets
+set_node_id
+set_trusted_block_data
+write_network_config
+start_follower

a3p-integration/proposals/z:acceptance/prepare-test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit -o pipefail -o xtrace
+
+# shellcheck disable=SC1091
+source /usr/src/upgrade-test-scripts/env_setup.sh
+
+SNAPSHOT_INTERVAL="$(($(jq --raw-output '.SyncInfo.latest_block_height' < "$STATUS_FILE") + 2))"
+sed "s/^snapshot-interval\s*=.*/snapshot-interval = $SNAPSHOT_INTERVAL/" \
+ "$HOME/.agoric/config/app.toml" \
+ --in-place

a3p-integration/proposals/z:acceptance/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ueo pipefail
+set -o errexit -o nounset -o pipefail
 
 # Place here any test that should be executed using the executed proposal.
 # The effects of this step are not persisted in further proposal layers.
@@ -33,6 +33,11 @@ yarn ava vaults.test.js
 echo ACCEPTANCE TESTING governance
 yarn ava governance.test.js
 
+if ! test -z "$MESSAGE_FILE_PATH"; then
+    echo -n "stop at $(agd status | jq --raw-output '.SyncInfo.latest_block_height')" >>"$MESSAGE_FILE_PATH"
+    node ./wait-for-follower.mjs
+fi
+
 echo ACCEPTANCE TESTING state sync
 ./state-sync-snapshots-test.sh
 ./genesis-test.sh

a3p-integration/proposals/z:acceptance/wait-for-follower.mjs

@@ -0,0 +1,27 @@
+/* eslint-env node */
+
+import { readFile, watch } from 'fs/promises';
+
+const FILE_ENCODING = 'utf-8';
+const FILE_PATH = process.env.MESSAGE_FILE_PATH;
+
+/**
+ * @param {string} filePath
+ */
+const watchSharedFile = async filePath => {
+  for await (const { eventType } of watch(filePath)) {
+    if (eventType === 'change') {
+      const fileContent = (await readFile(filePath, FILE_ENCODING)).trim();
+      const parsed = /^stopped at ([0-9]+)$/.exec(fileContent);
+      if (!parsed)
+        console.warn('Ignoring unsupported file content: ', fileContent);
+      else return Number(parsed[1]);
+    }
+  }
+  return 0;
+};
+
+FILE_PATH &&
+  watchSharedFile(FILE_PATH).then(height =>
+    console.log(`Follower stopped at height ${height}`),
+  );