Add auto-disable for explorers with unreachable RPCs

When an explorer's RPC becomes unreachable, the explorerSyncCheck job
keeps trying to start PM2 processes that immediately fail. This creates
unnecessary load on the PM2 server.

This change adds automatic sync deactivation after 3 consecutive failures:

- Track sync failures on Explorer model (syncFailedAttempts, syncDisabledAt,
  syncDisabledReason, nextRecoveryCheckAt)
- Auto-disable shouldSync after 3 consecutive RPC failures or PM2 timeouts
- Add syncRecoveryCheck job that periodically tests disabled explorers
- Use exponential backoff for recovery checks (5m -> 15m -> 1h -> 6h)
- Reset failure tracking when sync is manually re-enabled

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: Antoine de Chevigné

run/jobs/index.js

@@ -45,6 +45,7 @@ module.exports = {
     rpcHealthCheck: require('./rpcHealthCheck'),
     rpcHealthCheckStarter: require('./rpcHealthCheckStarter'),
     explorerSyncCheck: require('./explorerSyncCheck'),
+    syncRecoveryCheck: require('./syncRecoveryCheck'),
     workspaceReset: require('./workspaceReset'),
     batchBlockDelete: require('./batchBlockDelete'),
     batchContractDelete: require('./batchContractDelete'),

run/jobs/syncRecoveryCheck.js

@@ -0,0 +1,74 @@
+/**
+ * @fileoverview Sync recovery check job.
+ * Periodically checks auto-disabled explorers to see if their RPC has recovered.
+ * Re-enables sync when RPC becomes reachable, using exponential backoff for retries.
+ *
+ * Backoff schedule: 5m -> 15m -> 1h -> 6h (max)
+ *
+ * @module jobs/syncRecoveryCheck
+ */
+
+const { Explorer, Workspace } = require('../models');
+const { ProviderConnector } = require('../lib/rpc');
+const { withTimeout } = require('../lib/utils');
+const { Op } = require('sequelize');
+const logger = require('../lib/logger');
+
+module.exports = async () => {
+    // Find explorers that are due for a recovery check
+    const explorers = await Explorer.findAll({
+        where: {
+            syncDisabledReason: { [Op.ne]: null },
+            nextRecoveryCheckAt: { [Op.lte]: new Date() }
+        },
+        include: [{
+            model: Workspace,
+            as: 'workspace',
+            attributes: ['id', 'rpcServer']
+        }]
+    });
+
+    if (explorers.length === 0) {
+        return 'No explorers due for recovery check';
+    }
+
+    let recovered = 0;
+    let stillUnreachable = 0;
+
+    for (const explorer of explorers) {
+        try {
+            const provider = new ProviderConnector(explorer.workspace.rpcServer);
+            const block = await withTimeout(provider.fetchLatestBlock());
+
+            if (block) {
+                // RPC is reachable - re-enable sync
+                await explorer.enableSyncAfterRecovery();
+                recovered++;
+
+                logger.info({
+                    message: 'Explorer re-enabled after RPC recovery',
+                    explorerId: explorer.id,
+                    explorerSlug: explorer.slug,
+                    disabledReason: explorer.syncDisabledReason
+                });
+            } else {
+                // Block fetch returned null - still unreachable
+                await explorer.scheduleNextRecoveryCheck();
+                stillUnreachable++;
+            }
+        } catch (error) {
+            // RPC check failed - schedule next check with backoff
+            await explorer.scheduleNextRecoveryCheck();
+            stillUnreachable++;
+
+            logger.debug({
+                message: 'Explorer recovery check failed',
+                explorerId: explorer.id,
+                explorerSlug: explorer.slug,
+                error: error.message
+            });
+        }
+    }
+
+    return `Checked ${explorers.length} explorers: ${recovered} recovered, ${stillUnreachable} still unreachable`;
+};

run/jobs/updateExplorerSyncingProcess.js

@@ -1,11 +1,15 @@
 /**
  * @fileoverview Explorer sync process update job.
  * Manages PM2 processes for explorer block syncing based on status.
+ * Auto-disables sync after repeated RPC failures with exponential backoff recovery.
  * @module jobs/updateExplorerSyncingProcess
  */
 
 const { Explorer, Workspace, RpcHealthCheck, StripeSubscription, StripePlan } = require('../models');
 const PM2 = require('../lib/pm2');
+const logger = require('../lib/logger');
+
+const SYNC_FAILURE_THRESHOLD = 3;
 
 module.exports = async job => {
     const data = job.data;
@@ -57,7 +61,19 @@ module.exports = async job => {
         }
         else if (explorer.workspace.rpcHealthCheck && !explorer.workspace.rpcHealthCheck.isReachable && existingProcess) {
             await pm2.delete(explorer.slug);
-            return 'Process deleted: RPC is not reachable.';
+            // Track RPC failure and potentially auto-disable
+            const result = await explorer.incrementSyncFailures('rpc_unreachable');
+            if (result.disabled) {
+                logger.info({
+                    message: 'Explorer auto-disabled due to RPC failures',
+                    explorerId: explorer.id,
+                    explorerSlug: explorer.slug,
+                    attempts: result.attempts,
+                    reason: 'rpc_unreachable'
+                });
+                return `Process deleted and sync auto-disabled after ${result.attempts} RPC failures.`;
+            }
+            return `Process deleted: RPC is not reachable (attempt ${result.attempts}/${SYNC_FAILURE_THRESHOLD}).`;
         }
         else if (!explorer.shouldSync && existingProcess) {
             await pm2.delete(explorer.slug);
@@ -69,17 +85,41 @@ module.exports = async job => {
         }
         else if (explorer.shouldSync && !existingProcess) {
             await pm2.start(explorer.slug, explorer.workspaceId);
+            // Reset failure counter on successful start
+            if (explorer.syncFailedAttempts > 0) {
+                await explorer.update({ syncFailedAttempts: 0 });
+            }
             return 'Process started.';
         }
         else if (explorer.shouldSync && existingProcess && existingProcess.pm2_env.status == 'stopped') {
             await pm2.resume(explorer.slug, explorer.workspaceId);
+            // Reset failure counter on successful resume
+            if (explorer.syncFailedAttempts > 0) {
+                await explorer.update({ syncFailedAttempts: 0 });
+            }
             return 'Process resumed.';
         }
         else
             return 'No process change.';
     } catch(error) {
-        if (error.message.startsWith('Timed out after'))
+        if (error.message.startsWith('Timed out after')) {
+            // Track timeout as a failure if explorer exists and sync is enabled
+            if (explorer && explorer.shouldSync) {
+                const result = await explorer.incrementSyncFailures('pm2_timeout');
+                if (result.disabled) {
+                    logger.info({
+                        message: 'Explorer auto-disabled due to PM2 timeouts',
+                        explorerId: explorer.id,
+                        explorerSlug: explorer.slug,
+                        attempts: result.attempts,
+                        reason: 'pm2_timeout'
+                    });
+                    return `Timed out and sync auto-disabled after ${result.attempts} failures.`;
+                }
+                return `Timed out (attempt ${result.attempts}/${SYNC_FAILURE_THRESHOLD})`;
+            }
             return 'Timed out';
+        }
         else
             throw error;
     }

run/migrations/20260109161142-add-sync-failure-tracking.js

@@ -0,0 +1,59 @@
+'use strict';
+
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    const transaction = await queryInterface.sequelize.transaction();
+    try {
+      await queryInterface.addColumn('explorers', 'syncFailedAttempts', {
+        type: Sequelize.DataTypes.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      }, { transaction });
+
+      await queryInterface.addColumn('explorers', 'syncDisabledAt', {
+        type: Sequelize.DataTypes.DATE,
+        allowNull: true
+      }, { transaction });
+
+      await queryInterface.addColumn('explorers', 'syncDisabledReason', {
+        type: Sequelize.DataTypes.STRING,
+        allowNull: true
+      }, { transaction });
+
+      await queryInterface.addColumn('explorers', 'nextRecoveryCheckAt', {
+        type: Sequelize.DataTypes.DATE,
+        allowNull: true
+      }, { transaction });
+
+      // Add index on nextRecoveryCheckAt for efficient recovery job queries
+      await queryInterface.addIndex('explorers', ['nextRecoveryCheckAt'], {
+        name: 'explorers_next_recovery_check_at_idx',
+        where: { nextRecoveryCheckAt: { [Sequelize.Op.ne]: null } },
+        transaction
+      });
+
+      await transaction.commit();
+    } catch(error) {
+      console.log(error);
+      await transaction.rollback();
+      throw error;
+    }
+  },
+
+  async down(queryInterface, Sequelize) {
+    const transaction = await queryInterface.sequelize.transaction();
+    try {
+      await queryInterface.removeIndex('explorers', 'explorers_next_recovery_check_at_idx', { transaction });
+      await queryInterface.removeColumn('explorers', 'syncFailedAttempts', { transaction });
+      await queryInterface.removeColumn('explorers', 'syncDisabledAt', { transaction });
+      await queryInterface.removeColumn('explorers', 'syncDisabledReason', { transaction });
+      await queryInterface.removeColumn('explorers', 'nextRecoveryCheckAt', { transaction });
+
+      await transaction.commit();
+    } catch(error) {
+      console.log(error);
+      await transaction.rollback();
+      throw error;
+    }
+  }
+};

run/models/explorer.js

@@ -31,6 +31,15 @@ const IUniswapV2Router02 = require('../lib/abis/IUniswapV2Router02.json');
 const IUniswapV2Factory = require('../lib/abis/IUniswapV2Factory.json');
 const MAX_RPC_ATTEMPTS = 3;
 
+// Sync failure auto-disable configuration
+const SYNC_FAILURE_THRESHOLD = 3;
+const RECOVERY_BACKOFF_SCHEDULE = [
+    5 * 60 * 1000,       // 5 minutes
+    15 * 60 * 1000,      // 15 minutes
+    60 * 60 * 1000,      // 1 hour
+    6 * 60 * 60 * 1000   // 6 hours (max)
+];
+
 module.exports = (sequelize, DataTypes) => {
   class Explorer extends Model {
     /**
@@ -299,10 +308,18 @@ module.exports = (sequelize, DataTypes) => {
 
     /**
      * Starts block synchronization for the explorer.
+     * Resets all failure tracking when manually enabling sync.
      * @returns {Promise<Explorer>} Updated explorer
      */
-    startSync() {
-        return this.update({ shouldSync: true });
+    async startSync() {
+        await this.update({
+            shouldSync: true,
+            syncFailedAttempts: 0,
+            syncDisabledAt: null,
+            syncDisabledReason: null,
+            nextRecoveryCheckAt: null
+        });
+        return this;
     }
 
     /**
@@ -313,6 +330,101 @@ module.exports = (sequelize, DataTypes) => {
         return this.update({ shouldSync: false });
     }
 
+    /**
+     * Increments the sync failure counter and auto-disables if threshold reached.
+     * @param {string} [reason='rpc_unreachable'] - Reason for the failure
+     * @returns {Promise<{disabled: boolean, attempts: number}>} Result with disable status
+     */
+    async incrementSyncFailures(reason = 'rpc_unreachable') {
+        const newCount = (this.syncFailedAttempts || 0) + 1;
+        await this.update({ syncFailedAttempts: newCount });
+
+        if (newCount >= SYNC_FAILURE_THRESHOLD) {
+            await this.autoDisableSync(reason);
+            return { disabled: true, attempts: newCount };
+        }
+        return { disabled: false, attempts: newCount };
+    }
+
+    /**
+     * Auto-disables sync and schedules first recovery check.
+     * @param {string} reason - Reason for disabling (e.g., 'rpc_unreachable')
+     * @returns {Promise<Explorer>} Updated explorer
+     */
+    async autoDisableSync(reason) {
+        const nextCheck = new Date(Date.now() + RECOVERY_BACKOFF_SCHEDULE[0]);
+        await this.update({
+            shouldSync: false,
+            syncDisabledAt: new Date(),
+            syncDisabledReason: reason,
+            nextRecoveryCheckAt: nextCheck
+        });
+        return this;
+    }
+
+    /**
+     * Resets all sync failure tracking state.
+     * @returns {Promise<Explorer>} Updated explorer
+     */
+    async resetSyncState() {
+        await this.update({
+            syncFailedAttempts: 0,
+            syncDisabledAt: null,
+            syncDisabledReason: null,
+            nextRecoveryCheckAt: null
+        });
+        return this;
+    }
+
+    /**
+     * Schedules the next recovery check using exponential backoff.
+     * Backoff schedule: 5m -> 15m -> 1h -> 6h (max)
+     * @returns {Promise<Explorer>} Updated explorer
+     */
+    async scheduleNextRecoveryCheck() {
+        if (!this.syncDisabledAt) {
+            return this;
+        }
+
+        const timeSinceDisabled = Date.now() - new Date(this.syncDisabledAt).getTime();
+        let cumulativeTime = 0;
+        let backoffIndex = 0;
+
+        // Find which backoff interval we should use based on time since disabled
+        for (let i = 0; i < RECOVERY_BACKOFF_SCHEDULE.length; i++) {
+            cumulativeTime += RECOVERY_BACKOFF_SCHEDULE[i];
+            if (timeSinceDisabled < cumulativeTime) {
+                backoffIndex = i;
+                break;
+            }
+            backoffIndex = i;
+        }
+
+        // Cap at max backoff (last element)
+        if (backoffIndex >= RECOVERY_BACKOFF_SCHEDULE.length) {
+            backoffIndex = RECOVERY_BACKOFF_SCHEDULE.length - 1;
+        }
+
+        const nextCheck = new Date(Date.now() + RECOVERY_BACKOFF_SCHEDULE[backoffIndex]);
+        await this.update({ nextRecoveryCheckAt: nextCheck });
+        return this;
+    }
+
+    /**
+     * Re-enables sync after successful recovery check.
+     * @returns {Promise<Explorer>} Updated explorer
+     */
+    async enableSyncAfterRecovery() {
+        await this.update({
+            shouldSync: true,
+            syncFailedAttempts: 0,
+            syncDisabledAt: null,
+            syncDisabledReason: null,
+            nextRecoveryCheckAt: null
+        });
+        return this;
+    }
+
     /**
      * Creates a Uniswap V2 compatible DEX for the explorer.
      * @param {string} routerAddress - DEX router contract address
@@ -690,7 +802,11 @@ module.exports = (sequelize, DataTypes) => {
     shouldEnforceQuota: DataTypes.BOOLEAN,
     isDemo: DataTypes.BOOLEAN,
     gasAnalyticsEnabled: DataTypes.BOOLEAN,
-    displayTopAccounts: DataTypes.BOOLEAN
+    displayTopAccounts: DataTypes.BOOLEAN,
+    syncFailedAttempts: DataTypes.INTEGER,
+    syncDisabledAt: DataTypes.DATE,
+    syncDisabledReason: DataTypes.STRING,
+    nextRecoveryCheckAt: DataTypes.DATE
   }, {
     hooks: {
         afterCreate(explorer, options) {

run/scheduler.js

@@ -3,6 +3,7 @@ const { enqueue } = require('./lib/queue');
 const INTEGRITY_CHECK_INTERVAL = 5 * 60 * 1000;
 const RPC_HEALTH_CHECK_INTERVAL = 5 * 60 * 1000;
 const SUBSCRIPTION_CHECK_INTERVAL = 5 * 60 * 1000;
+const SYNC_RECOVERY_CHECK_INTERVAL = 5 * 60 * 1000;
 const QUEUE_MONITORING_INTERVAL = 60 * 1000;
 const CANCEL_DEMO_INTERVAL = 60 * 60 * 1000;
 const BLOCK_SYNC_MONITORING_INTERVAL = 60 * 1000;
@@ -48,6 +49,14 @@ const BLOCK_SYNC_MONITORING_INTERVAL = 60 * 1000;
         { every: SUBSCRIPTION_CHECK_INTERVAL }
     );
 
+    await enqueue(
+        'syncRecoveryCheck',
+        'syncRecoveryCheck',
+        {},
+        10,
+        { every: SYNC_RECOVERY_CHECK_INTERVAL }
+    );
+
     await enqueue(
         'queueMonitoring',
         'queueMonitoring',