Chain fault testing framework + Enable auto-rejoin

rust/lit-core/lit-blockchain/Cargo.toml

@@ -8,6 +8,8 @@ edition.workspace = true
 [features]
 default = []
 env-override = []
+testing = []
+proxy_chatter = []
 
 [dependencies]
 alloy.workspace = true

rust/lit-core/lit-blockchain/src/contracts/mod.rs

@@ -145,6 +145,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -207,6 +209,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -318,6 +322,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -363,6 +369,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -428,6 +436,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -525,6 +535,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -571,13 +583,41 @@ pub fn default_local_client(
     let chain = cfg.blockchain_chain_name()?;
     let wallet = load_wallet(cfg, wallet_key)?;
     let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+    #[cfg(feature = "proxy_chatter")]
+    let provider = change_port(provider, cfg)?;
 
     Ok(Arc::new(SignerMiddleware::new(provider, wallet)))
 }
 
 pub fn default_local_client_no_wallet(cfg: &LitConfig) -> Result<Arc<Provider<Http>>> {
     let chain = cfg.blockchain_chain_name()?;
     let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+    #[cfg(feature = "proxy_chatter")]
+    let provider = change_port(provider, cfg)?;
 
     Ok(provider)
 }
+
+#[cfg(feature = "proxy_chatter")]
+pub fn change_port(provider: Arc<Provider<Http>>, cfg: &LitConfig) -> Result<Arc<Provider<Http>>> {
+    let chain = cfg.blockchain_chain_name()?;
+    if chain.to_lowercase() == "localchain" {
+        let cfg2 = cfg.config();
+        let port = cfg2
+            .get_int("node.http.port")
+            .map_err(|e| crate::error::unexpected_err(e.to_string(), None))?;
+        let port = port as u16;
+        let port = 11075 + port; // 10000 + 8545 ( anvil default port ) - 7470 ( lit-node default starting port ) + actual port value
+        tracing::trace!(
+            "Changing port for proxy provider {} to {}.",
+            provider.url().as_str(),
+            port
+        );
+        let mut proxy_provider = (*provider).clone();
+        proxy_provider.url_mut().set_port(Some(port)).map_err(|_e| {
+            crate::error::unexpected_err("Could not set port for proxy provider", None)
+        })?;
+        return Ok(Arc::new(proxy_provider));
+    }
+    Ok(provider)
+}

rust/lit-node/lit-node-testnet/Cargo.toml

@@ -8,7 +8,6 @@ edition.workspace = true
 # this needs to happen at runtime, so is just a place holder for now.
 sign_test = ["lit-actions"] #enables unsafe endpoint allowing a test to use a PK directly .
 lit-actions = []
-proxy-chatter = []
 proxy_chatter = []
 lit-actions-server = ["lit-actions"] # start internal lit_actions server for testing
 testing = [] # enables testing features

rust/lit-node/lit-node-testnet/src/lib.rs

@@ -29,6 +29,7 @@ pub struct TestSetupBuilder {
     register_inactive_validators: bool,
     enable_payment: Option<String>,
     chain_polling_interval: Option<String>,
+    signing_round_timeout: Option<String>,
     epoch_length: Option<U256>,
     max_presign_count: Option<u64>,
     min_presign_count: Option<u64>,
@@ -51,6 +52,7 @@ impl Default for TestSetupBuilder {
             register_inactive_validators: false,
             enable_payment: Some("true".to_string()),
             chain_polling_interval: None,
+            signing_round_timeout: None,
             epoch_length: None,
             max_presign_count: Some(0),
             min_presign_count: Some(0),
@@ -114,6 +116,11 @@ impl TestSetupBuilder {
         self
     }
 
+    pub fn signing_round_timeout_ms(mut self, signing_round_timeout: Option<String>) -> Self {
+        self.signing_round_timeout = signing_round_timeout;
+        self
+    }
+
     pub fn epoch_length(mut self, epoch_length: usize) -> Self {
         self.epoch_length = Some(U256::from(epoch_length));
         self
@@ -161,10 +168,18 @@ impl TestSetupBuilder {
         }
         fs::create_dir_all(node_keys_path).unwrap();
 
+        let signing_round_timeout_ms = if self.signing_round_timeout.is_some() {
+            self.signing_round_timeout
+        } else {
+            // if not in CI, set a default signing round timeout of 8000ms
+            Some("8000".to_string())
+        };
+
         let custom_node_runtime_config = CustomNodeRuntimeConfig::builder()
             .enable_payment(self.enable_payment)
             .payment_interval_ms(self.payment_interval_ms)
             .chain_polling_interval(self.chain_polling_interval)
+            .signing_round_timeout_ms(signing_round_timeout_ms)
             .build();
 
         let mut testnet = Testnet::builder()

rust/lit-node/lit-node-testnet/src/testnet/actions.rs

@@ -1269,4 +1269,75 @@ impl Actions {
         }
         Ok(())
     }
+
+    // shortcut function to update all complaint configs to the same interval and tolerance for testing
+    pub async fn update_all_complaint_configs(
+        &self,
+        interval_secs: Option<u64>,
+        tolerance: Option<u64>,
+        kick_penalty_percent: Option<u64>,
+        kick_penalty_demerits: Option<u64>,
+    ) -> Result<()> {
+        info!(
+            "Updating all complaint reason configs interval_secs to {:?} and tolerance to {:?}",
+            interval_secs, tolerance
+        );
+
+        let interval_secs = if interval_secs.is_some() {
+            Some(U256::from(interval_secs.unwrap()))
+        } else {
+            None
+        };
+        let tolerance = if tolerance.is_some() {
+            Some(U256::from(tolerance.unwrap()))
+        } else {
+            None
+        };
+        let kick_penalty_percent = if kick_penalty_percent.is_some() {
+            Some(U256::from(kick_penalty_percent.unwrap()))
+        } else {
+            None
+        };
+        let kick_penalty_demerits = if kick_penalty_demerits.is_some() {
+            Some(U256::from(kick_penalty_demerits.unwrap()))
+        } else {
+            None
+        };
+        for i in 0..=MAX_COMPLAINT_REASON_VALUE {
+            let reason = U256::from(i);
+            // First, get current chain config for this reason.
+            let current_config: lit_blockchain::contracts::staking::ComplaintConfig = self
+                .contracts
+                .staking
+                .complaint_config(reason)
+                .call()
+                .await
+                .map_err(|e| anyhow::anyhow!("unable to get complaint config: {:?}", e))?;
+
+            // Then, set the config with any new values.
+            let cc = self.contracts.staking.set_complaint_config(
+                reason,
+                lit_blockchain::contracts::staking::ComplaintConfig {
+                    tolerance: tolerance.unwrap_or(current_config.tolerance),
+                    interval_secs: interval_secs.unwrap_or(current_config.interval_secs),
+                    kick_penalty_percent: kick_penalty_percent
+                        .unwrap_or(current_config.kick_penalty_percent),
+                    kick_penalty_demerits: kick_penalty_demerits
+                        .unwrap_or(current_config.kick_penalty_demerits),
+                },
+            );
+            if !Contracts::process_contract_call(
+                cc,
+                format!("updating staking complaint config for reason {:?}", reason).as_str(),
+            )
+            .await
+            {
+                return Err(anyhow::anyhow!(
+                    "Error updating complaint config for reason {:?}",
+                    reason.as_u64()
+                ));
+            }
+        }
+        Ok(())
+    }
 }

rust/lit-node/lit-node-testnet/src/testnet/node_config.rs

@@ -9,11 +9,9 @@ use lit_node_common::config::config_names::{
     CFG_KEY_CHAIN_POLLING_INTERVAL_MS, CFG_KEY_CHATTER_CLIENT_TIMEOUT,
     CFG_KEY_ENABLE_EPOCH_TRANSITIONS, CFG_KEY_ENABLE_PAYMENT,
     CFG_KEY_ENABLE_PROXIED_CHATTER_CLIENT, CFG_KEY_PAYMENT_INTERVAL_MS,
+    CFG_KEY_SIGNING_ROUND_TIMEOUT,
 };
 
-#[cfg(all(feature = "proxy_chatter", feature = "testing"))]
-use lit_node_common::config::config_names::CFG_KEY_SIGNING_ROUND_TIMEOUT;
-
 use tracing::trace;
 
 pub const CUSTOM_NODE_RUNTIME_CONFIG_PATH: &str = "config/test/custom_node_runtime_config.toml";
@@ -36,6 +34,7 @@ pub struct CustomNodeRuntimeConfigBuilder {
     enable_payment: Option<String>,
     chain_polling_interval: Option<String>,
     payment_interval_ms: Option<String>,
+    signing_round_timeout_ms: Option<String>,
 }
 
 impl Default for CustomNodeRuntimeConfigBuilder {
@@ -50,6 +49,7 @@ impl CustomNodeRuntimeConfigBuilder {
             enable_payment: Some("true".to_string()),
             chain_polling_interval: None,
             payment_interval_ms: None,
+            signing_round_timeout_ms: None,
         }
     }
 
@@ -68,11 +68,17 @@ impl CustomNodeRuntimeConfigBuilder {
         self
     }
 
+    pub fn signing_round_timeout_ms(mut self, signing_round_timeout_ms: Option<String>) -> Self {
+        self.signing_round_timeout_ms = signing_round_timeout_ms;
+        self
+    }
+
     pub fn build(self) -> CustomNodeRuntimeConfig {
         CustomNodeRuntimeConfig {
             enable_payment: self.enable_payment,
             chain_polling_interval: self.chain_polling_interval,
             payment_interval_ms: self.payment_interval_ms,
+            signing_round_timeout_ms: self.signing_round_timeout_ms,
         }
     }
 }
@@ -82,6 +88,7 @@ pub struct CustomNodeRuntimeConfig {
     enable_payment: Option<String>,
     chain_polling_interval: Option<String>,
     payment_interval_ms: Option<String>,
+    signing_round_timeout_ms: Option<String>,
 }
 
 impl CustomNodeRuntimeConfig {
@@ -171,6 +178,13 @@ pub fn generate_custom_node_runtime_config(
             .unwrap_or("1000".into()),
     );
 
+    if let Some(signing_round_timeout_ms) = custom_config.signing_round_timeout_ms.clone() {
+        cfg.insertstr(
+            section,
+            CFG_KEY_SIGNING_ROUND_TIMEOUT,
+            &signing_round_timeout_ms,
+        );
+    }
     match node_config_path {
         Some(path) => {
             cfg.write_file(Path::new(&path))

rust/lit-node/lit-node/Cargo.toml

@@ -17,11 +17,12 @@ lit-actions-server = [
 testing = [
     "lit-actions-server",
     "lit-observability/testing",
+    "lit-blockchain/testing",
 ] # enables testing features
 proxy_chatter = [
     "lit-node-testnet/proxy_chatter",
-    "lit-node-testnet/testing",
     "lit-node-testnet/lit-actions",
+    "lit-blockchain/proxy_chatter",
 ] # enables proxy http for testing
 
 [build-dependencies]

rust/lit-node/lit-node/src/git_info.rs

@@ -1 +1 @@
-pub const GIT_COMMIT_HASH: &str = "b04446f9037be567924971aad3cb78a1f17b7414";
+pub const GIT_COMMIT_HASH: &str = "50083ccf8061c4bfe17761be8b496bdab82851a6";

rust/lit-node/lit-node/src/peers/peer_state/chain_update.rs

@@ -1,13 +1,7 @@
 use super::super::PeerState;
 use crate::error::unexpected_err;
-use crate::{
-    error::{EC, Result, blockchain_err, blockchain_err_code},
-    utils::eth::EthereumAddress,
-};
-use ethers::{
-    providers::Middleware,
-    types::{U64, U256},
-};
+use crate::error::{EC, Result, blockchain_err, blockchain_err_code};
+use ethers::types::{U64, U256};
 use lit_blockchain::util::decode_revert;
 use std::time::Duration;
 use tracing::{Instrument, debug_span, instrument, trace};
@@ -202,40 +196,6 @@ impl PeerState {
             return Err(unexpected_err("No realm id set", None));
         };
 
-        let provider = self.staking_contract.client().provider().clone();
-        let wallet_address = self
-            .wallet_keys
-            .verifying_key()
-            .to_eth_address()
-            .map_err(|e| {
-                blockchain_err(
-                    e,
-                    Some(
-                        "Failed to convert verifying key to eth address during request to join."
-                            .to_string(),
-                    ),
-                )
-            })?;
-        let balance = provider
-            .get_balance(wallet_address, None)
-            .await
-            .map_err(|e| {
-                blockchain_err(
-                    e,
-                    Some(
-                        "Failed to get balance of attested node wallet during request to join."
-                            .to_string(),
-                    ),
-                )
-            })?;
-
-        if balance.is_zero() {
-            return Err(blockchain_err(
-                "Aborting request to join as attested node wallet balance is 0.",
-                None,
-            ));
-        }
-
         let func = self
             .staking_contract
             .request_to_join_as_node(realm_id, self.staker_address);

rust/lit-node/lit-node/src/tasks/fsm/mod.rs

@@ -161,7 +161,7 @@ pub async fn node_fsm_worker(
                 }
 
                 // if the epoch seems to have jumped, we need to figure out why and handle it.
-                if epoch_number > previous_included_epoch_number {
+                if epoch_number >= previous_included_epoch_number {
                     // this could be the state if we haven't checked the chain - check it and continue.
                     if network_state == NetworkState::NextValidatorSetLocked {
                         wait_on_next_validator_set_locked(