Chain fault testing framework + Enable auto-rejoin

rust/lit-core/lit-blockchain/Cargo.toml

@@ -8,6 +8,8 @@ edition.workspace = true
 [features]
 default = []
 env-override = []
+testing = []
+proxy_chatter = []
 
 [dependencies]
 alloy.workspace = true

rust/lit-core/lit-blockchain/src/contracts/mod.rs

@@ -145,6 +145,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -207,6 +209,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -318,6 +322,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -363,6 +369,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -428,6 +436,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -525,6 +535,8 @@ impl
     > {
         let chain = cfg.blockchain_chain_name()?;
         let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+        #[cfg(feature = "proxy_chatter")]
+        let provider = change_port(provider, cfg)?;
         let meta_signer = meta_signer_key.into();
         let meta_wallet = LocalWallet::from(meta_signer.clone());
         let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -571,13 +583,41 @@ pub fn default_local_client(
     let chain = cfg.blockchain_chain_name()?;
     let wallet = load_wallet(cfg, wallet_key)?;
     let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+    #[cfg(feature = "proxy_chatter")]
+    let provider = change_port(provider, cfg)?;
 
     Ok(Arc::new(SignerMiddleware::new(provider, wallet)))
 }
 
 pub fn default_local_client_no_wallet(cfg: &LitConfig) -> Result<Arc<Provider<Http>>> {
     let chain = cfg.blockchain_chain_name()?;
     let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
+    #[cfg(feature = "proxy_chatter")]
+    let provider = change_port(provider, cfg)?;
 
     Ok(provider)
 }
+
+#[cfg(feature = "proxy_chatter")]
+pub fn change_port(provider: Arc<Provider<Http>>, cfg: &LitConfig) -> Result<Arc<Provider<Http>>> {
+    let chain = cfg.blockchain_chain_name()?;
+    if chain.to_lowercase() == "localchain" {
+        let cfg2 = cfg.config();
+        let port = cfg2
+            .get_int("node.http.port")
+            .map_err(|e| crate::error::unexpected_err(e.to_string(), None))?;
+        let port = port as u16;
+        let port = 11075 + port; // 10000 + 8545 ( anvil default port ) - 7470 ( lit-node default starting port ) + actual port value
+        tracing::trace!(
+            "Changing port for proxy provider {} to {}.",
+            provider.url().as_str(),
+            port
+        );
+        let mut proxy_provider = (*provider).clone();
+        proxy_provider.url_mut().set_port(Some(port)).map_err(|_e| {
+            crate::error::unexpected_err("Could not set port for proxy provider", None)
+        })?;
+        return Ok(Arc::new(proxy_provider));
+    }
+    Ok(provider)
+}

rust/lit-node/lit-node-testnet/Cargo.toml

@@ -8,7 +8,6 @@ edition.workspace = true
 # this needs to happen at runtime, so is just a place holder for now.
 sign_test = ["lit-actions"] #enables unsafe endpoint allowing a test to use a PK directly .
 lit-actions = []
-proxy-chatter = []
 proxy_chatter = []
 lit-actions-server = ["lit-actions"] # start internal lit_actions server for testing
 testing = [] # enables testing features

rust/lit-node/lit-node-testnet/src/lib.rs

@@ -29,6 +29,7 @@ pub struct TestSetupBuilder {
     register_inactive_validators: bool,
     enable_payment: Option<String>,
     chain_polling_interval: Option<String>,
+    signing_round_timeout: Option<String>,
     epoch_length: Option<U256>,
     max_presign_count: Option<u64>,
     min_presign_count: Option<u64>,
@@ -51,6 +52,7 @@ impl Default for TestSetupBuilder {
             register_inactive_validators: false,
             enable_payment: Some("true".to_string()),
             chain_polling_interval: None,
+            signing_round_timeout: None,
             epoch_length: None,
             max_presign_count: Some(0),
             min_presign_count: Some(0),
@@ -114,6 +116,11 @@ impl TestSetupBuilder {
         self
     }
 
+    pub fn signing_round_timeout_ms(mut self, signing_round_timeout: Option<String>) -> Self {
+        self.signing_round_timeout = signing_round_timeout;
+        self
+    }
+
     pub fn epoch_length(mut self, epoch_length: usize) -> Self {
         self.epoch_length = Some(U256::from(epoch_length));
         self
@@ -161,10 +168,18 @@ impl TestSetupBuilder {
         }
         fs::create_dir_all(node_keys_path).unwrap();
 
+        let signing_round_timeout_ms = if self.signing_round_timeout.is_some() {
+            self.signing_round_timeout
+        } else {
+            // if not in CI, set a default signing round timeout of 8000ms
+            Some("8000".to_string())
+        };
+
         let custom_node_runtime_config = CustomNodeRuntimeConfig::builder()
             .enable_payment(self.enable_payment)
             .payment_interval_ms(self.payment_interval_ms)
             .chain_polling_interval(self.chain_polling_interval)
+            .signing_round_timeout_ms(signing_round_timeout_ms)
             .build();
 
         let mut testnet = Testnet::builder()

rust/lit-node/lit-node-testnet/src/testnet/actions.rs

@@ -1269,4 +1269,75 @@ impl Actions {
         }
         Ok(())
     }
+
+    // shortcut function to update all complaint configs to the same interval and tolerance for testing
+    pub async fn update_all_complaint_configss(
+        &self,
+        interval_secs: Option<u64>,
+        tolerance: Option<u64>,
+        kick_penalty_percent: Option<u64>,
+        kick_penalty_demerits: Option<u64>,
+    ) -> Result<()> {
+        info!(
+            "Updating all complaint reason configs interval_secs to {:?} and tolerance to {:?}",
+            interval_secs, tolerance
+        );
+
+        let interval_secs = if interval_secs.is_some() {
+            Some(U256::from(interval_secs.unwrap()))
+        } else {
+            None
+        };
+        let tolerance = if tolerance.is_some() {
+            Some(U256::from(tolerance.unwrap()))
+        } else {
+            None
+        };
+        let kick_penalty_percent = if kick_penalty_percent.is_some() {
+            Some(U256::from(kick_penalty_percent.unwrap()))
+        } else {
+            None
+        };
+        let kick_penalty_demerits = if kick_penalty_demerits.is_some() {
+            Some(U256::from(kick_penalty_demerits.unwrap()))
+        } else {
+            None
+        };
+        for i in 0..=MAX_COMPLAINT_REASON_VALUE {
+            let reason = U256::from(i);
+            // First, get current chain config for this reason.
+            let current_config: lit_blockchain::contracts::staking::ComplaintConfig = self
+                .contracts
+                .staking
+                .complaint_config(reason)
+                .call()
+                .await
+                .map_err(|e| anyhow::anyhow!("unable to get complaint config: {:?}", e))?;
+
+            // Then, set the config with any new values.
+            let cc = self.contracts.staking.set_complaint_config(
+                reason,
+                lit_blockchain::contracts::staking::ComplaintConfig {
+                    tolerance: tolerance.unwrap_or(current_config.tolerance),
+                    interval_secs: interval_secs.unwrap_or(current_config.interval_secs),
+                    kick_penalty_percent: kick_penalty_percent
+                        .unwrap_or(current_config.kick_penalty_percent),
+                    kick_penalty_demerits: kick_penalty_demerits
+                        .unwrap_or(current_config.kick_penalty_demerits),
+                },
+            );
+            if !Contracts::process_contract_call(
+                cc,
+                format!("updating staking complaint config for reason {:?}", reason).as_str(),
+            )
+            .await
+            {
+                return Err(anyhow::anyhow!(
+                    "Error updating complaint config for reason {:?}",
+                    reason.as_u64()
+                ));
+            }
+        }
+        Ok(())
+    }
 }

rust/lit-node/lit-node-testnet/src/testnet/node_config.rs

@@ -36,6 +36,7 @@ pub struct CustomNodeRuntimeConfigBuilder {
     enable_payment: Option<String>,
     chain_polling_interval: Option<String>,
     payment_interval_ms: Option<String>,
+    signing_round_timeout_ms: Option<String>,
 }
 
 impl Default for CustomNodeRuntimeConfigBuilder {
@@ -50,6 +51,7 @@ impl CustomNodeRuntimeConfigBuilder {
             enable_payment: Some("true".to_string()),
             chain_polling_interval: None,
             payment_interval_ms: None,
+            signing_round_timeout_ms: None,
         }
     }
 
@@ -68,11 +70,17 @@ impl CustomNodeRuntimeConfigBuilder {
         self
     }
 
+    pub fn signing_round_timeout_ms(mut self, signing_round_timeout_ms: Option<String>) -> Self {
+        self.signing_round_timeout_ms = signing_round_timeout_ms;
+        self
+    }
+
     pub fn build(self) -> CustomNodeRuntimeConfig {
         CustomNodeRuntimeConfig {
             enable_payment: self.enable_payment,
             chain_polling_interval: self.chain_polling_interval,
             payment_interval_ms: self.payment_interval_ms,
+            signing_round_timeout_ms: self.signing_round_timeout_ms,
         }
     }
 }
@@ -82,6 +90,7 @@ pub struct CustomNodeRuntimeConfig {
     enable_payment: Option<String>,
     chain_polling_interval: Option<String>,
     payment_interval_ms: Option<String>,
+    signing_round_timeout_ms: Option<String>,
 }
 
 impl CustomNodeRuntimeConfig {
@@ -171,6 +180,13 @@ pub fn generate_custom_node_runtime_config(
             .unwrap_or("1000".into()),
     );
 
+    if let Some(signing_round_timeout_ms) = custom_config.signing_round_timeout_ms.clone() {
+        cfg.insertstr(
+            section,
+            CFG_KEY_SIGNING_ROUND_TIMEOUT,
+            &signing_round_timeout_ms,
+        );
+    }
     match node_config_path {
         Some(path) => {
             cfg.write_file(Path::new(&path))

rust/lit-node/lit-node/Cargo.toml

@@ -17,11 +17,12 @@ lit-actions-server = [
 testing = [
     "lit-actions-server",
     "lit-observability/testing",
+    "lit-blockchain/testing",
 ] # enables testing features
 proxy_chatter = [
     "lit-node-testnet/proxy_chatter",
-    "lit-node-testnet/testing",
     "lit-node-testnet/lit-actions",
+    "lit-blockchain/proxy_chatter",
 ] # enables proxy http for testing
 
 [build-dependencies]

rust/lit-node/lit-node/src/git_info.rs

@@ -1 +1 @@
-pub const GIT_COMMIT_HASH: &str = "54d2b195ed373e9067276959ef9252fecf054b20";
+pub const GIT_COMMIT_HASH: &str = "26605433eeb1eda1c5a7bd750237fc054855438e";