fix: connection stability improvements for production issues

.gitignore

@@ -1,5 +1,8 @@
 aider_stdlib_map.md
 
+# Local tools and notes
+.local-tools/
+
 ### Rust ###
 # Generated by Cargo
 # will have compiled files and executables

apps/freenet-ping/test-prod-logs/config/gateways.toml

@@ -0,0 +1,11 @@
+[[gateways]]
+public_key = "/home/ian/code/freenet/freenet-core/main/apps/freenet-ping/test-prod-logs/data/secrets/public.vega.gw.pem"
+
+[gateways.address]
+hostname = "vega.locut.us:31337"
+
+[[gateways]]
+public_key = "/home/ian/code/freenet/freenet-core/main/apps/freenet-ping/test-prod-logs/data/secrets/public.ziggy.gw.pem"
+
+[gateways.address]
+hostname = "technic.locut.us:31337"

apps/freenet-ping/test-prod-logs/data/secrets/public.vega.gw.pem

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3ZECtEbAcxfvnpweDHiV
+OtHvHtyYE5omuNWn3AHyNHPyFMRWd/LAF4l7Q0WunjL4uAXUws1U2OBroW+Rw91C
+753kUtecaCelU2f4dLOfz7hIcbFL+A9w4Qzwc3bLUqTKmfASmqVX9qXP1SE4ZAKQ
+VxgS/9dVNBEo6wvLkrCNvq2LVQvWslEJb+0QKKbUPrFmYVnweiNGCmP6dqL37u5d
+C1jsuE72TaJ7FSj+hWFja3gl6o+Wz/Bw+9qbByUOchD7LtQ//9zl4Xb9lHjyUJ4V
++Tmhg2EiKy3EFEUWILB391cLR3YSzS6B4kw/PptZ/L/TSWUsZH33QiGnAw56jNBW
+cFw3TIlQu3PvZWcttyD2dDO6tqZDusI2YHh8hyAb9Z5N8px1jlfHPpK6qHEqhB9N
+XjrkD2lTuRbB55UurZ5L2ajvysIjqOOtYDPcLRZHk8YjvxrYmMIk17GKE/6YI01h
++QcqcYxcBCBv0ME9WL3ohKvWTziHVNfrcWYebISHhpc4F/CHbaj2O4bs4I9a9aXV
+INlmFHVprCl7UYI7lb4BHWu0tYxsy3oZeLG7ARWffRsz3L2aRHbtqgthSu6X7yYL
+YB/qWjUmIE0cAszAULCjbynplZNtwJT324+77DPD85+N9nTrS+Po8T2xJ8pCweu+
+5LUJQ0Jh+QPJeIQ2TzA7RSUCAwEAAQ==
+-----END PUBLIC KEY-----

apps/freenet-ping/test-prod-logs/data/secrets/public.ziggy.gw.pem

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArpQ/xgxqczEpsNaKtBdh
+MmhiocOjfNimI6rD5cTYfMAsahKS0ElY6leXzNBgsZYLGxYPKqkrODPxpttejD9v
+fdcOaN1QNW+Rl85BSf5DlBxeBdC2NTMeadC207RZOdD12pdEdIz1W1nS/PGlDqRq
+Kh2LPSmuT9Kqxzk1PMO7HuIgMvQVpLBJiFnQMWJ4+meHINtI8vOkbYjLqoE2yrSK
+GSapQ/zptsjzzB4Jd0Rm/OV5qT7SF14yhv+XVhPy2gw/lJmhcIDw2rCEBe7hPbDX
+E+p9jCgGMi0HWN8QeSDTapY9hlFy4dzXW5xK0ySrGUAp1kcPDdwqA2npa3oJlVDZ
+TstAMDt3RhdfrrwfmdQefvygBUIAlAk69y1FvNGEOO9TcikQjDaJdfQSP9V0qTEV
+BsKNpkX7VHgG1i8Cc6c7JlDyeHSB+3TXSfO9esYcAFxjwx1SbyInHBMpYG/iYoX+
+jGqQLJVMV6EasQ5xgryW2jTD4pBjkcrlgz0KKUlCg0cMTafaOpTrnaTHjmWYIP2c
+KKxVR2Iu+cPdNPvFeywr2mF2Bz7I4cRxJOVhwupDH7XaWf8gfMMvIlsr2f2ChQzc
+gRCVduKEeMp1bcuKisSbJnFIluVOgJ3MZJi7RS+0W31dkePWtJYP0eXgdlHtDVV6
+scCTSgvaL0QFtu7+5ds0qqkCAwEAAQ==
+-----END PUBLIC KEY-----

crates/core/src/transport/connection_handler.rs

@@ -646,6 +646,7 @@ impl<S: Socket> UdpPacketsListener<S> {
                 inbound_symmetric_key: inbound_key,
                 inbound_symmetric_key_bytes: inbound_key_bytes,
                 my_address: None,
+                transport_secret_key: secret,
             };
 
             let inbound_conn = InboundRemoteConnection {
@@ -842,6 +843,8 @@ impl<S: Socket> UdpPacketsListener<S> {
                                                     inbound_symmetric_key_bytes:
                                                         inbound_sym_key_bytes,
                                                     my_address: Some(my_address),
+                                                    transport_secret_key: transport_secret_key
+                                                        .clone(),
                                                 },
                                                 InboundRemoteConnection {
                                                     inbound_packet_sender: inbound_sender,
@@ -907,6 +910,7 @@ impl<S: Socket> UdpPacketsListener<S> {
                                         inbound_symmetric_key: inbound_sym_key,
                                         inbound_symmetric_key_bytes: inbound_sym_key_bytes,
                                         my_address: None,
+                                        transport_secret_key: transport_secret_key.clone(),
                                     },
                                     InboundRemoteConnection {
                                         inbound_packet_sender: inbound_sender,

crates/core/src/transport/peer_connection.rs

@@ -5,6 +5,7 @@ use std::time::Duration;
 use std::{collections::HashMap, time::Instant};
 
 use crate::transport::connection_handler::NAT_TRAVERSAL_MAX_ATTEMPTS;
+use crate::transport::crypto::TransportSecretKey;
 use crate::transport::packet_data::UnknownEncryption;
 use crate::transport::sent_packet_tracker::MESSAGE_CONFIRMATION_TIMEOUT;
 use aes_gcm::Aes128Gcm;
@@ -47,6 +48,7 @@ pub(crate) struct RemoteConnection {
     pub(super) inbound_symmetric_key: Aes128Gcm,
     pub(super) inbound_symmetric_key_bytes: [u8; 16],
     pub(super) my_address: Option<SocketAddr>,
+    pub(super) transport_secret_key: TransportSecretKey,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -144,9 +146,11 @@ impl PeerConnection {
         outbound_symmetric_key: Aes128Gcm,
         inbound_symmetric_key: Aes128Gcm,
     ) -> PeerConnectionMock {
+        use crate::transport::crypto::TransportKeypair;
         use parking_lot::Mutex;
         let (outbound_packets, outbound_packets_recv) = mpsc::channel(1);
         let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(1);
+        let keypair = TransportKeypair::new();
         let remote = RemoteConnection {
             outbound_packets,
             outbound_symmetric_key,
@@ -157,6 +161,7 @@ impl PeerConnection {
             inbound_symmetric_key,
             inbound_symmetric_key_bytes: [1; 16],
             my_address: Some(my_address),
+            transport_secret_key: keypair.secret,
         };
         (
             Self::new(remote),
@@ -172,9 +177,11 @@ impl PeerConnection {
         outbound_symmetric_key: Aes128Gcm,
         inbound_symmetric_key: Aes128Gcm,
     ) -> RemoteConnectionMock {
+        use crate::transport::crypto::TransportKeypair;
         use parking_lot::Mutex;
         let (outbound_packets, outbound_packets_recv) = mpsc::channel(1);
         let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(1);
+        let keypair = TransportKeypair::new();
         (
             RemoteConnection {
                 outbound_packets,
@@ -186,6 +193,7 @@ impl PeerConnection {
                 inbound_symmetric_key,
                 inbound_symmetric_key_bytes: [1; 16],
                 my_address: Some(my_address),
+                transport_secret_key: keypair.secret,
             },
             inbound_packet_sender,
             outbound_packets_recv,
@@ -215,14 +223,8 @@ impl PeerConnection {
         // listen for incoming messages or receipts or wait until is time to do anything else again
         let mut resend_check = Some(tokio::time::sleep(tokio::time::Duration::from_millis(10)));
 
-        #[cfg(debug_assertions)]
-        const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(2);
-        #[cfg(not(debug_assertions))]
-        const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(20);
-        #[cfg(debug_assertions)]
-        const KILL_CONNECTION_AFTER: Duration = Duration::from_secs(6);
-        #[cfg(not(debug_assertions))]
-        const KILL_CONNECTION_AFTER: Duration = Duration::from_secs(60);
+        const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(10);
+        const KILL_CONNECTION_AFTER: Duration = Duration::from_secs(30);
 
         let mut keep_alive = tokio::time::interval(KEEP_ALIVE_INTERVAL);
         keep_alive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
@@ -236,14 +238,85 @@ impl PeerConnection {
                 inbound = self.remote_conn.inbound_packet_recv.recv() => {
                     let packet_data = inbound.ok_or(TransportError::ConnectionClosed(self.remote_addr()))?;
                     last_received = std::time::Instant::now();
+
+                    // Debug logging for 256-byte packets
+                    if packet_data.data().len() == 256 {
+                        tracing::warn!(
+                            remote = ?self.remote_conn.remote_addr,
+                            packet_bytes = ?&packet_data.data()[..32], // First 32 bytes
+                            packet_len = packet_data.data().len(),
+                            "Received 256-byte packet"
+                        );
+                    }
+
                     let Ok(decrypted) = packet_data.try_decrypt_sym(&self.remote_conn.inbound_symmetric_key).inspect_err(|error| {
                         tracing::warn!(
                             %error,
                             remote = ?self.remote_conn.remote_addr,
                             inbound_key = ?self.remote_conn.inbound_symmetric_key_bytes,
+                            packet_len = packet_data.data().len(),
+                            packet_first_bytes = ?&packet_data.data()[..std::cmp::min(32, packet_data.data().len())],
                             "Failed to decrypt packet, might be an intro packet or a partial packet"
                         );
                     }) else {
+                        // Check if this is a 256-byte RSA intro packet
+                        if packet_data.data().len() == 256 {
+                            tracing::info!(
+                                remote = ?self.remote_conn.remote_addr,
+                                "Attempting to decrypt potential RSA intro packet"
+                            );
+
+                            // Try to decrypt as RSA intro packet
+                            match self.remote_conn.transport_secret_key.decrypt(packet_data.data()) {
+                                Ok(_decrypted_intro) => {
+                                    tracing::info!(
+                                        remote = ?self.remote_conn.remote_addr,
+                                        "Successfully decrypted RSA intro packet, sending ACK"
+                                    );
+
+                                    // Send ACK response for intro packet
+                                    let ack_packet = SymmetricMessage::ack_ok(
+                                        &self.remote_conn.outbound_symmetric_key,
+                                        self.remote_conn.inbound_symmetric_key_bytes,
+                                        self.remote_conn.remote_addr,
+                                    );
+
+                                    if let Ok(ack) = ack_packet {
+                                        if let Err(send_err) = self.remote_conn
+                                            .outbound_packets
+                                            .send((self.remote_conn.remote_addr, ack.data().into()))
+                                            .await
+                                        {
+                                            tracing::warn!(
+                                                remote = ?self.remote_conn.remote_addr,
+                                                error = ?send_err,
+                                                "Failed to send ACK for intro packet"
+                                            );
+                                        } else {
+                                            tracing::info!(
+                                                remote = ?self.remote_conn.remote_addr,
+                                                "Successfully sent ACK for intro packet"
+                                            );
+                                        }
+                                    } else {
+                                        tracing::warn!(
+                                            remote = ?self.remote_conn.remote_addr,
+                                            "Failed to create ACK packet for intro"
+                                        );
+                                    }
+
+                                    // Continue to next packet
+                                    continue;
+                                }
+                                Err(rsa_err) => {
+                                    tracing::debug!(
+                                        remote = ?self.remote_conn.remote_addr,
+                                        error = ?rsa_err,
+                                        "256-byte packet is not a valid RSA intro packet"
+                                    );
+                                }
+                            }
+                        }
                         let now = Instant::now();
                         if let Some(first_failure_time) = self.first_failure_time {
                             if now.duration_since(first_failure_time) <= FAILURE_TIME_WINDOW {