fix: resolve cross-compilation GLIBC issues and create deployment script (#1675)

Author: sanity

Cross.toml

@@ -0,0 +1,6 @@
+[target.aarch64-unknown-linux-gnu]
+# Use a newer cross image with updated GLIBC
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge"
+
+[target.x86_64-unknown-linux-gnu]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:edge"

binaries.local/binaries-arm64-fdev.zip

@@ -1,6 +1,11 @@
 use std::process::Command;
 
 fn main() {
+    // Skip flatbuffers generation for cross-compilation
+    if std::env::var("CARGO_BUILD_TARGET").is_ok() {
+        return;
+    }
+
     let status = Command::new("flatc")
         .arg("--rust")
         .arg("-o")

binaries.local/binaries-arm64-fdev/fdev

@@ -0,0 +1,35 @@
+# Channel Overflow Analysis
+
+## Problem
+- Dropping 2251 packets in 10 seconds (225 packets/second)
+- Channels in `peer_connection.rs` have buffer size of only 1
+- Channels in `connection_handler.rs` use buffer size of 100
+
+## Root Cause
+In `peer_connection.rs` lines 261-262:
+```rust
+let (outbound_packets, outbound_packets_recv) = mpsc::channel(1);
+let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(1);
+```
+
+Buffer size of 1 means:
+- Only 1 packet can be queued
+- Any additional packets are dropped if receiver hasn't processed the previous one
+- With UDP's bursty nature, this causes massive packet loss
+
+## Evidence
+From connection_handler.rs:317-322:
+```rust
+match remote_conn.inbound_packet_sender.try_send(packet_data) {
+    Ok(_) => { /* success */ }
+    Err(mpsc::error::TrySendError::Full(_)) => {
+        // Channel full - this is happening 225 times/second!
+    }
+}
+```
+
+## Solution
+Increase channel buffer size from 1 to 100 to match other channels in the codebase.
+
+## Impact
+This should significantly reduce packet drops and improve connection stability.

binaries.local/binaries-arm64-freenet.zip

@@ -67,7 +67,20 @@ pub(crate) async fn create_connection_handler<S: Socket>(
     bandwidth_limit: Option<usize>,
 ) -> Result<(OutboundConnectionHandler, InboundConnectionHandler), TransportError> {
     // Bind the UDP socket to the specified port
-    let socket = S::bind((listen_host, listen_port).into()).await?;
+    let bind_addr: SocketAddr = (listen_host, listen_port).into();
+    tracing::info!(
+        target: "freenet_core::transport::send_debug",
+        %bind_addr,
+        is_gateway,
+        "Binding UDP socket"
+    );
+    let socket = S::bind(bind_addr).await?;
+    tracing::info!(
+        target: "freenet_core::transport::send_debug",
+        %bind_addr,
+        is_gateway,
+        "UDP socket bound successfully"
+    );
     let (och, new_connection_notifier) = OutboundConnectionHandler::config_listener(
         Arc::new(socket),
         keypair,

binaries.local/binaries-arm64-freenet/freenet

@@ -147,16 +147,37 @@ impl PeerConnection {
         let last_packet_id = remote_conn.last_packet_id.clone();
 
         let keep_alive_handle = tokio::spawn(async move {
+            tracing::info!(
+                target: "freenet_core::transport::keepalive_lifecycle",
+                remote = ?remote_addr,
+                "Keep-alive task STARTED for connection"
+            );
+
             let mut interval = tokio::time::interval(KEEP_ALIVE_INTERVAL);
             interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
 
             // Skip the first immediate tick
             interval.tick().await;
 
+            let mut tick_count = 0u64;
+            let task_start = std::time::Instant::now();
+
             loop {
+                let tick_start = std::time::Instant::now();
                 interval.tick().await;
+                tick_count += 1;
+
+                let elapsed_since_start = task_start.elapsed();
+                let elapsed_since_last_tick = tick_start.elapsed();
 
-                tracing::trace!(remote = ?remote_addr, "Keep-alive timer tick - sending NoOp");
+                tracing::info!(
+                    target: "freenet_core::transport::keepalive_lifecycle",
+                    remote = ?remote_addr,
+                    tick_count,
+                    elapsed_since_start_secs = elapsed_since_start.as_secs_f64(),
+                    tick_interval_ms = elapsed_since_last_tick.as_millis(),
+                    "Keep-alive tick - attempting to send NoOp"
+                );
 
                 // Create a NoOp packet
                 let packet_id = last_packet_id.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
@@ -174,17 +195,43 @@ impl PeerConnection {
                 };
 
                 // Send the keep-alive packet
-                if outbound_packets
-                    .send((remote_addr, noop_packet))
-                    .await
-                    .is_err()
-                {
-                    tracing::debug!(remote = ?remote_addr, "Keep-alive task stopping - channel closed");
-                    break;
+                tracing::info!(
+                    target: "freenet_core::transport::keepalive_lifecycle",
+                    remote = ?remote_addr,
+                    packet_id,
+                    "Sending keep-alive NoOp packet"
+                );
+
+                match outbound_packets.send((remote_addr, noop_packet)).await {
+                    Ok(_) => {
+                        tracing::info!(
+                            target: "freenet_core::transport::keepalive_lifecycle",
+                            remote = ?remote_addr,
+                            packet_id,
+                            "Keep-alive NoOp packet sent successfully"
+                        );
+                    }
+                    Err(e) => {
+                        tracing::warn!(
+                            target: "freenet_core::transport::keepalive_lifecycle",
+                            remote = ?remote_addr,
+                            error = ?e,
+                            elapsed_since_start_secs = task_start.elapsed().as_secs_f64(),
+                            total_ticks = tick_count,
+                            "Keep-alive task STOPPING - channel closed"
+                        );
+                        break;
+                    }
                 }
             }
 
-            tracing::debug!(remote = ?remote_addr, "Keep-alive task exiting");
+            tracing::warn!(
+                target: "freenet_core::transport::keepalive_lifecycle",
+                remote = ?remote_addr,
+                total_lifetime_secs = task_start.elapsed().as_secs_f64(),
+                total_ticks = tick_count,
+                "Keep-alive task EXITING"
+            );
         });
 
         tracing::info!(remote = ?remote_addr, "PeerConnection created with persistent keep-alive task");
@@ -211,8 +258,8 @@ impl PeerConnection {
     ) -> PeerConnectionMock {
         use crate::transport::crypto::TransportKeypair;
         use parking_lot::Mutex;
-        let (outbound_packets, outbound_packets_recv) = mpsc::channel(1);
-        let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(1);
+        let (outbound_packets, outbound_packets_recv) = mpsc::channel(100);
+        let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(100);
         let keypair = TransportKeypair::new();
         let remote = RemoteConnection {
             outbound_packets,
@@ -243,8 +290,8 @@ impl PeerConnection {
     ) -> RemoteConnectionMock {
         use crate::transport::crypto::TransportKeypair;
         use parking_lot::Mutex;
-        let (outbound_packets, outbound_packets_recv) = mpsc::channel(1);
-        let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(1);
+        let (outbound_packets, outbound_packets_recv) = mpsc::channel(100);
+        let (inbound_packet_sender, inbound_packet_recv) = mpsc::channel(100);
         let keypair = TransportKeypair::new();
         (
             RemoteConnection {
@@ -411,6 +458,28 @@ impl PeerConnection {
                         confirm_receipt,
                         payload,
                     } = msg;
+
+                    // Log keep-alive packets specifically
+                    if matches!(payload, SymmetricMessagePayload::NoOp) {
+                        if confirm_receipt.is_empty() {
+                            tracing::info!(
+                                target: "freenet_core::transport::keepalive_received",
+                                remote = ?self.remote_conn.remote_addr,
+                                packet_id,
+                                time_since_last_received_ms = last_received.elapsed().as_millis(),
+                                "Received NoOp keep-alive packet (no receipts)"
+                            );
+                        } else {
+                            tracing::debug!(
+                                target: "freenet_core::transport::keepalive_received",
+                                remote = ?self.remote_conn.remote_addr,
+                                packet_id,
+                                receipt_count = confirm_receipt.len(),
+                                "Received NoOp receipt packet"
+                            );
+                        }
+                    }
+
                     {
                         tracing::trace!(
                             remote = %self.remote_conn.remote_addr,
@@ -484,9 +553,43 @@ impl PeerConnection {
                     res.map_err(|e| TransportError::Other(e.into()))??
                 }
                 _ = timeout_check.tick() => {
-                    if last_received.elapsed() > KILL_CONNECTION_AFTER {
-                        tracing::warn!(remote = ?self.remote_conn.remote_addr, "connection timed out - no packets received for {:?}", last_received.elapsed());
+                    let elapsed = last_received.elapsed();
+                    if elapsed > KILL_CONNECTION_AFTER {
+                        tracing::warn!(
+                            target: "freenet_core::transport::keepalive_timeout",
+                            remote = ?self.remote_conn.remote_addr,
+                            elapsed_seconds = elapsed.as_secs_f64(),
+                            timeout_threshold_secs = KILL_CONNECTION_AFTER.as_secs(),
+                            "CONNECTION TIMEOUT - no packets received for {:.8}s",
+                            elapsed.as_secs_f64()
+                        );
+
+                        // Check if keep-alive task is still alive
+                        if let Some(ref handle) = self.keep_alive_handle {
+                            if !handle.is_finished() {
+                                tracing::error!(
+                                    target: "freenet_core::transport::keepalive_timeout",
+                                    remote = ?self.remote_conn.remote_addr,
+                                    "Keep-alive task is STILL RUNNING despite timeout!"
+                                );
+                            } else {
+                                tracing::error!(
+                                    target: "freenet_core::transport::keepalive_timeout",
+                                    remote = ?self.remote_conn.remote_addr,
+                                    "Keep-alive task has ALREADY FINISHED before timeout!"
+                                );
+                            }
+                        }
+
                         return Err(TransportError::ConnectionClosed(self.remote_addr()));
+                    } else {
+                        tracing::trace!(
+                            target: "freenet_core::transport::keepalive_health",
+                            remote = ?self.remote_conn.remote_addr,
+                            elapsed_seconds = elapsed.as_secs_f64(),
+                            remaining_seconds = (KILL_CONNECTION_AFTER - elapsed).as_secs_f64(),
+                            "Connection health check - still alive"
+                        );
                     }
                 }
                 _ = resend_check.take().unwrap_or(tokio::time::sleep(Duration::from_millis(10))) => {

binaries.local/binaries-x86_64-fdev.zip

@@ -39,15 +39,14 @@ impl<T: TimeSource> PacketRateLimiter<T> {
         bandwidth_limit: Option<usize>,
         socket: Arc<S>,
     ) {
-        tracing::info!(bandwidth_limit, "Rate limiter task started");
+        tracing::info!("Rate limiter task started");
         while let Some((socket_addr, packet)) = self.outbound_packets.recv().await {
             // tracing::trace!(%socket_addr, packet_len = %packet.len(), "Sending outbound packet");
             if let Some(bandwidth_limit) = bandwidth_limit {
                 self.rate_limiting(bandwidth_limit, &*socket, packet, socket_addr)
                     .await;
-            } else if let Err(error) = socket.send_to(&packet, socket_addr).await {
-                tracing::error!(%socket_addr, "Error sending packet: {}", error);
-                continue;
+            } else {
+                let _ = socket.send_to(&packet, socket_addr).await;
             }
         }
         tracing::debug!("Rate limiter task ended unexpectedly");
@@ -64,11 +63,62 @@ impl<T: TimeSource> PacketRateLimiter<T> {
         if let Some(wait_time) = self.can_send_packet(bandwidth_limit, packet.len()) {
             tokio::time::sleep(wait_time).await;
             tracing::debug!(%socket_addr, "Sending outbound packet after waiting {:?}", wait_time);
-            if let Err(error) = socket.send_to(&packet, socket_addr).await {
-                tracing::error!("Error sending packet: {}", error);
+
+            tracing::info!(
+                target: "freenet_core::transport::send_debug",
+                dest_addr = %socket_addr,
+                packet_len = packet.len(),
+                wait_time_ms = wait_time.as_millis(),
+                "Attempting to send packet (after rate limit wait)"
+            );
+
+            match socket.send_to(&packet, socket_addr).await {
+                Ok(bytes_sent) => {
+                    tracing::info!(
+                        target: "freenet_core::transport::send_debug",
+                        dest_addr = %socket_addr,
+                        bytes_sent,
+                        "Socket send_to completed (after wait)"
+                    );
+                }
+                Err(error) => {
+                    tracing::error!(
+                        target: "freenet_core::transport::send_debug",
+                        dest_addr = %socket_addr,
+                        error = %error,
+                        "Socket send_to failed (after wait)"
+                    );
+                }
+            }
+        } else {
+            tracing::info!(
+                target: "freenet_core::transport::send_debug",
+                dest_addr = %socket_addr,
+                packet_len = packet.len(),
+                first_bytes = ?&packet[..std::cmp::min(32, packet.len())],
+                "Attempting to send packet (with rate limit)"
+            );
+
+            match socket.send_to(&packet, socket_addr).await {
+                Ok(bytes_sent) => {
+                    tracing::info!(
+                        target: "freenet_core::transport::send_debug",
+                        dest_addr = %socket_addr,
+                        bytes_sent,
+                        expected_len = packet.len(),
+                        "Socket send_to completed (rate limited)"
+                    );
+                }
+                Err(error) => {
+                    tracing::error!(
+                        target: "freenet_core::transport::send_debug",
+                        dest_addr = %socket_addr,
+                        error = %error,
+                        error_kind = ?error.kind(),
+                        "Socket send_to failed (rate limited)"
+                    );
+                }
             }
-        } else if let Err(error) = socket.send_to(&packet, socket_addr).await {
-            tracing::debug!(%socket_addr, "Error sending packet: {:?}", error);
         }
         self.add_packet(packet.len());
     }

binaries.local/binaries-x86_64-fdev/fdev

@@ -0,0 +1,42 @@
+# Keep-alive Analysis Results
+
+## Key Finding: Keep-alive packets are being SENT but NOT RECEIVED
+
+### Evidence from logs:
+
+1. **Keep-alive task lifecycle (Hypothesis #1: REFUTED)**
+   - Keep-alive tasks START correctly when connections are established
+   - Keep-alive tasks are STILL RUNNING when timeout occurs
+   - ERROR log: "Keep-alive task is STILL RUNNING despite timeout!"
+   - Tasks do NOT exit prematurely
+
+2. **Keep-alive sending pattern (vega 136.62.52.28)**
+   - First connection (13:58:59 - 13:59:30): 
+     - Sent 3 keep-alives (packet IDs: 8, 9, 10) at 10s intervals
+     - Timeout after exactly 30 seconds
+   - Subsequent reconnections show same pattern
+
+3. **Keep-alive receiving pattern**
+   - ZERO keep-alive packets received from vega
+   - This explains the 30-second timeout
+
+## Timeline for first vega connection:
+- 13:58:59.243: Keep-alive task STARTED
+- 13:59:09.244: Keep-alive sent (packet 8) - 10s after start
+- 13:59:19.244: Keep-alive sent (packet 9) - 20s after start  
+- 13:59:29.244: Keep-alive sent (packet 10) - 30s after start
+- 13:59:30.131: CONNECTION TIMEOUT at 30.001s
+- 13:59:30.131: ERROR: Keep-alive task is STILL RUNNING
+
+## Confirmed Hypothesis:
+**Hypothesis #2: Keep-alive packets are sent but not received**
+- Status: CONFIRMED
+- We are sending keep-alives every 10 seconds
+- We are NOT receiving any keep-alives back
+- The connection times out after exactly 30 seconds
+
+## Next Investigation:
+- Why are keep-alive packets not being received?
+- Are they being sent on the wire?
+- Are they being dropped/filtered?
+- Is vega gateway not sending them?