Improve connection robustness

 - Attempt reconnections from webrtc task when disconnections happen
   - Reconnections are handled with new task in LwsApiCalls.c
   - Workflow was broken with LwsApiCallsESP.c: We now handle it from webrtc_task itself
 - Do not reboot the device when using wifi-set CLI command

Author: vikramdattu

esp_port/components/esp_webrtc_utils/src/wifi_cli.c

@@ -14,13 +14,19 @@ static const char *TAG = "wifi_cli";
 
 static int wifi_set_cli_handler(int argc, char *argv[])
 {
+
     /* Just to go to the next line */
     printf("\n");
     if (argc != 3) {
         printf("%s: Incorrect arguments\n", TAG);
         return 0;
     }
 
+    /* Stop the Wi-Fi */
+    if (esp_wifi_stop() != ESP_OK) {
+        printf("%s: Failed to stop wifi\n", TAG);
+    }
+
     wifi_config_t wifi_cfg = {
         .sta = {
             /* Setting a password implies station will connect to all security modes including WEP/WPA.
@@ -44,9 +50,16 @@ static int wifi_set_cli_handler(int argc, char *argv[])
         return 0;
     }
 
-    ESP_LOGI(TAG, "Rebooting with new wi-fi configuration...");
-    vTaskDelay(pdMS_TO_TICKS(2 * 1000));
-    esp_restart(); // for now we simply re-start the device
+    /* (Re)Start Wi-Fi */
+    if (esp_wifi_start() != ESP_OK) {
+        printf("%s: Failed to start wifi\n", TAG);
+    }
+
+    /* Connect to AP */
+    if (esp_wifi_connect() != ESP_OK) {
+        printf("%s: Failed to connect wifi\n", TAG);
+    }
+
     return 0;
 }
 

esp_port/components/kvs_webrtc/src/LwsApiCallsESP.c

@@ -323,20 +323,58 @@ static void esp_websocket_event_handler(void *handler_args, esp_event_base_t bas
 
         case WEBSOCKET_EVENT_DISCONNECTED:
             ESP_LOGI(TAG, "WEBSOCKET_EVENT_DISCONNECTED");
+
+            BOOL connected = FALSE;
+            STATUS retStatus = STATUS_SUCCESS;
+
             MUTEX_LOCK(pEspWrapper->wsClientLock);
+            connected = pEspWrapper->isConnected;
             pEspWrapper->isConnected = FALSE;
             MUTEX_UNLOCK(pEspWrapper->wsClientLock);
 
+            // Now DISCONNECT is the primary event for handling connection cleanup and reconnection
+            BOOL wasConnected = connected;
             ATOMIC_STORE_BOOL(&pSignalingClient->connected, FALSE);
-            if ((SERVICE_CALL_RESULT) ATOMIC_LOAD(&pSignalingClient->result) == SERVICE_CALL_RESULT_NOT_SET) {
-                ATOMIC_STORE(&pSignalingClient->result, (SIZE_T) SERVICE_CALL_UNKNOWN);
-            }
 
-            // Signal the condition variable to wake up the waiting thread
+            // Signal condition variables to wake up waiting threads
             MUTEX_LOCK(pSignalingClient->connectedLock);
             CVAR_SIGNAL(pSignalingClient->connectedCvar);
             MUTEX_UNLOCK(pSignalingClient->connectedLock);
 
+            CVAR_BROADCAST(pSignalingClient->receiveCvar);
+            CVAR_BROADCAST(pSignalingClient->sendCvar);
+            ATOMIC_STORE(&pSignalingClient->messageResult, (SIZE_T) SERVICE_CALL_RESULT_OK);
+
+            // Handle reconnection logic
+            if (wasConnected && ATOMIC_LOAD(&pSignalingClient->result) != SERVICE_CALL_RESULT_SIGNALING_RECONNECT_ICE &&
+                !ATOMIC_LOAD_BOOL(&pSignalingClient->shutdown)) {
+
+                ESP_LOGI(TAG, "WebSocket disconnected from active connection, triggering reconnection via error callback");
+
+                // Set the result failed to indicate disconnection
+                ATOMIC_STORE(&pSignalingClient->result, (SIZE_T) SERVICE_CALL_UNKNOWN);
+
+                // Use the existing error callback mechanism to trigger reconnection
+                if (pSignalingClient->signalingClientCallbacks.errorReportFn != NULL) {
+                    CHAR errorMsg[] = "WebSocket connection lost, triggering reconnection";
+                    ESP_LOGI(TAG, "DEBUG: Calling errorReportFn with STATUS_SIGNALING_RECONNECT_FAILED=0x%08x", STATUS_SIGNALING_RECONNECT_FAILED);
+                    pSignalingClient->signalingClientCallbacks.errorReportFn(
+                        pSignalingClient->signalingClientCallbacks.customData,
+                        STATUS_SIGNALING_RECONNECT_FAILED,
+                        errorMsg,
+                        (UINT32) STRLEN(errorMsg));
+                    ESP_LOGI(TAG, "Signaled for reconnection via error callback");
+                } else {
+                    ESP_LOGW(TAG, "No error callback registered, cannot signal for reconnection");
+                }
+            } else if (!wasConnected) {
+                ESP_LOGI(TAG, "WebSocket disconnected but was not previously connected, not triggering reconnection");
+            } else if (ATOMIC_LOAD_BOOL(&pSignalingClient->shutdown)) {
+                ESP_LOGI(TAG, "WebSocket disconnected during shutdown, not triggering reconnection");
+            } else {
+                ESP_LOGI(TAG, "WebSocket disconnected during ICE reconnect, not triggering full reconnection");
+            }
+
             // Free the WebSocket buffer on disconnect
             freeWebSocketBuffer();
             break;
@@ -400,20 +438,19 @@ static void esp_websocket_event_handler(void *handler_args, esp_event_base_t bas
                 }
             }
 
+            // For ERROR events, just log and wake up waiting threads
+            // Let DISCONNECT event handle all state changes and reconnection logic
             if (pSignalingClient != NULL) {
-                ATOMIC_STORE_BOOL(&pSignalingClient->connected, FALSE);
-                // Set result if not already set
-                if ((SERVICE_CALL_RESULT) ATOMIC_LOAD(&pSignalingClient->result) == SERVICE_CALL_RESULT_NOT_SET) {
-                    ATOMIC_STORE(&pSignalingClient->result, (SIZE_T) SERVICE_CALL_UNKNOWN);
-
-                    // Signal the waiting thread about the error
-                    // The mutex must be locked before signaling
-                    MUTEX_LOCK(pSignalingClient->connectedLock);
-                    CVAR_SIGNAL(pSignalingClient->connectedCvar);
-                    MUTEX_UNLOCK(pSignalingClient->connectedLock);
-                }
-                // Reset the buffer on error
-                resetWebSocketBuffer();
+                ESP_LOGI(TAG, "WebSocket error occurred, waiting for disconnect event to handle cleanup and reconnection");
+
+                // Signal condition variables to wake up waiting threads
+                MUTEX_LOCK(pSignalingClient->connectedLock);
+                CVAR_SIGNAL(pSignalingClient->connectedCvar);
+                MUTEX_UNLOCK(pSignalingClient->connectedLock);
+
+                CVAR_BROADCAST(pSignalingClient->receiveCvar);
+                CVAR_BROADCAST(pSignalingClient->sendCvar);
+                ATOMIC_STORE(&pSignalingClient->messageResult, (SIZE_T) SERVICE_CALL_RESULT_OK);
             } else {
                 ESP_LOGW(TAG, "pSignalingClient is NULL in WEBSOCKET_EVENT_ERROR!");
             }
@@ -920,18 +957,16 @@ STATUS connectEspSignalingClient(PSignalingClient pSignalingClient)
     // Set ping interval to match SIGNALING_SERVICE_WSS_PING_PONG_INTERVAL_IN_SECONDS (10 seconds)
     ws_cfg.ping_interval_sec = 10;
 
-    // Enable auto reconnect to handle network issues gracefully
-    ws_cfg.disable_auto_reconnect = FALSE;
+    // Disable auto reconnect to let the state machine handle reconnection with fresh credentials
+    // Auto-reconnect would fail after 5 minutes when the signed URL expires
+    ws_cfg.disable_auto_reconnect = TRUE;
 
     // Set reconnect timeout to match ESP-IDF recommendations
     ws_cfg.reconnect_timeout_ms = 10000; // 10 seconds
 
     // Configure TCP keepalive to match SIGNALING_SERVICE_TCP_KEEPALIVE settings
     ws_cfg.network_timeout_ms = SIGNALING_SERVICE_TCP_KEEPALIVE_IN_SECONDS * 1000;
 
-    // Configure transport - since we're using wss:// in the URL, this will automatically be SSL
-    // ws_cfg.transport = WEBSOCKET_TRANSPORT_OVER_SSL;
-
     // For development, set skip_cert_common_name_check to true to skip cert verification
     // In production environments, this should be set to false
 #ifdef CONFIG_SKIP_COMMON_NAME_CHECK

esp_port/components/kvs_webrtc/src/app_webrtc.c

@@ -478,10 +478,17 @@ STATUS signalingClientError(UINT64 customData, STATUS status, PCHAR msg, UINT32
     SNPRINTF(errorMsg, SIZEOF(errorMsg), "Signaling error: 0x%08" PRIx32 " - %.*s", status, (int) msgLen, msg);
     raiseEvent(APP_WEBRTC_EVENT_SIGNALING_ERROR, status, NULL, errorMsg);
 
+    // DEBUG: Show status comparison
+    DLOGI("DEBUG: signalingClientError received status=0x%08x, STATUS_SIGNALING_RECONNECT_FAILED=0x%08x, STATUS_SIGNALING_ICE_CONFIG_REFRESH_FAILED=0x%08x",
+          status, STATUS_SIGNALING_RECONNECT_FAILED, STATUS_SIGNALING_ICE_CONFIG_REFRESH_FAILED);
+
     // We will force re-create the signaling client on the following errors
     if (status == STATUS_SIGNALING_ICE_CONFIG_REFRESH_FAILED || status == STATUS_SIGNALING_RECONNECT_FAILED) {
+        DLOGI("DEBUG: Status matches, setting recreateSignalingClient=TRUE");
         ATOMIC_STORE_BOOL(&pSampleConfiguration->recreateSignalingClient, TRUE);
         CVAR_BROADCAST(pSampleConfiguration->cvar);
+    } else {
+        DLOGI("DEBUG: Status does not match reconnect conditions, not setting recreateSignalingClient flag");
     }
 
     return STATUS_SUCCESS;
@@ -1682,24 +1689,70 @@ STATUS sessionCleanupWait(PSampleConfiguration pSampleConfiguration, BOOL isSign
             sessionFreed = FALSE;
         }
 
-        // Check if we need to re-create the signaling client on-the-fly
-        if (ATOMIC_LOAD_BOOL(&pSampleConfiguration->recreateSignalingClient) &&
+        // Check if we need to re-create the signaling client on-the-fly with retry mechanism
+        BOOL needsRecreate = ATOMIC_LOAD_BOOL(&pSampleConfiguration->recreateSignalingClient);
+        if (needsRecreate) {
+            DLOGD("recreateSignalingClient flag is TRUE, checking conditions");
+        }
+
+        if (needsRecreate &&
             gWebRtcAppConfig.pSignalingClientInterface != NULL &&
             gSignalingClientData != NULL) {
 
-            DLOGI("Reconnecting signaling client");
-
-            // Disconnect and reconnect
-            CHK_STATUS(gWebRtcAppConfig.pSignalingClientInterface->disconnect(gSignalingClientData));
-            retStatus = gWebRtcAppConfig.pSignalingClientInterface->connect(gSignalingClientData);
-
-            if (STATUS_SUCCEEDED(retStatus)) {
-                // Re-set the variable again
-                ATOMIC_STORE_BOOL(&pSampleConfiguration->recreateSignalingClient, FALSE);
+            // Static variables to track retry attempts and timing
+            static UINT32 retryCount = 0;
+            static UINT64 lastRetryTime = 0;
+            UINT64 currentTime = GETTIME();
+
+            // Exponential backoff: 5s, 10s, 20s, 40s, 60s (max)
+            UINT64 retryDelays[] = {
+                5 * HUNDREDS_OF_NANOS_IN_A_SECOND,   // 5 seconds
+                10 * HUNDREDS_OF_NANOS_IN_A_SECOND,  // 10 seconds
+                20 * HUNDREDS_OF_NANOS_IN_A_SECOND,  // 20 seconds
+                40 * HUNDREDS_OF_NANOS_IN_A_SECOND,  // 40 seconds
+                60 * HUNDREDS_OF_NANOS_IN_A_SECOND   // 60 seconds (max)
+            };
+            UINT32 maxRetryIndex = ARRAY_SIZE(retryDelays) - 1;
+            UINT32 currentRetryIndex = MIN(retryCount, maxRetryIndex);
+            UINT64 retryDelay = retryDelays[currentRetryIndex];
+
+            // Check if enough time has passed since last retry attempt
+            BOOL shouldRetry = (lastRetryTime == 0) || (currentTime - lastRetryTime >= retryDelay);
+
+            if (shouldRetry) {
+                DLOGI("Reconnecting signaling client (attempt %d, delay: %llu seconds)",
+                      retryCount + 1, retryDelay / HUNDREDS_OF_NANOS_IN_A_SECOND);
+
+                // Disconnect and reconnect
+                CHK_STATUS(gWebRtcAppConfig.pSignalingClientInterface->disconnect(gSignalingClientData));
+                retStatus = gWebRtcAppConfig.pSignalingClientInterface->connect(gSignalingClientData);
+
+                if (STATUS_SUCCEEDED(retStatus)) {
+                    // Reset retry tracking on successful reconnection
+                    DLOGI("Signaling client reconnected successfully after %d attempts", retryCount + 1);
+                    ATOMIC_STORE_BOOL(&pSampleConfiguration->recreateSignalingClient, FALSE);
+                    retryCount = 0;
+                    lastRetryTime = 0;
+                    raiseEvent(APP_WEBRTC_EVENT_SIGNALING_CONNECTED, 0, NULL, "Signaling client reconnected");
+                } else {
+                    // Update retry tracking on failure
+                    retryCount++;
+                    lastRetryTime = currentTime;
+                    DLOGE("Failed to reconnect signaling client: 0x%08x (attempt %d, next retry in %llu seconds)",
+                          retStatus, retryCount,
+                          retryDelays[MIN(retryCount, maxRetryIndex)] / HUNDREDS_OF_NANOS_IN_A_SECOND);
+
+                    // Keep the recreateSignalingClient flag set to continue retrying
+                    // Don't reset it - we want to keep trying
+
+                    // Reset status to avoid breaking the loop
+                    retStatus = STATUS_SUCCESS;
+                }
             } else {
-                DLOGE("Failed to reconnect signaling client: 0x%08x", retStatus);
-                // Reset status to avoid breaking the loop
-                retStatus = STATUS_SUCCESS;
+                // Not time to retry yet
+                UINT64 timeUntilNextRetry = retryDelay - (currentTime - lastRetryTime);
+                DLOGD("Waiting %llu more seconds before next reconnection attempt",
+                      timeUntilNextRetry / HUNDREDS_OF_NANOS_IN_A_SECOND);
             }
         }
 

esp_port/components/kvs_webrtc/src/kvs_signaling.c

@@ -124,8 +124,9 @@ static STATUS adapterErrorCallback(UINT64 customData, STATUS errorStatus, PCHAR
         return STATUS_SUCCESS;
     }
 
-    WEBRTC_STATUS webrtcStatus = convertStatusToWebrtcStatus(errorStatus);
-    WEBRTC_STATUS result = pAdapterData->originalOnError(pAdapterData->originalCustomData, webrtcStatus, errorMessage, subErrorCode);
+    // Preserve the original status code instead of converting to generic WEBRTC_STATUS_INTERNAL_ERROR
+    // This ensures signaling reconnection status codes are passed through correctly
+    WEBRTC_STATUS result = pAdapterData->originalOnError(pAdapterData->originalCustomData, (WEBRTC_STATUS)errorStatus, errorMessage, subErrorCode);
 
     return (result == WEBRTC_STATUS_SUCCESS) ? STATUS_SUCCESS : STATUS_INTERNAL_ERROR;
 }