From aaf851142b8a6426c5bd95bffc4f2d3239ad3491 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Tue, 15 Jul 2025 17:50:10 +0100 Subject: [PATCH 01/39] Add RTCM1005/1006 parsing - resolves #256 - requires SEMP v1.0.4 --- Firmware/RTK_Everywhere/Begin.ino | 16 +++++++++++ Firmware/RTK_Everywhere/ESPNOW.ino | 1 + Firmware/RTK_Everywhere/GNSS_ZED.ino | 28 ------------------- Firmware/RTK_Everywhere/LoRa.ino | 6 ++++ Firmware/RTK_Everywhere/MQTT_Client.ino | 2 ++ Firmware/RTK_Everywhere/NtripClient.ino | 1 + .../RTK_Everywhere/PointPerfectLibrary.ino | 1 + Firmware/RTK_Everywhere/RTK_Everywhere.ino | 4 +++ Firmware/RTK_Everywhere/System.ino | 24 ++++++++++++++++ Firmware/RTK_Everywhere/Tasks.ino | 1 + Firmware/RTK_Everywhere/menuMain.ino | 7 +++++ 11 files changed, 63 insertions(+), 28 deletions(-) diff --git a/Firmware/RTK_Everywhere/Begin.ino b/Firmware/RTK_Everywhere/Begin.ino index 353387d7f..dba2e6995 100644 --- a/Firmware/RTK_Everywhere/Begin.ino +++ b/Firmware/RTK_Everywhere/Begin.ino @@ -1071,6 +1071,22 @@ void beginGnssUart2() serial2GNSS->begin(115200, SERIAL_8N1, pin_GnssUart2_RX, pin_GnssUart2_TX); } +void beginRtcmParse() +{ + SEMP_PARSE_ROUTINE const rtcmParserTable[] = { sempRtcmPreamble }; + const char *const rtcmParserNames[] = { "RTCM" }; + + // Begin the RTCM parser - which will extract the base location from RTCM1005 / 1006 + rtcmParse = sempBeginParser(rtcmParserTable, 1, rtcmParserNames, 1, + 0, // Scratchpad bytes + 1050, // Buffer length + processRTCMMessage, // eom Call Back + "rtcmParse"); // Parser Name + if (!rtcmParse) + reportFatalError("Failed to initialize the RTCM parser"); + +} + void beginFS() { if (online.fs == false) diff --git a/Firmware/RTK_Everywhere/ESPNOW.ino b/Firmware/RTK_Everywhere/ESPNOW.ino index 3a444e985..b725a4a84 100644 --- a/Firmware/RTK_Everywhere/ESPNOW.ino +++ b/Firmware/RTK_Everywhere/ESPNOW.ino @@ -161,6 +161,7 @@ void espNowOnDataReceived(const esp_now_recv_info *mac, const uint8_t *incomingD { // Pass RTCM bytes (presumably) from ESP NOW out ESP32-UART to GNSS gnss->pushRawData((uint8_t *)incomingData, len); + sempParseNextBytes(rtcmParse, (uint8_t *)incomingData, len); // Parse the data for RTCM1005/1006 if ((settings.debugEspNow == true || settings.debugCorrections == true) && !inMainMenu) systemPrintf("ESPNOW received %d RTCM bytes, pushed to GNSS, RSSI: %d\r\n", len, espNowRSSI); diff --git a/Firmware/RTK_Everywhere/GNSS_ZED.ino b/Firmware/RTK_Everywhere/GNSS_ZED.ino index 7a3f8fcb0..c07b92b32 100644 --- a/Firmware/RTK_Everywhere/GNSS_ZED.ino +++ b/Firmware/RTK_Everywhere/GNSS_ZED.ino @@ -541,10 +541,6 @@ bool GNSS_ZED::configureGNSS() _zed->setAutoPVTcallbackPtr(&storePVTdata, VAL_LAYER_ALL); // Enable automatic NAV PVT messages with callback to storePVTdata response &= _zed->setAutoHPPOSLLHcallbackPtr( &storeHPdata, VAL_LAYER_ALL); // Enable automatic NAV HPPOSLLH messages with callback to storeHPdata - _zed->setRTCM1005InputcallbackPtr( - &storeRTCM1005data); // Configure a callback for RTCM 1005 - parsed from pushRawData - _zed->setRTCM1006InputcallbackPtr( - &storeRTCM1006data); // Configure a callback for RTCM 1006 - parsed from pushRawData if (present.timePulseInterrupt) response &= _zed->setAutoTIMTPcallbackPtr( @@ -2597,30 +2593,6 @@ void GNSS_ZED::storeMONCOMMSdataRadio(UBX_MON_COMMS_data_t *ubxDataStruct) } } -//---------------------------------------- -// Callback to save ARPECEF* -//---------------------------------------- -void storeRTCM1005data(RTCM_1005_data_t *rtcmData1005) -{ - ARPECEFX = rtcmData1005->AntennaReferencePointECEFX; - ARPECEFY = rtcmData1005->AntennaReferencePointECEFY; - ARPECEFZ = rtcmData1005->AntennaReferencePointECEFZ; - ARPECEFH = 0; - newARPAvailable = true; -} - -//---------------------------------------- -// Callback to save ARPECEF* -//---------------------------------------- -void storeRTCM1006data(RTCM_1006_data_t *rtcmData1006) -{ - ARPECEFX = rtcmData1006->AntennaReferencePointECEFX; - ARPECEFY = rtcmData1006->AntennaReferencePointECEFY; - ARPECEFZ = rtcmData1006->AntennaReferencePointECEFZ; - ARPECEFH = rtcmData1006->AntennaHeight; - newARPAvailable = true; -} - //---------------------------------------- void storeTIMTPdata(UBX_TIM_TP_data_t *ubxDataStruct) { diff --git a/Firmware/RTK_Everywhere/LoRa.ino b/Firmware/RTK_Everywhere/LoRa.ino index 7901fc3a7..0141ef48a 100644 --- a/Firmware/RTK_Everywhere/LoRa.ino +++ b/Firmware/RTK_Everywhere/LoRa.ino @@ -175,6 +175,9 @@ void updateLora() // Pass RTCM bytes (presumably) from LoRa out ESP32-UART to GNSS gnss->pushRawData(rtcmData, rtcmCount); // Push RTCM to GNSS module + // Parse the data for RTCM1005/1006 + sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); + if (((settings.debugCorrections == true) || (settings.debugLora == true)) && !inMainMenu) { systemFlush(); // Complete prints @@ -254,6 +257,9 @@ void updateLora() // Pass RTCM bytes (presumably) from LoRa out ESP32-UART to GNSS gnss->pushRawData(rtcmData, rtcmCount); // Push RTCM to GNSS module + // Parse the data for RTCM1005/1006 + sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); + if (((settings.debugCorrections == true) || (settings.debugLora == true)) && !inMainMenu) { systemFlush(); // Complete prints diff --git a/Firmware/RTK_Everywhere/MQTT_Client.ino b/Firmware/RTK_Everywhere/MQTT_Client.ino index b5a6e905d..ba970bc67 100644 --- a/Firmware/RTK_Everywhere/MQTT_Client.ino +++ b/Firmware/RTK_Everywhere/MQTT_Client.ino @@ -514,6 +514,7 @@ int mqttClientProcessZedMessage(uint8_t *mqttData, uint16_t mqttCount, int bytes zed->updateCorrectionsSource(0); // Set SOURCE to 0 (IP) if needed gnss->pushRawData(mqttData, mqttCount); + sempParseNextBytes(rtcmParse, mqttData, mqttCount); // Parse the data for RTCM1005/1006 bytesPushed += mqttCount; mqttClientDataReceived = true; @@ -542,6 +543,7 @@ int mqttClientProcessZedMessage(uint8_t *mqttData, uint16_t mqttCount, int bytes } gnss->pushRawData(mqttData, mqttCount); + sempParseNextBytes(rtcmParse, mqttData, mqttCount); // Parse the data for RTCM1005/1006 bytesPushed += mqttCount; } #endif // COMPILE_ZED diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index a686367d6..67d7900bb 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -919,6 +919,7 @@ void ntripClientUpdate() { // Push RTCM to GNSS module over I2C / SPI gnss->pushRawData(rtcmData, rtcmCount); + sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); // Parse the data for RTCM1005/1006 if ((settings.debugCorrections || settings.debugNtripClientRtcm || PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && diff --git a/Firmware/RTK_Everywhere/PointPerfectLibrary.ino b/Firmware/RTK_Everywhere/PointPerfectLibrary.ino index 1116dd4ca..93e3175ec 100644 --- a/Firmware/RTK_Everywhere/PointPerfectLibrary.ino +++ b/Firmware/RTK_Everywhere/PointPerfectLibrary.ino @@ -48,6 +48,7 @@ void updatePplTask(void *e) } gnss->pushRawData(pplRtcmBuffer, rtcmLength); + sempParseNextBytes(rtcmParse, pplRtcmBuffer, rtcmLength); // Parse the data for RTCM1005/1006 if (settings.debugCorrections == true && !inMainMenu) systemPrintf("Received %d RTCM bytes from PPL. Pushed to the GNSS.\r\n", rtcmLength); diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 43627718f..77290fffb 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -480,6 +480,7 @@ bool usbSerialIncomingRtcm; // Incoming RTCM over the USB serial port SEMP_PARSE_STATE *rtkParse = nullptr; SEMP_PARSE_STATE *sbfParse = nullptr; // mosaic-X5 SEMP_PARSE_STATE *spartnParse = nullptr; // mosaic-X5 +SEMP_PARSE_STATE *rtcmParse = nullptr; // Parse incoming corrections for RTCM1005 / 1006 base locations //=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= @@ -1229,6 +1230,9 @@ void setup() DMW_b("beginGnssUart2"); beginGnssUart2(); + DMW_b("beginRtcmParse"); + beginRtcmParse(); + DMW_b("displaySplash"); displaySplash(); // Display the RTK product name and firmware version diff --git a/Firmware/RTK_Everywhere/System.ino b/Firmware/RTK_Everywhere/System.ino index b48fbc647..774c8d843 100644 --- a/Firmware/RTK_Everywhere/System.ino +++ b/Firmware/RTK_Everywhere/System.ino @@ -872,3 +872,27 @@ void getMacAddresses(uint8_t *macAddress, const char *name, esp_mac_type_t type, systemPrintf("%02X:%02X:%02X:%02X:%02X:%02X - %s\r\n", macAddress[0], macAddress[1], macAddress[2], macAddress[3], macAddress[4], macAddress[5], name); }; + +// Check and record the base location in RTCM1005/1006 +void processRTCMMessage(SEMP_PARSE_STATE *parse, uint16_t type) +{ + SEMP_SCRATCH_PAD *scratchPad = (SEMP_SCRATCH_PAD *)parse->scratchPad; + + if (sempRtcmGetMessageNumber(parse) == 1005) + { + ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); + ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); + ARPECEFZ = sempRtcmGetSignedBits(parse, 114, 38); + ARPECEFH = 0; + newARPAvailable = true; + } + + if (sempRtcmGetMessageNumber(parse) == 1005) + { + ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); + ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); + ARPECEFZ = sempRtcmGetSignedBits(parse, 114, 38); + ARPECEFH = sempRtcmGetUnsignedBits(parse, 152, 16); + newARPAvailable = true; + } +} \ No newline at end of file diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index 685556ec6..0ba9bf576 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -287,6 +287,7 @@ void sendGnssBuffer() { if (correctionLastSeen(CORR_BLUETOOTH)) { + sempParseNextBytes(rtcmParse, bluetoothOutgoingToGnss, bluetoothOutgoingToGnssHead); // Parse the data for RTCM1005/1006 if (gnss->pushRawData(bluetoothOutgoingToGnss, bluetoothOutgoingToGnssHead)) { if ((settings.debugCorrections || PERIODIC_DISPLAY(PD_GNSS_DATA_TX)) && !inMainMenu) diff --git a/Firmware/RTK_Everywhere/menuMain.ino b/Firmware/RTK_Everywhere/menuMain.ino index e6174f9d1..579b35a95 100644 --- a/Firmware/RTK_Everywhere/menuMain.ino +++ b/Firmware/RTK_Everywhere/menuMain.ino @@ -32,7 +32,11 @@ void terminalUpdate() // Push RTCM to GNSS module over I2C / SPI if (correctionLastSeen(CORR_USB)) + { gnss->pushRawData((uint8_t *)buffer, length); + sempParseNextBytes(rtcmParse, (uint8_t *)buffer, length); // Parse the data for RTCM1005/1006 + } + } // Does incoming data consist of RTCM correction messages @@ -51,7 +55,10 @@ void terminalUpdate() // Push RTCM to GNSS module over I2C / SPI if (correctionLastSeen(CORR_USB)) + { gnss->pushRawData((uint8_t *)buffer, length); + sempParseNextBytes(rtcmParse, (uint8_t *)buffer, length); // Parse the data for RTCM1005/1006 + } } else { From 2ca58411df0723948e5dd1d5fdeed3dd9879b3a3 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 16 Jul 2025 12:26:39 +0100 Subject: [PATCH 02/39] Move rtcmParserTable etc. into Tasks.ino. Make table global --- Firmware/RTK_Everywhere/Begin.ino | 16 -------- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 9 +++-- Firmware/RTK_Everywhere/System.ino | 24 ------------ Firmware/RTK_Everywhere/Tasks.ino | 43 ++++++++++++++++++++++ 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/Firmware/RTK_Everywhere/Begin.ino b/Firmware/RTK_Everywhere/Begin.ino index dba2e6995..353387d7f 100644 --- a/Firmware/RTK_Everywhere/Begin.ino +++ b/Firmware/RTK_Everywhere/Begin.ino @@ -1071,22 +1071,6 @@ void beginGnssUart2() serial2GNSS->begin(115200, SERIAL_8N1, pin_GnssUart2_RX, pin_GnssUart2_TX); } -void beginRtcmParse() -{ - SEMP_PARSE_ROUTINE const rtcmParserTable[] = { sempRtcmPreamble }; - const char *const rtcmParserNames[] = { "RTCM" }; - - // Begin the RTCM parser - which will extract the base location from RTCM1005 / 1006 - rtcmParse = sempBeginParser(rtcmParserTable, 1, rtcmParserNames, 1, - 0, // Scratchpad bytes - 1050, // Buffer length - processRTCMMessage, // eom Call Back - "rtcmParse"); // Parser Name - if (!rtcmParse) - reportFatalError("Failed to initialize the RTCM parser"); - -} - void beginFS() { if (online.fs == false) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 77290fffb..9877e5966 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -807,7 +807,7 @@ std::vector setupButtons; // A vector (linked list) of the setup 'b bool firstRoverStart; // Used to detect if the user is toggling the power button at POR to enter the test menu -bool newEventToRecord; // Goes true when INT pin goes high +bool newEventToRecord; // Goes true when INT pin goes high. Currently this is ZED-specific. uint32_t triggerCount; // Global copy - TM2 event counter uint32_t triggerTowMsR; // Global copy - Time Of Week of rising edge (ms) uint32_t triggerTowSubMsR; // Global copy - Millisecond fraction of Time Of Week of rising edge in nanoseconds @@ -1489,7 +1489,8 @@ void logUpdate() // Record any pending trigger events if (newEventToRecord == true) { - systemPrintln("Recording event"); + if (settings.enablePrintLogFileStatus) + systemPrintln("Log file: recording event"); // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of // rising edge (ns), and accuracy estimate (ns) @@ -1527,7 +1528,8 @@ void logUpdate() if (newARPAvailable == true && settings.enableARPLogging && ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) { - systemPrintln("Recording Antenna Reference Position"); + if (settings.enablePrintLogFileStatus) + systemPrintln("Log file: recording Antenna Reference Position"); lastARPLog = millis(); newARPAvailable = false; @@ -1554,7 +1556,6 @@ void logUpdate() logFile->println(nmeaMessage); xSemaphoreGive(sdCardSemaphore); - newEventToRecord = false; } else { diff --git a/Firmware/RTK_Everywhere/System.ino b/Firmware/RTK_Everywhere/System.ino index 774c8d843..b48fbc647 100644 --- a/Firmware/RTK_Everywhere/System.ino +++ b/Firmware/RTK_Everywhere/System.ino @@ -872,27 +872,3 @@ void getMacAddresses(uint8_t *macAddress, const char *name, esp_mac_type_t type, systemPrintf("%02X:%02X:%02X:%02X:%02X:%02X - %s\r\n", macAddress[0], macAddress[1], macAddress[2], macAddress[3], macAddress[4], macAddress[5], name); }; - -// Check and record the base location in RTCM1005/1006 -void processRTCMMessage(SEMP_PARSE_STATE *parse, uint16_t type) -{ - SEMP_SCRATCH_PAD *scratchPad = (SEMP_SCRATCH_PAD *)parse->scratchPad; - - if (sempRtcmGetMessageNumber(parse) == 1005) - { - ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); - ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); - ARPECEFZ = sempRtcmGetSignedBits(parse, 114, 38); - ARPECEFH = 0; - newARPAvailable = true; - } - - if (sempRtcmGetMessageNumber(parse) == 1005) - { - ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); - ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); - ARPECEFZ = sempRtcmGetSignedBits(parse, 114, 38); - ARPECEFH = sempRtcmGetUnsignedBits(parse, 152, 16); - newARPAvailable = true; - } -} \ No newline at end of file diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index 0ba9bf576..36acb5efc 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -94,6 +94,7 @@ const char *const sbfParserNames[] = { "SBF", }; const int sbfParserNameCount = sizeof(sbfParserNames) / sizeof(sbfParserNames[0]); + SEMP_PARSE_ROUTINE const spartnParserTable[] = {sempSpartnPreamble}; const int spartnParserCount = sizeof(spartnParserTable) / sizeof(spartnParserTable[0]); const char *const spartnParserNames[] = { @@ -101,6 +102,11 @@ const char *const spartnParserNames[] = { }; const int spartnParserNameCount = sizeof(spartnParserNames) / sizeof(spartnParserNames[0]); +SEMP_PARSE_ROUTINE const rtcmParserTable[] = { sempRtcmPreamble }; +const int rtcmParserCount = sizeof(rtcmParserTable) / sizeof(rtcmParserTable[0]); +const char *const rtcmParserNames[] = { "RTCM" }; +const int rtcmParserNameCount = sizeof(rtcmParserNames) / sizeof(rtcmParserNames[0]); + //---------------------------------------- // Locals //---------------------------------------- @@ -2196,3 +2202,40 @@ void bluetoothCommandTask(void *pvParameters) task.bluetoothCommandTaskRunning = false; vTaskDelete(NULL); } + +void beginRtcmParse() +{ + // Begin the RTCM parser - which will extract the base location from RTCM1005 / 1006 + rtcmParse = sempBeginParser(rtcmParserTable, rtcmParserCount, rtcmParserNames, rtcmParserNameCount, + 0, // Scratchpad bytes + 1050, // Buffer length + processRTCMMessage, // eom Call Back + "rtcmParse"); // Parser Name + if (!rtcmParse) + reportFatalError("Failed to initialize the RTCM parser"); + +} + +// Check and record the base location in RTCM1005/1006 +void processRTCMMessage(SEMP_PARSE_STATE *parse, uint16_t type) +{ + SEMP_SCRATCH_PAD *scratchPad = (SEMP_SCRATCH_PAD *)parse->scratchPad; + + if (sempRtcmGetMessageNumber(parse) == 1005) + { + ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); + ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); + ARPECEFZ = sempRtcmGetSignedBits(parse, 114, 38); + ARPECEFH = 0; + newARPAvailable = true; + } + + if (sempRtcmGetMessageNumber(parse) == 1005) + { + ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); + ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); + ARPECEFZ = sempRtcmGetSignedBits(parse, 114, 38); + ARPECEFH = sempRtcmGetUnsignedBits(parse, 152, 16); + newARPAvailable = true; + } +} \ No newline at end of file From 6dd3503295d98c02bf412e369e6d3fa7ee78043e Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 16 Jul 2025 13:22:01 +0100 Subject: [PATCH 03/39] Make rtcmData[] and rtcmCount static --- Firmware/RTK_Everywhere/NtripClient.ino | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index 67d7900bb..a2fb0eaf2 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -897,8 +897,8 @@ void ntripClientUpdate() else { // Receive data from the NTRIP Caster - uint8_t rtcmData[RTCM_DATA_SIZE]; - size_t rtcmCount = 0; + static uint8_t rtcmData[RTCM_DATA_SIZE]; + static size_t rtcmCount = 0; // Collect any available RTCM data if (ntripClientReceiveDataAvailable() > 0) From 46dbdf7a4390831de35a51ffc553098fbc8dc8fc Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 16 Jul 2025 13:24:23 +0100 Subject: [PATCH 04/39] Better ARP log file debug print --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 9877e5966..96ad0306d 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1223,6 +1223,9 @@ void setup() beginVersion(); // Assemble platform name. Requires settings/LFS. + //if (esp_reset_reason() == ESP_RST_PANIC) // Halt on PANIC - to trap rare crashes + // reportFatalError("ESP_RST_PANIC"); + DMW_b("beginGnssUart"); beginGnssUart(); // Requires settings. Start the UART connected to the GNSS receiver on core 0. Start before // gnssBegin in case it is needed (Torch). @@ -1528,9 +1531,6 @@ void logUpdate() if (newARPAvailable == true && settings.enableARPLogging && ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) { - if (settings.enablePrintLogFileStatus) - systemPrintln("Log file: recording Antenna Reference Position"); - lastARPLog = millis(); newARPAvailable = false; @@ -1545,6 +1545,9 @@ void logUpdate() char ARPData[82]; // Max NMEA sentence length is 82 snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); + if (settings.enablePrintLogFileStatus) + systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); + char nmeaMessage[82]; // Max NMEA sentence length is 82 createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), ARPData); // textID, buffer, sizeOfBuffer, text From e2df4349d444b351f05047a949c8edc97cbad249 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 11:08:10 +0100 Subject: [PATCH 05/39] Add missing menuPeriodicPrint systemPrints --- Firmware/RTK_Everywhere/menuSystem.ino | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Firmware/RTK_Everywhere/menuSystem.ino b/Firmware/RTK_Everywhere/menuSystem.ino index 4bb1a6134..df4f8e020 100644 --- a/Firmware/RTK_Everywhere/menuSystem.ino +++ b/Firmware/RTK_Everywhere/menuSystem.ino @@ -1279,12 +1279,14 @@ void menuPeriodicPrint() else if (incoming == 20) { + systemPrint("Enter the new periodic print mask: "); int value = getUserInputNumber(); if ((value != INPUT_RESPONSE_GETNUMBER_EXIT) && (value != INPUT_RESPONSE_GETNUMBER_TIMEOUT)) settings.periodicDisplay = value; } else if (incoming == 21) { + systemPrint("Enter the new periodic display interval (s): "); int seconds = getUserInputNumber(); if ((seconds != INPUT_RESPONSE_GETNUMBER_EXIT) && (seconds != INPUT_RESPONSE_GETNUMBER_TIMEOUT)) settings.periodicDisplayInterval = seconds * 1000; From a7f048a421070ff70d7ebf7142904b02375eb532 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 11:10:05 +0100 Subject: [PATCH 06/39] Add FUNCTION_ARPWRITE --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 5 ++++- Firmware/RTK_Everywhere/settings.h | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 96ad0306d..f198c250e 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1554,7 +1554,7 @@ void logUpdate() if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) { - markSemaphore(FUNCTION_EVENT); + markSemaphore(FUNCTION_ARPWRITE); logFile->println(nmeaMessage); @@ -1815,5 +1815,8 @@ void getSemaphoreFunction(char *functionName) case FUNCTION_NTPEVENT: strcpy(functionName, "NTP Event"); break; + case FUNCTION_ARPWRITE: + strcpy(functionName, "ARP Write"); + break; } } diff --git a/Firmware/RTK_Everywhere/settings.h b/Firmware/RTK_Everywhere/settings.h index 955af0f64..e0e956f08 100644 --- a/Firmware/RTK_Everywhere/settings.h +++ b/Firmware/RTK_Everywhere/settings.h @@ -495,6 +495,7 @@ typedef enum FUNCTION_LOG_CLOSURE, FUNCTION_PRINT_FILE_LIST, FUNCTION_NTPEVENT, + FUNCTION_ARPWRITE, } SemaphoreFunction; // Print the base coordinates in different formats, depending on the type the user has entered From 87e6b90859dd1ab55cea8960a801f4f17bfd8956 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 15:03:10 +0100 Subject: [PATCH 07/39] Restructure sdCardPresent * On postcard, avoid the SPI 0xFF SD card detect test * Begin the GPIO expander before the SD * Use the semaphore to protect the SPI 0xFF card detect test --- Firmware/RTK_Everywhere/Buttons.ino | 2 +- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 19 +++-- Firmware/RTK_Everywhere/SD.ino | 88 +++++++++++++++------- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/Firmware/RTK_Everywhere/Buttons.ino b/Firmware/RTK_Everywhere/Buttons.ino index 76c240300..3a6a7d644 100644 --- a/Firmware/RTK_Everywhere/Buttons.ino +++ b/Firmware/RTK_Everywhere/Buttons.ino @@ -119,7 +119,7 @@ void buttonRead() gpioChanged = false; // Get all the pins in one read - uint8_t currentState = io.getInputRegister() & 0b00111111; // Ignore unconnected GPIO6/7 + uint8_t currentState = io.getInputRegister() & 0b00011111; // Mask the five buttons. Ignore SD detect if (currentState != gpioExpander_previousState) { diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index f198c250e..a3e71c51c 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1242,6 +1242,9 @@ void setup() DMW_b("gnss->begin"); gnss->begin(); // Requires settings. Connect to GNSS to get module type + DMW_b("beginButtons"); + beginButtons(); // Start task for button monitoring. Needed for beginSD (gpioExpander) + DMW_b("beginSD"); beginSD(); // Requires settings. Test if SD is present @@ -1286,9 +1289,6 @@ void setup() DMW_b("beginInterrupts"); beginInterrupts(); // Begin the TP interrupts - DMW_b("beginButtons"); - beginButtons(); // Start task for button monitoring. - DMW_b("beginSystemState"); beginSystemState(); // Determine initial system state. @@ -1509,7 +1509,11 @@ void logUpdate() { markSemaphore(FUNCTION_EVENT); - logFile->println(nmeaMessage); + if (logFile) + { + logFile->println(nmeaMessage); + logFile->sync(); + } xSemaphoreGive(sdCardSemaphore); newEventToRecord = false; @@ -1556,7 +1560,12 @@ void logUpdate() { markSemaphore(FUNCTION_ARPWRITE); - logFile->println(nmeaMessage); + if (logFile) + { + // See #695. This seems to cause the rare ntripClient->read "(pbuf_free: p->ref > 0)" error + logFile->println(nmeaMessage); + logFile->sync(); + } xSemaphoreGive(sdCardSemaphore); } diff --git a/Firmware/RTK_Everywhere/SD.ino b/Firmware/RTK_Everywhere/SD.ino index 2cf8e48c7..e37d25f49 100644 --- a/Firmware/RTK_Everywhere/SD.ino +++ b/Firmware/RTK_Everywhere/SD.ino @@ -49,7 +49,10 @@ void sdUpdate() // Check if SD card is still present if (sdCardPresent() == false) + { + systemPrintln("SD removed"); endSD(false, true); //(alreadyHaveSemaphore, releaseSemaphore) Close down SD. + } } /* @@ -94,44 +97,75 @@ bool sdCardPresent(void) return (true); // Card detect high = SD in place return (false); // Card detect low = No SD } - else if (present.microSdCardDetectGpioExpanderHigh == true && online.gpioExpander == true) + else if (present.microSdCardDetectGpioExpanderHigh == true) { - if (io.digitalRead(gpioExpander_cardDetect) == GPIO_EXPANDER_CARD_INSERTED) - return (true); // Card detect high = SD in place - return (false); // Card detect low = No SD + if (online.gpioExpander == true) + { + if (io.digitalRead(gpioExpander_cardDetect) == GPIO_EXPANDER_CARD_INSERTED) + return (true); // Card detect high = SD in place + return (false); // Card detect low = No SD + } + else + { + reportFatalError("sdCardPresent: gpioExpander not online."); + return (false); + } } // else - no card detect pin. Use software detect - // Use software to detect a card - DMW_if systemPrintf("pin_microSD_CS: %d\r\n", pin_microSD_CS); - if (pin_microSD_CS == -1) - reportFatalError("Illegal SD CS pin assignment."); + // Note: even though this is protected by the semaphore, + // this will probably cause issues / corruption if + // a SdFile is open for writing...? + + static bool previousCardPresentBySW = false; + + if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_longWait_ms) == pdPASS) + { + markSemaphore(FUNCTION_RECORDSETTINGS); + + // Use software to detect a card + DMW_if systemPrintf("pin_microSD_CS: %d\r\n", pin_microSD_CS); + if (pin_microSD_CS == -1) + reportFatalError("Illegal SD CS pin assignment."); + + byte response = 0; - byte response = 0; + beginSPI(false); + SPI.setClockDivider(SPI_CLOCK_DIV2); + SPI.setDataMode(SPI_MODE0); + SPI.setBitOrder(MSBFIRST); + pinMode(pin_microSD_CS, OUTPUT); - beginSPI(false); - SPI.setClockDivider(SPI_CLOCK_DIV2); - SPI.setDataMode(SPI_MODE0); - SPI.setBitOrder(MSBFIRST); - pinMode(pin_microSD_CS, OUTPUT); + // Sending clocks while card power stabilizes... + sdDeselectCard(); // always make sure + for (byte i = 0; i < 30; i++) // send several clocks while card power stabilizes + xchg(0xff); + + // Sending CMD0 - GO IDLE... + for (byte i = 0; i < 0x10; i++) // Attempt to go idle + { + response = sdSendCommand(SD_GO_IDLE, 0); // send CMD0 - go to idle state + if (response == 1) + break; + } - // Sending clocks while card power stabilizes... - sdDeselectCard(); // always make sure - for (byte i = 0; i < 30; i++) // send several clocks while card power stabilizes - xchg(0xff); + xSemaphoreGive(sdCardSemaphore); + + if (response != 1) + { + previousCardPresentBySW = false; + return (false); // Card failed to respond to idle + } - // Sending CMD0 - GO IDLE... - for (byte i = 0; i < 0x10; i++) // Attempt to go idle + previousCardPresentBySW = true; + return (true); // Card detected + } + else { - response = sdSendCommand(SD_GO_IDLE, 0); // send CMD0 - go to idle state - if (response == 1) - break; + // Could not get semaphore. Return previous state + return previousCardPresentBySW; } - if (response != 1) - return (false); // Card failed to respond to idle - - return (true); // Card detected } /* From 411852a2ee9266e0ef0cd305fb2ca6d0ae130d5f Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 15:05:38 +0100 Subject: [PATCH 08/39] Protect ntripClient = new NetworkClient(); --- Firmware/RTK_Everywhere/NtripClient.ino | 75 ++++++++++++------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index a2fb0eaf2..58f321954 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -138,7 +138,7 @@ static const uint32_t NTRIP_CLIENT_RESPONSE_TIMEOUT = 10 * 1000; // Milliseconds static const uint32_t NTRIP_CLIENT_RECEIVE_DATA_TIMEOUT = 30 * 1000; // Milliseconds // Most incoming data is around 500 bytes but may be larger -static const int RTCM_DATA_SIZE = 512 * 4; +static const size_t RTCM_DATA_SIZE = 512 * 4; // NTRIP client server request buffer size static const int SERVER_BUFFER_SIZE = CREDENTIALS_BUFFER_SIZE + 3; @@ -248,7 +248,7 @@ bool ntripClientConnect() length = strlen(serverRequest); serverRequest[length++] = '\r'; serverRequest[length++] = '\n'; - serverRequest[length++] = 0; + serverRequest[length] = 0; // Set up the credentials char credentials[CREDENTIALS_BUFFER_SIZE]; @@ -657,7 +657,8 @@ void ntripClientUpdate() { // Allocate the ntripClient structure networkUseDefaultInterface(); - ntripClient = new NetworkClient(); + if (!ntripClient) + ntripClient = new NetworkClient(); if (!ntripClient) { // Failed to allocate the ntripClient structure @@ -897,49 +898,47 @@ void ntripClientUpdate() else { // Receive data from the NTRIP Caster - static uint8_t rtcmData[RTCM_DATA_SIZE]; - static size_t rtcmCount = 0; + uint8_t rtcmData[RTCM_DATA_SIZE]; - // Collect any available RTCM data - if (ntripClientReceiveDataAvailable() > 0) + // See #695 + // Rare LWIP ASSERT "(pbuf_free: p->ref > 0)" errors have been seen here + // - which trigger an esp_system_abort + int rtcmCount = ntripClient->read(rtcmData, RTCM_DATA_SIZE); + if (rtcmCount > 0) { - rtcmCount = ntripClient->read(rtcmData, sizeof(rtcmData)); - if (rtcmCount) - { - // Restart the NTRIP receive data timer - ntripClientTimer = millis(); + // Restart the NTRIP receive data timer + ntripClientTimer = millis(); + + // Record the arrival of RTCM from the WiFi connection. This resets the RTCM timeout used on the + // L-Band. + rtcmLastPacketReceived = millis(); - // Record the arrival of RTCM from the WiFi connection. This resets the RTCM timeout used on the - // L-Band. - rtcmLastPacketReceived = millis(); + netIncomingRTCM = true; - netIncomingRTCM = true; + if (correctionLastSeen(CORR_TCP)) + { + // Push RTCM to GNSS module over I2C / SPI + gnss->pushRawData(rtcmData, rtcmCount); + sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); // Parse the data for RTCM1005/1006 - if (correctionLastSeen(CORR_TCP)) + if ((settings.debugCorrections || settings.debugNtripClientRtcm || + PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && + (!inMainMenu)) { - // Push RTCM to GNSS module over I2C / SPI - gnss->pushRawData(rtcmData, rtcmCount); - sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); // Parse the data for RTCM1005/1006 - - if ((settings.debugCorrections || settings.debugNtripClientRtcm || - PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && - (!inMainMenu)) - { - PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); - systemPrintf("NTRIP Client received %d RTCM bytes, pushed to GNSS\r\n", rtcmCount); - } + PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); + systemPrintf("NTRIP Client received %d RTCM bytes, pushed to GNSS\r\n", rtcmCount); } - else + } + else + { + if ((settings.debugCorrections || settings.debugNtripClientRtcm || + PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && + (!inMainMenu)) { - if ((settings.debugCorrections || settings.debugNtripClientRtcm || - PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && - (!inMainMenu)) - { - PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); - systemPrintf( - "NTRIP Client received %d RTCM bytes, NOT pushed to GNSS due to priority\r\n", - rtcmCount); - } + PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); + systemPrintf( + "NTRIP Client received %d RTCM bytes, NOT pushed to GNSS due to priority\r\n", + rtcmCount); } } } From 52e77deed1de1f11361560f7173575231782b045 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 15:30:24 +0100 Subject: [PATCH 09/39] Initialize sdCardSemaphore = NULL --- Firmware/RTK_Everywhere/Begin.ino | 2 +- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Firmware/RTK_Everywhere/Begin.ino b/Firmware/RTK_Everywhere/Begin.ino index ed582782d..3f042ed21 100644 --- a/Firmware/RTK_Everywhere/Begin.ino +++ b/Firmware/RTK_Everywhere/Begin.ino @@ -846,7 +846,7 @@ void beginSD() while (settings.enableSD == true) { // Setup SD card access semaphore - if (sdCardSemaphore == nullptr) + if (sdCardSemaphore == NULL) sdCardSemaphore = xSemaphoreCreateMutex(); else if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) != pdPASS) { diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index a3e71c51c..67f1e55cd 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -328,7 +328,8 @@ int startCurrentLogTime_minutes; // System crashes if two tasks access a file at the same time // So we use a semaphore to see if the file system is available -SemaphoreHandle_t sdCardSemaphore; +// https://github.com/espressif/arduino-esp32/blob/master/libraries/ESP32/examples/FreeRTOS/Mutex/Mutex.ino#L11 +SemaphoreHandle_t sdCardSemaphore = NULL; TickType_t loggingSemaphoreWait_ms = 10 / portTICK_PERIOD_MS; const TickType_t fatSemaphore_shortWait_ms = 10 / portTICK_PERIOD_MS; const TickType_t fatSemaphore_longWait_ms = 200 / portTICK_PERIOD_MS; From 997e405d69c948e4335b3c7a2587d589ff55d01c Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 15:49:16 +0100 Subject: [PATCH 10/39] Sync log file before printing ARP data --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 2 ++ Firmware/RTK_Everywhere/menuMessages.ino | 1 + 2 files changed, 3 insertions(+) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 67f1e55cd..fc01179d7 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1512,6 +1512,7 @@ void logUpdate() if (logFile) { + logFile->sync(); // Sync any partially written data logFile->println(nmeaMessage); logFile->sync(); } @@ -1564,6 +1565,7 @@ void logUpdate() if (logFile) { // See #695. This seems to cause the rare ntripClient->read "(pbuf_free: p->ref > 0)" error + logFile->sync(); // Sync any partially written data logFile->println(nmeaMessage); logFile->sync(); } diff --git a/Firmware/RTK_Everywhere/menuMessages.ino b/Firmware/RTK_Everywhere/menuMessages.ino index aa726b230..cedb206d8 100644 --- a/Firmware/RTK_Everywhere/menuMessages.ino +++ b/Firmware/RTK_Everywhere/menuMessages.ino @@ -422,6 +422,7 @@ void endLogging(bool gotSemaphore, bool releaseSemaphore) char nmeaMessage[82]; // Max NMEA sentence length is 82 createNMEASentence(CUSTOM_NMEA_TYPE_PARSER_STATS, nmeaMessage, sizeof(nmeaMessage), parserStats); // textID, buffer, sizeOfBuffer, text + logFile->sync(); // Sync any partially written data logFile->println(nmeaMessage); logFile->sync(); From 05d3da14ac5395340a5e0075f7f887152ccfaa96 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 16:10:58 +0100 Subject: [PATCH 11/39] Don't record ARP if web server is running --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 140 +++++++++++---------- Firmware/RTK_Everywhere/menuMessages.ino | 2 + 2 files changed, 75 insertions(+), 67 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index fc01179d7..bfb6892eb 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1493,43 +1493,46 @@ void logUpdate() // Record any pending trigger events if (newEventToRecord == true) { - if (settings.enablePrintLogFileStatus) - systemPrintln("Log file: recording event"); + if (webServerIsRunning() == false) // Don't try to access the SD if the web server is running + { + if (settings.enablePrintLogFileStatus) + systemPrintln("Log file: recording event"); - // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of - // rising edge (ns), and accuracy estimate (ns) - char eventData[82]; // Max NMEA sentence length is 82 - snprintf(eventData, sizeof(eventData), "%d,%d,%d,%d", triggerCount, triggerTowMsR, triggerTowSubMsR, - triggerAccEst); + // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of + // rising edge (ns), and accuracy estimate (ns) + char eventData[82]; // Max NMEA sentence length is 82 + snprintf(eventData, sizeof(eventData), "%d,%d,%d,%d", triggerCount, triggerTowMsR, triggerTowSubMsR, + triggerAccEst); - char nmeaMessage[82]; // Max NMEA sentence length is 82 - createNMEASentence(CUSTOM_NMEA_TYPE_EVENT, nmeaMessage, sizeof(nmeaMessage), - eventData); // textID, buffer, sizeOfBuffer, text + char nmeaMessage[82]; // Max NMEA sentence length is 82 + createNMEASentence(CUSTOM_NMEA_TYPE_EVENT, nmeaMessage, sizeof(nmeaMessage), + eventData); // textID, buffer, sizeOfBuffer, text - if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) - { - markSemaphore(FUNCTION_EVENT); + if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) + { + markSemaphore(FUNCTION_EVENT); + + if (logFile) + { + logFile->sync(); // Sync any partially written data + logFile->println(nmeaMessage); + logFile->sync(); + } - if (logFile) + xSemaphoreGive(sdCardSemaphore); + newEventToRecord = false; + } + else { - logFile->sync(); // Sync any partially written data - logFile->println(nmeaMessage); - logFile->sync(); + char semaphoreHolder[50]; + getSemaphoreFunction(semaphoreHolder); + + // While a retry does occur during the next loop, it is possible to lose + // trigger events if they occur too rapidly or if the log file is closed + // before the trigger event is written! + log_w("sdCardSemaphore failed to yield, held by %s, RTK_Everywhere.ino line %d", semaphoreHolder, + __LINE__); } - - xSemaphoreGive(sdCardSemaphore); - newEventToRecord = false; - } - else - { - char semaphoreHolder[50]; - getSemaphoreFunction(semaphoreHolder); - - // While a retry does occur during the next loop, it is possible to lose - // trigger events if they occur too rapidly or if the log file is closed - // before the trigger event is written! - log_w("sdCardSemaphore failed to yield, held by %s, RTK_Everywhere.ino line %d", semaphoreHolder, - __LINE__); } } @@ -1538,46 +1541,49 @@ void logUpdate() ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) { lastARPLog = millis(); - newARPAvailable = false; - - double x = ARPECEFX; - x /= 10000.0; // Convert to m - double y = ARPECEFY; - y /= 10000.0; // Convert to m - double z = ARPECEFZ; - z /= 10000.0; // Convert to m - double h = ARPECEFH; - h /= 10000.0; // Convert to m - char ARPData[82]; // Max NMEA sentence length is 82 - snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); - - if (settings.enablePrintLogFileStatus) - systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); - - char nmeaMessage[82]; // Max NMEA sentence length is 82 - createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), - ARPData); // textID, buffer, sizeOfBuffer, text - - if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) + newARPAvailable = false; // Clear flag. It doesn't matter if the ARP cannot be logged + + if (webServerIsRunning() == false) { - markSemaphore(FUNCTION_ARPWRITE); + double x = ARPECEFX; + x /= 10000.0; // Convert to m + double y = ARPECEFY; + y /= 10000.0; // Convert to m + double z = ARPECEFZ; + z /= 10000.0; // Convert to m + double h = ARPECEFH; + h /= 10000.0; // Convert to m + char ARPData[82]; // Max NMEA sentence length is 82 + snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); - if (logFile) + if (settings.enablePrintLogFileStatus) + systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); + + char nmeaMessage[82]; // Max NMEA sentence length is 82 + createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), + ARPData); // textID, buffer, sizeOfBuffer, text + + if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) { - // See #695. This seems to cause the rare ntripClient->read "(pbuf_free: p->ref > 0)" error - logFile->sync(); // Sync any partially written data - logFile->println(nmeaMessage); - logFile->sync(); - } + markSemaphore(FUNCTION_ARPWRITE); - xSemaphoreGive(sdCardSemaphore); - } - else - { - char semaphoreHolder[50]; - getSemaphoreFunction(semaphoreHolder); - log_w("sdCardSemaphore failed to yield, held by %s, RTK_Everywhere.ino line %d", semaphoreHolder, - __LINE__); + if (logFile) + { + // See #695. This seems to cause the rare ntripClient->read "(pbuf_free: p->ref > 0)" error + logFile->sync(); // Sync any partially written data + logFile->println(nmeaMessage); + logFile->sync(); + } + + xSemaphoreGive(sdCardSemaphore); + } + else + { + char semaphoreHolder[50]; + getSemaphoreFunction(semaphoreHolder); + log_w("sdCardSemaphore failed to yield, held by %s, RTK_Everywhere.ino line %d", semaphoreHolder, + __LINE__); + } } } diff --git a/Firmware/RTK_Everywhere/menuMessages.ino b/Firmware/RTK_Everywhere/menuMessages.ino index cedb206d8..0b13be456 100644 --- a/Firmware/RTK_Everywhere/menuMessages.ino +++ b/Firmware/RTK_Everywhere/menuMessages.ino @@ -378,6 +378,8 @@ bool beginLogging(const char *customFileName) currentDate); // textID, buffer, sizeOfBuffer, text logFile->println(nmeaMessage); + logFile->sync(); // Sync any partially written data + if (reuseLastLog == true) { systemPrintln("Appending last available log"); From 90081259621e6309e146fbdd4a90f88727ee89d7 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 17:15:54 +0100 Subject: [PATCH 12/39] For SD, write all available data to prevent ARP Write gatecrashing regular logged data --- Firmware/RTK_Everywhere/Tasks.ino | 61 +++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index 1e14f56e4..6e78137a3 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -1063,7 +1063,7 @@ void handleGnssDataTask(void *e) uint32_t deltaMillis; int32_t freeSpace; static uint32_t maxMillis[RBC_MAX]; - uint32_t startMillis; + unsigned long startMillis; int32_t usedSpace; // Start notification @@ -1293,10 +1293,6 @@ void handleGnssDataTask(void *e) { markSemaphore(FUNCTION_WRITESD); - // Reduce bytes to record if we have more then the end of the buffer - if ((sdRingBufferTail + bytesToSend) > settings.gnssHandlerBufferSize) - bytesToSend = settings.gnssHandlerBufferSize - sdRingBufferTail; - if (settings.enablePrintSDBuffers && (!inMainMenu)) { int bufferAvailable = serialGNSS->available(); @@ -1309,12 +1305,46 @@ void handleGnssDataTask(void *e) bufferOverruns); } - // Write the data to the file - long startTime = millis(); + // For the SD card, we need to write everything we've got + // to prevent the ARP Write and Events from gatecrashing... + + int32_t sendTheseBytes = bytesToSend; + + // Reduce bytes to record if we have more then the end of the buffer + if ((sdRingBufferTail + sendTheseBytes) > settings.gnssHandlerBufferSize) + sendTheseBytes = settings.gnssHandlerBufferSize - sdRingBufferTail; + startMillis = millis(); - bytesToSend = logFile->write(&ringBuffer[sdRingBufferTail], bytesToSend); - if (PERIODIC_DISPLAY(PD_SD_LOG_WRITE) && (bytesToSend > 0)) + // Write the data to the file + int32_t bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); + + if (bytesSent != sendTheseBytes) + systemPrintf("SD write mismatch: wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + + // Account for the sent data or dropped + sdRingBufferTail += sendTheseBytes; // Use sendTheseBytes, not bytesSent. It's the best we can do + if (sdRingBufferTail >= settings.gnssHandlerBufferSize) + sdRingBufferTail -= settings.gnssHandlerBufferSize; + + // If we have more data to write + if (bytesToSend > sendTheseBytes) + { + sendTheseBytes = bytesToSend - sendTheseBytes; + + bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); + + if (bytesSent != sendTheseBytes) + systemPrintf("SD write mismatch: wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + + // Account for the sent data or dropped + sdRingBufferTail += sendTheseBytes; // Use sendTheseBytes, not bytesSent. It's the best we can do + if (sdRingBufferTail >= settings.gnssHandlerBufferSize) // Should be redundant + sdRingBufferTail -= settings.gnssHandlerBufferSize; + + } + + if (PERIODIC_DISPLAY(PD_SD_LOG_WRITE) && (bytesSent > 0)) { PERIODIC_CLEAR(PD_SD_LOG_WRITE); systemPrintf("SD %d bytes written to log file\r\n", bytesToSend); @@ -1341,25 +1371,16 @@ void handleGnssDataTask(void *e) deltaMillis = millis() - startMillis; if (maxMillis[RBC_SD_CARD] < deltaMillis) maxMillis[RBC_SD_CARD] = deltaMillis; - long endTime = millis(); if (settings.enablePrintBufferOverrun) { - if (endTime - startTime > 150) + if (deltaMillis > 150) systemPrintf("Long Write! Time: %ld ms / Location: %ld / Recorded %d bytes / " "spaceRemaining %d bytes\r\n", - endTime - startTime, logFileSize, bytesToSend, combinedSpaceRemaining); + deltaMillis, logFileSize, bytesToSend, combinedSpaceRemaining); } xSemaphoreGive(sdCardSemaphore); - - // Account for the sent data or dropped - if (bytesToSend > 0) - { - sdRingBufferTail += bytesToSend; - if (sdRingBufferTail >= settings.gnssHandlerBufferSize) - sdRingBufferTail -= settings.gnssHandlerBufferSize; - } } // End sdCardSemaphore else { From 3e1a59391412220b8c0a01a3f15ecf0d10621cc2 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 17:31:54 +0100 Subject: [PATCH 13/39] Update required library versions --- .github/workflows/compile-rtk-everywhere.yml | 4 ++-- .github/workflows/non-release-build.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/compile-rtk-everywhere.yml b/.github/workflows/compile-rtk-everywhere.yml index 6abbfafff..2de0932a9 100644 --- a/.github/workflows/compile-rtk-everywhere.yml +++ b/.github/workflows/compile-rtk-everywhere.yml @@ -89,12 +89,12 @@ jobs: "SparkFun u-blox GNSS v3"@3.1.8 "SparkFun Qwiic OLED Arduino Library"@1.0.13 SSLClientESP32@2.0.0 - "SparkFun Extensible Message Parser"@1.0.2 + "SparkFun Extensible Message Parser"@1.0.4 "SparkFun BQ40Z50 Battery Manager Arduino Library"@1.0.0 "ArduinoMqttClient"@0.1.8 "SparkFun u-blox PointPerfect Library"@1.11.4 "SparkFun IM19 IMU Arduino Library"@1.0.1 - "SparkFun UM980 Triband RTK GNSS Arduino Library"@1.0.4 + "SparkFun UM980 Triband RTK GNSS Arduino Library"@1.0.5 "SparkFun LG290P Quadband RTK GNSS Arduino Library"@1.0.8 "SparkFun I2C Expander Arduino Library"@1.0.1 diff --git a/.github/workflows/non-release-build.yml b/.github/workflows/non-release-build.yml index 7756986d8..d9db34303 100644 --- a/.github/workflows/non-release-build.yml +++ b/.github/workflows/non-release-build.yml @@ -88,13 +88,13 @@ jobs: "SparkFun u-blox GNSS v3"@3.1.8 "SparkFun Qwiic OLED Arduino Library"@1.0.13 SSLClientESP32@2.0.0 - "SparkFun Extensible Message Parser"@1.0.2 + "SparkFun Extensible Message Parser"@1.0.4 "SparkFun BQ40Z50 Battery Manager Arduino Library"@1.0.0 "ArduinoMqttClient"@0.1.8 "SparkFun u-blox PointPerfect Library"@1.11.4 "SparkFun IM19 IMU Arduino Library"@1.0.1 - "SparkFun UM980 Triband RTK GNSS Arduino Library"@1.0.4 - "SparkFun LG290P Quadband RTK GNSS Arduino Library"@1.0.7 + "SparkFun UM980 Triband RTK GNSS Arduino Library"@1.0.5 + "SparkFun LG290P Quadband RTK GNSS Arduino Library"@1.0.8 "SparkFun I2C Expander Arduino Library"@1.0.1 - name: Patch libmbedtls From 794fe34840153968e780f49982a7ca45ef7bac87 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 17 Jul 2025 17:51:58 +0100 Subject: [PATCH 14/39] Change default ARP logging interval to 60s --- Firmware/RTK_Everywhere/AP-Config/index.html | 2 +- Firmware/RTK_Everywhere/AP-Config/src/main.js | 4 ++-- Firmware/RTK_Everywhere/menuMessages.ino | 2 +- Firmware/RTK_Everywhere/settings.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Firmware/RTK_Everywhere/AP-Config/index.html b/Firmware/RTK_Everywhere/AP-Config/index.html index 4a024b7d9..778362620 100644 --- a/Firmware/RTK_Everywhere/AP-Config/index.html +++ b/Firmware/RTK_Everywhere/AP-Config/index.html @@ -2112,7 +2112,7 @@ diff --git a/Firmware/RTK_Everywhere/AP-Config/src/main.js b/Firmware/RTK_Everywhere/AP-Config/src/main.js index f1852ac82..ef218613f 100644 --- a/Firmware/RTK_Everywhere/AP-Config/src/main.js +++ b/Firmware/RTK_Everywhere/AP-Config/src/main.js @@ -1012,10 +1012,10 @@ function validateFields() { } if (ge("enableARPLogging").checked == true) { - checkElementValue("ARPLoggingInterval", 1, 600, "Must be 1 to 600", "collapseSystemConfig"); + checkElementValue("ARPLoggingInterval", 15, 600, "Must be 15 to 600", "collapseSystemConfig"); } else { - clearElement("ARPLoggingInterval", 10); + clearElement("ARPLoggingInterval", 60); } if (ge("enableAutoFirmwareUpdate").checked == true) { diff --git a/Firmware/RTK_Everywhere/menuMessages.ino b/Firmware/RTK_Everywhere/menuMessages.ino index 0b13be456..f08186292 100644 --- a/Firmware/RTK_Everywhere/menuMessages.ino +++ b/Firmware/RTK_Everywhere/menuMessages.ino @@ -120,7 +120,7 @@ void menuLog() else if (incoming == 6 && settings.enableLogging == true && settings.enableARPLogging == true) { // Arbitrary 10 minute limit - getNewSetting("Enter the ARP logging interval in seconds", 0, 60 * 10, &settings.ARPLoggingInterval_s); + getNewSetting("Enter the ARP logging interval in seconds", 15, 60 * 10, &settings.ARPLoggingInterval_s); } else if (incoming == 7) { diff --git a/Firmware/RTK_Everywhere/settings.h b/Firmware/RTK_Everywhere/settings.h index e0e956f08..a756a0b67 100644 --- a/Firmware/RTK_Everywhere/settings.h +++ b/Firmware/RTK_Everywhere/settings.h @@ -639,7 +639,7 @@ struct Settings // Antenna int16_t antennaHeight_mm = 1800; // Aka Pole length float antennaPhaseCenter_mm = 0.0; // Aka ARP - uint16_t ARPLoggingInterval_s = 10; // Log the ARP every 10 seconds - if available + uint16_t ARPLoggingInterval_s = 60; // Log the ARP every 60 seconds - if available bool enableARPLogging = false; // Log the Antenna Reference Position from RTCM 1005/1006 - if available // Base operation From d73180c2bd4788253f13939359a8e5be022e9701 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Fri, 18 Jul 2025 13:02:37 +0100 Subject: [PATCH 15/39] Return default ARP logging interval to 10s --- Firmware/RTK_Everywhere/AP-Config/index.html | 2 +- Firmware/RTK_Everywhere/AP-Config/src/main.js | 4 ++-- Firmware/RTK_Everywhere/menuMessages.ino | 2 +- Firmware/RTK_Everywhere/settings.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Firmware/RTK_Everywhere/AP-Config/index.html b/Firmware/RTK_Everywhere/AP-Config/index.html index 778362620..42fa8000d 100644 --- a/Firmware/RTK_Everywhere/AP-Config/index.html +++ b/Firmware/RTK_Everywhere/AP-Config/index.html @@ -2112,7 +2112,7 @@ diff --git a/Firmware/RTK_Everywhere/AP-Config/src/main.js b/Firmware/RTK_Everywhere/AP-Config/src/main.js index ef218613f..f1852ac82 100644 --- a/Firmware/RTK_Everywhere/AP-Config/src/main.js +++ b/Firmware/RTK_Everywhere/AP-Config/src/main.js @@ -1012,10 +1012,10 @@ function validateFields() { } if (ge("enableARPLogging").checked == true) { - checkElementValue("ARPLoggingInterval", 15, 600, "Must be 15 to 600", "collapseSystemConfig"); + checkElementValue("ARPLoggingInterval", 1, 600, "Must be 1 to 600", "collapseSystemConfig"); } else { - clearElement("ARPLoggingInterval", 60); + clearElement("ARPLoggingInterval", 10); } if (ge("enableAutoFirmwareUpdate").checked == true) { diff --git a/Firmware/RTK_Everywhere/menuMessages.ino b/Firmware/RTK_Everywhere/menuMessages.ino index f08186292..5898ba83d 100644 --- a/Firmware/RTK_Everywhere/menuMessages.ino +++ b/Firmware/RTK_Everywhere/menuMessages.ino @@ -120,7 +120,7 @@ void menuLog() else if (incoming == 6 && settings.enableLogging == true && settings.enableARPLogging == true) { // Arbitrary 10 minute limit - getNewSetting("Enter the ARP logging interval in seconds", 15, 60 * 10, &settings.ARPLoggingInterval_s); + getNewSetting("Enter the ARP logging interval in seconds", 1, 60 * 10, &settings.ARPLoggingInterval_s); } else if (incoming == 7) { diff --git a/Firmware/RTK_Everywhere/settings.h b/Firmware/RTK_Everywhere/settings.h index a756a0b67..e0e956f08 100644 --- a/Firmware/RTK_Everywhere/settings.h +++ b/Firmware/RTK_Everywhere/settings.h @@ -639,7 +639,7 @@ struct Settings // Antenna int16_t antennaHeight_mm = 1800; // Aka Pole length float antennaPhaseCenter_mm = 0.0; // Aka ARP - uint16_t ARPLoggingInterval_s = 60; // Log the ARP every 60 seconds - if available + uint16_t ARPLoggingInterval_s = 10; // Log the ARP every 10 seconds - if available bool enableARPLogging = false; // Log the Antenna Reference Position from RTCM 1005/1006 - if available // Base operation From 5816dc6cefa46d5438e9dd4e4430f127a8e7412c Mon Sep 17 00:00:00 2001 From: PaulZC Date: Fri, 18 Jul 2025 13:03:35 +0100 Subject: [PATCH 16/39] Initialize newARPAvailable --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index bfb6892eb..04ec134b8 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -461,7 +461,7 @@ uint32_t timTpEpoch; uint32_t timTpMicros; unsigned long lastARPLog; // Time of the last ARP log event -bool newARPAvailable; +bool newARPAvailable = false; int64_t ARPECEFX; // ARP ECEF is 38-bit signed int64_t ARPECEFY; int64_t ARPECEFZ; @@ -1569,7 +1569,6 @@ void logUpdate() if (logFile) { - // See #695. This seems to cause the rare ntripClient->read "(pbuf_free: p->ref > 0)" error logFile->sync(); // Sync any partially written data logFile->println(nmeaMessage); logFile->sync(); From 4a2e73297837f8b911e39712fb47c8df0bd781c6 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Fri, 18 Jul 2025 14:13:50 +0100 Subject: [PATCH 17/39] Fix that typo! --- Firmware/RTK_Everywhere/Tasks.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index 6e78137a3..edd8a339e 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -2256,7 +2256,7 @@ void processRTCMMessage(SEMP_PARSE_STATE *parse, uint16_t type) newARPAvailable = true; } - if (sempRtcmGetMessageNumber(parse) == 1005) + if (sempRtcmGetMessageNumber(parse) == 1006) { ARPECEFX = sempRtcmGetSignedBits(parse, 34, 38); ARPECEFY = sempRtcmGetSignedBits(parse, 74, 38); From 248ba0316b1dce1dbbeb4e86a414e80416a00ec2 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Fri, 18 Jul 2025 15:08:44 +0100 Subject: [PATCH 18/39] Add ntripClient timeouts: * Use `NTRIP_CLIENT_RESPONSE_TIMEOUT` for the initial `connect` * Use `NTRIP_CLIENT_RECEIVE_DATA_TIMEOUT` for the ongoing connection --- Firmware/RTK_Everywhere/NtripClient.ino | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index 58f321954..4f09e9ef1 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -228,7 +228,7 @@ bool ntripClientConnect() systemPrintf("NTRIP Client connecting to %s:%d\r\n", settings.ntripClient_CasterHost, settings.ntripClient_CasterPort); - int connectResponse = ntripClient->connect(settings.ntripClient_CasterHost, settings.ntripClient_CasterPort); + int connectResponse = ntripClient->connect(settings.ntripClient_CasterHost, settings.ntripClient_CasterPort, NTRIP_CLIENT_RESPONSE_TIMEOUT); if (connectResponse < 1) { @@ -804,6 +804,7 @@ void ntripClientUpdate() // We don't use a task because we use I2C hardware (and don't have a semaphore). online.ntripClient = true; ntripClientStartTime = millis(); + ntripClient->setConnectionTimeout(NTRIP_CLIENT_RECEIVE_DATA_TIMEOUT); ntripClientSetState(NTRIP_CLIENT_CONNECTED); } } From b2c6be33a70f4008507f41c9c8a2df67ccfba4e8 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Fri, 18 Jul 2025 16:50:29 +0100 Subject: [PATCH 19/39] Switch to NetworkClient readBytes --- Firmware/RTK_Everywhere/NtripClient.ino | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index 4f09e9ef1..38af39e0e 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -871,7 +871,8 @@ void ntripClientUpdate() } // Check for timeout receiving NTRIP data - if (ntripClientReceiveDataAvailable() == 0) + int avail = ntripClientReceiveDataAvailable(); + if (avail <= 0) { // Don't fail during retransmission attempts if ((millis() - ntripClientTimer) > NTRIP_CLIENT_RECEIVE_DATA_TIMEOUT) @@ -904,7 +905,7 @@ void ntripClientUpdate() // See #695 // Rare LWIP ASSERT "(pbuf_free: p->ref > 0)" errors have been seen here // - which trigger an esp_system_abort - int rtcmCount = ntripClient->read(rtcmData, RTCM_DATA_SIZE); + int rtcmCount = ntripClient->readBytes(rtcmData, avail < RTCM_DATA_SIZE ? avail : RTCM_DATA_SIZE); if (rtcmCount > 0) { // Restart the NTRIP receive data timer From 8207da4b1dbf89c83182ef81148a8a4456a6e112 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Tue, 22 Jul 2025 14:27:59 +0100 Subject: [PATCH 20/39] Update comment --- Firmware/RTK_Everywhere/NtripClient.ino | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index 38af39e0e..f00ca3b74 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -904,7 +904,8 @@ void ntripClientUpdate() // See #695 // Rare LWIP ASSERT "(pbuf_free: p->ref > 0)" errors have been seen here - // - which trigger an esp_system_abort + // - which trigger an esp_system_abort (unless CONFIG_LWIP_ESP_LWIP_ASSERT is disabled) + // readBytes seems to perform better and cause fewer crashes than ->read int rtcmCount = ntripClient->readBytes(rtcmData, avail < RTCM_DATA_SIZE ? avail : RTCM_DATA_SIZE); if (rtcmCount > 0) { From 971899b62e16eda731286dd5a6da3cf2324b168c Mon Sep 17 00:00:00 2001 From: PaulZC Date: Tue, 22 Jul 2025 16:48:13 +0100 Subject: [PATCH 21/39] Correctly handle zero maxLogTime_minutes and maxLogLength_minutes --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 21 ++++++++++++++++++--- Firmware/RTK_Everywhere/Tasks.ino | 2 +- Firmware/RTK_Everywhere/settings.h | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 04ec134b8..62ecc78d7 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1445,6 +1445,22 @@ void loopDelay() delay(10); } +bool logTimeExceeded() +{ + if (settings.maxLogTime_minutes == 0) + return false; + + return ((systemTime_minutes - startLogTime_minutes) >= settings.maxLogTime_minutes); +} + +bool logLengthExceeded() +{ + if (settings.maxLogLength_minutes == 0) + return false; + + return ((systemTime_minutes - startCurrentLogTime_minutes) >= settings.maxLogLength_minutes); +} + // Create or close files as needed (startup or as the user changes settings) // Push new data to log as needed void logUpdate() @@ -1482,8 +1498,7 @@ void logUpdate() // Close down file endSD(false, true); } - else if (online.logging == true && settings.enableLogging == true && - (systemTime_minutes - startCurrentLogTime_minutes) >= settings.maxLogLength_minutes) + else if (online.logging == true && settings.enableLogging == true && logLengthExceeded()) { endSD(false, true); // Close down file. A new one will be created at the next calling of updateLogs(). } @@ -1597,7 +1612,7 @@ void logUpdate() { systemPrintf("Log file size: %lld", logFileSize); - if ((systemTime_minutes - startLogTime_minutes) < settings.maxLogTime_minutes) + if (!logTimeExceeded()) { // Calculate generation and write speeds every 5 seconds uint64_t fileSizeDelta = logFileSize - lastLogSize; diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index edd8a339e..da8943477 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -1268,7 +1268,7 @@ void handleGnssDataTask(void *e) //---------------------------------------------------------------------- // Determine if the SD card is enabled for logging - connected = online.logging && ((systemTime_minutes - startLogTime_minutes) < settings.maxLogTime_minutes); + connected = online.logging && (!logTimeExceeded()); // Block logging during Web Config to avoid SD collisions // See issue: https://github.com/sparkfun/SparkFun_RTK_Everywhere_Firmware/issues/693 diff --git a/Firmware/RTK_Everywhere/settings.h b/Firmware/RTK_Everywhere/settings.h index e0e956f08..a3388bc8e 100644 --- a/Firmware/RTK_Everywhere/settings.h +++ b/Firmware/RTK_Everywhere/settings.h @@ -730,7 +730,7 @@ struct Settings bool enablePrintLogFileMessages = false; bool enablePrintLogFileStatus = true; int maxLogLength_minutes = 60 * 24; // Default to 24 hours - int maxLogTime_minutes = 60 * 24; // Default to 24 hours + int maxLogTime_minutes = 60 * 24; // Default to 24 hours // MQTT bool debugMqttClientData = false; // Debug the MQTT SPARTAN data flow From 51526b75045089384c17116120c5a3a20e43697d Mon Sep 17 00:00:00 2001 From: PaulZC Date: Tue, 22 Jul 2025 17:33:25 +0100 Subject: [PATCH 22/39] Add comments --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 62ecc78d7..c57bb4eb2 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1445,7 +1445,7 @@ void loopDelay() delay(10); } -bool logTimeExceeded() +bool logTimeExceeded() // Limit total logging time to maxLogTime_minutes { if (settings.maxLogTime_minutes == 0) return false; @@ -1453,7 +1453,7 @@ bool logTimeExceeded() return ((systemTime_minutes - startLogTime_minutes) >= settings.maxLogTime_minutes); } -bool logLengthExceeded() +bool logLengthExceeded() // Limit individual files to maxLogLength_minutes { if (settings.maxLogLength_minutes == 0) return false; From 8ab22c583c93ed423ccbc23ebb7341a802c1d6cd Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 23 Jul 2025 09:41:08 +0100 Subject: [PATCH 23/39] Add inMainMenu check for NTRIP GGA printing --- Firmware/RTK_Everywhere/GNSS.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/GNSS.ino b/Firmware/RTK_Everywhere/GNSS.ino index d4cc47ec9..e3776b075 100644 --- a/Firmware/RTK_Everywhere/GNSS.ino +++ b/Firmware/RTK_Everywhere/GNSS.ino @@ -85,7 +85,7 @@ void pushGPGGA(char *ggaData) { lastGGAPush = millis(); - if (settings.debugNtripClientRtcm || PERIODIC_DISPLAY(PD_NTRIP_CLIENT_GGA)) + if ((settings.debugNtripClientRtcm || PERIODIC_DISPLAY(PD_NTRIP_CLIENT_GGA)) && !inMainMenu) { PERIODIC_CLEAR(PD_NTRIP_CLIENT_GGA); systemPrintf("NTRIP Client pushing GGA to server: %s", (const char *)ggaData); From a62eb75dd4f77e31371a018c71d1434d04ddf6f0 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 23 Jul 2025 10:56:34 +0100 Subject: [PATCH 24/39] Logging improvements: Avoid endSD when opening a new log file due to logLengthExceeded Avoid beginLogging if log time is exceeded --- Firmware/RTK_Everywhere/Begin.ino | 2 +- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Firmware/RTK_Everywhere/Begin.ino b/Firmware/RTK_Everywhere/Begin.ino index 3f042ed21..2798e62c3 100644 --- a/Firmware/RTK_Everywhere/Begin.ino +++ b/Firmware/RTK_Everywhere/Begin.ino @@ -843,7 +843,7 @@ void beginSD() gotSemaphore = false; - while (settings.enableSD == true) + while (settings.enableSD == true) // Note: settings.enableSD is never set to false { // Setup SD card access semaphore if (sdCardSemaphore == NULL) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index c57bb4eb2..5f0365b5b 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -315,6 +315,8 @@ unsigned long syncRTCInterval = 1000; // To begin, sync RTC every second. Interv void beginSPI(bool force = false); // Header +// Important note: the firmware currently requires SdFat v2.1.1 +// sd->begin will crash second time around with ~v2.2.3 #include "SdFat.h" //http://librarymanager/All#sdfat_exfat by Bill Greiman. SdFat *sd; @@ -1482,7 +1484,7 @@ void logUpdate() if (outOfSDSpace == true) return; // We can't log if we are out of SD space - if (online.logging == false && settings.enableLogging == true && blockLogging == false) + if (online.logging == false && settings.enableLogging == true && blockLogging == false && !logTimeExceeded()) { if (beginLogging() == false) { @@ -1498,9 +1500,16 @@ void logUpdate() // Close down file endSD(false, true); } - else if (online.logging == true && settings.enableLogging == true && logLengthExceeded()) + else if (online.logging == true && settings.enableLogging == true && (logLengthExceeded() || logTimeExceeded())) { - endSD(false, true); // Close down file. A new one will be created at the next calling of updateLogs(). + if (logTimeExceeded()) + endSD(false, true); // Close down SD. + else + { + endLogging(false, true); //(gotSemaphore, releaseSemaphore) Close file. Reset parser stats. + beginLogging(); // Create new file based on current RTC. + setLoggingType(); // Determine if we are standard, PPP, or custom. Changes logging icon accordingly. + } } if (online.logging == true) From f0c80765da1f9167e8bb3e117c8bd83a4d223f21 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 23 Jul 2025 11:54:43 +0100 Subject: [PATCH 25/39] Better use of startLogTime_minutes --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 8 ++++++-- Firmware/RTK_Everywhere/menuMessages.ino | 13 ++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 5f0365b5b..27eef7eb2 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1449,7 +1449,7 @@ void loopDelay() bool logTimeExceeded() // Limit total logging time to maxLogTime_minutes { - if (settings.maxLogTime_minutes == 0) + if (settings.maxLogTime_minutes == 0) // No limit if maxLogTime_minutes is zero return false; return ((systemTime_minutes - startLogTime_minutes) >= settings.maxLogTime_minutes); @@ -1457,7 +1457,7 @@ bool logTimeExceeded() // Limit total logging time to maxLogTime_minutes bool logLengthExceeded() // Limit individual files to maxLogLength_minutes { - if (settings.maxLogLength_minutes == 0) + if (settings.maxLogLength_minutes == 0) // No limit if maxLogLength_minutes is zero return false; return ((systemTime_minutes - startCurrentLogTime_minutes) >= settings.maxLogLength_minutes); @@ -1503,9 +1503,13 @@ void logUpdate() else if (online.logging == true && settings.enableLogging == true && (logLengthExceeded() || logTimeExceeded())) { if (logTimeExceeded()) + { + systemPrintln("Log file: maximum logging time reached"); endSD(false, true); // Close down SD. + } else { + systemPrintln("Log file: log length reached"); endLogging(false, true); //(gotSemaphore, releaseSemaphore) Close file. Reset parser stats. beginLogging(); // Create new file based on current RTC. setLoggingType(); // Determine if we are standard, PPP, or custom. Changes logging icon accordingly. diff --git a/Firmware/RTK_Everywhere/menuMessages.ino b/Firmware/RTK_Everywhere/menuMessages.ino index 5898ba83d..fe45c6d64 100644 --- a/Firmware/RTK_Everywhere/menuMessages.ino +++ b/Firmware/RTK_Everywhere/menuMessages.ino @@ -90,11 +90,10 @@ void menuLog() { settings.enableLogging ^= 1; - // Reset the maximum logging time when logging is disabled to ensure that - // the next time logging is enabled that the maximum amount of data can be - // captured. - if (settings.enableLogging == false) - startLogTime_minutes = 0; + // Reset the start logging time when logging is enabled to ensure that + // data can be captured. + if (settings.enableLogging == true) + startLogTime_minutes = millis() / 1000L / 60; } else if (incoming == 2 && settings.enableLogging == true) { @@ -291,10 +290,6 @@ bool beginLogging(const char *customFileName) startCurrentLogTime_minutes = millis() / 1000L / 60; // Mark now as start of logging - // If it hasn't been done before, mark the initial start of logging for total run time - if (startLogTime_minutes == 0) - startLogTime_minutes = millis() / 1000L / 60; - // Add NMEA txt message with restart reason char rstReason[30]; switch (esp_reset_reason()) From f19cb6e9559999065f0905e45255a19f95bddf93 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 23 Jul 2025 15:36:18 +0100 Subject: [PATCH 26/39] Sanity check --- Firmware/RTK_Everywhere/Tasks.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index da8943477..1b6f31f5a 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -1285,7 +1285,7 @@ void handleGnssDataTask(void *e) bytesToSend = dataHead - sdRingBufferTail; if (bytesToSend < 0) bytesToSend += settings.gnssHandlerBufferSize; - if (bytesToSend > 0) + if ((bytesToSend > 0) && (logFile)) { // Attempt to gain access to the SD card, avoids collisions with file // writing from other functions like recordSystemSettingsToFile() From 82681f708b9741520652c38ab52d0a23433fdc1c Mon Sep 17 00:00:00 2001 From: PaulZC Date: Wed, 23 Jul 2025 15:36:51 +0100 Subject: [PATCH 27/39] Update lastUBXLogSyncTime when writing ARP and Events --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 27eef7eb2..def2e4fb0 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1545,6 +1545,7 @@ void logUpdate() logFile->sync(); // Sync any partially written data logFile->println(nmeaMessage); logFile->sync(); + lastUBXLogSyncTime = millis(); } xSemaphoreGive(sdCardSemaphore); @@ -1600,6 +1601,7 @@ void logUpdate() logFile->sync(); // Sync any partially written data logFile->println(nmeaMessage); logFile->sync(); + lastUBXLogSyncTime = millis(); } xSemaphoreGive(sdCardSemaphore); From e4026003d2f7ad315dd73a395a3b9ed1a9692622 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 24 Jul 2025 11:47:39 +0100 Subject: [PATCH 28/39] Move Event and ARP logging into handleGnssDataTask Improve the SD writing. Correctly report write failures Add missing USB discardRingBufferBytes --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 98 ---------- Firmware/RTK_Everywhere/Tasks.ino | 202 ++++++++++++++------- 2 files changed, 136 insertions(+), 164 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index def2e4fb0..7a763a6dc 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -1518,104 +1518,6 @@ void logUpdate() if (online.logging == true) { - // Record any pending trigger events - if (newEventToRecord == true) - { - if (webServerIsRunning() == false) // Don't try to access the SD if the web server is running - { - if (settings.enablePrintLogFileStatus) - systemPrintln("Log file: recording event"); - - // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of - // rising edge (ns), and accuracy estimate (ns) - char eventData[82]; // Max NMEA sentence length is 82 - snprintf(eventData, sizeof(eventData), "%d,%d,%d,%d", triggerCount, triggerTowMsR, triggerTowSubMsR, - triggerAccEst); - - char nmeaMessage[82]; // Max NMEA sentence length is 82 - createNMEASentence(CUSTOM_NMEA_TYPE_EVENT, nmeaMessage, sizeof(nmeaMessage), - eventData); // textID, buffer, sizeOfBuffer, text - - if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) - { - markSemaphore(FUNCTION_EVENT); - - if (logFile) - { - logFile->sync(); // Sync any partially written data - logFile->println(nmeaMessage); - logFile->sync(); - lastUBXLogSyncTime = millis(); - } - - xSemaphoreGive(sdCardSemaphore); - newEventToRecord = false; - } - else - { - char semaphoreHolder[50]; - getSemaphoreFunction(semaphoreHolder); - - // While a retry does occur during the next loop, it is possible to lose - // trigger events if they occur too rapidly or if the log file is closed - // before the trigger event is written! - log_w("sdCardSemaphore failed to yield, held by %s, RTK_Everywhere.ino line %d", semaphoreHolder, - __LINE__); - } - } - } - - // Record the Antenna Reference Position - if available - if (newARPAvailable == true && settings.enableARPLogging && - ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) - { - lastARPLog = millis(); - newARPAvailable = false; // Clear flag. It doesn't matter if the ARP cannot be logged - - if (webServerIsRunning() == false) - { - double x = ARPECEFX; - x /= 10000.0; // Convert to m - double y = ARPECEFY; - y /= 10000.0; // Convert to m - double z = ARPECEFZ; - z /= 10000.0; // Convert to m - double h = ARPECEFH; - h /= 10000.0; // Convert to m - char ARPData[82]; // Max NMEA sentence length is 82 - snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); - - if (settings.enablePrintLogFileStatus) - systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); - - char nmeaMessage[82]; // Max NMEA sentence length is 82 - createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), - ARPData); // textID, buffer, sizeOfBuffer, text - - if (xSemaphoreTake(sdCardSemaphore, fatSemaphore_shortWait_ms) == pdPASS) - { - markSemaphore(FUNCTION_ARPWRITE); - - if (logFile) - { - logFile->sync(); // Sync any partially written data - logFile->println(nmeaMessage); - logFile->sync(); - lastUBXLogSyncTime = millis(); - } - - xSemaphoreGive(sdCardSemaphore); - } - else - { - char semaphoreHolder[50]; - getSemaphoreFunction(semaphoreHolder); - log_w("sdCardSemaphore failed to yield, held by %s, RTK_Everywhere.ino line %d", semaphoreHolder, - __LINE__); - } - } - } - // Report file sizes to show recording is working if ((millis() - lastFileReport) > 5000) { diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index 1b6f31f5a..c72a1e141 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -958,8 +958,11 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) } // Discard the oldest data from the ring buffer + // Printing the slow consumer is not that useful as any consumer will be + // considered 'slow' if its data wraps over the end of the buffer and + // needs a second write to clear... if (consumer) - systemPrintf("Ring buffer full: discarding %d bytes, %s is slow\r\n", discardedBytes, consumer); + systemPrintf("Ring buffer full: discarding %d bytes, %s could be slow\r\n", discardedBytes, consumer); else systemPrintf("Ring buffer full: discarding %d bytes\r\n", discardedBytes); updateRingBufferTails(previousTail, rbOffsetArray[rbOffsetTail]); @@ -1009,10 +1012,11 @@ void updateRingBufferTails(RING_BUFFER_OFFSET previousTail, RING_BUFFER_OFFSET n { // Trim any long or medium tails discardRingBufferBytes(&btRingBufferTail, previousTail, newTail); - discardRingBufferBytes(&sdRingBufferTail, previousTail, newTail); tcpClientDiscardBytes(previousTail, newTail); tcpServerDiscardBytes(previousTail, newTail); udpServerDiscardBytes(previousTail, newTail); + discardRingBufferBytes(&sdRingBufferTail, previousTail, newTail); + discardRingBufferBytes(&usbRingBufferTail, previousTail, newTail); } // Remove previous messages from the ring buffer @@ -1077,6 +1081,7 @@ void handleGnssDataTask(void *e) tcpServerZeroTail(); udpServerZeroTail(); sdRingBufferTail = 0; + usbRingBufferTail = 0; // Run task until a request is raised task.handleGnssDataTaskStopRequest = false; @@ -1285,7 +1290,7 @@ void handleGnssDataTask(void *e) bytesToSend = dataHead - sdRingBufferTail; if (bytesToSend < 0) bytesToSend += settings.gnssHandlerBufferSize; - if ((bytesToSend > 0) && (logFile)) + if (bytesToSend > 0) { // Attempt to gain access to the SD card, avoids collisions with file // writing from other functions like recordSystemSettingsToFile() @@ -1293,92 +1298,157 @@ void handleGnssDataTask(void *e) { markSemaphore(FUNCTION_WRITESD); - if (settings.enablePrintSDBuffers && (!inMainMenu)) + do // Do the SD write in a do loop so we can break out if needed { - int bufferAvailable = serialGNSS->available(); + if (settings.enablePrintSDBuffers && (!inMainMenu)) + { + int bufferAvailable = serialGNSS->available(); - int availableUARTSpace = settings.uartReceiveBufferSize - bufferAvailable; + int availableUARTSpace = settings.uartReceiveBufferSize - bufferAvailable; - systemPrintf("SD Incoming Serial: %04d\tToRead: %04d\tMovedToBuffer: %04d\tavailableUARTSpace: " - "%04d\tavailableHandlerSpace: %04d\tToRecord: %04d\tRecorded: %04d\tBO: %d\r\n", - bufferAvailable, 0, 0, availableUARTSpace, availableHandlerSpace, bytesToSend, 0, - bufferOverruns); - } + systemPrintf("SD Incoming Serial: %04d\tToRead: %04d\tMovedToBuffer: %04d\tavailableUARTSpace: " + "%04d\tavailableHandlerSpace: %04d\tToRecord: %04d\tRecorded: %04d\tBO: %d\r\n", + bufferAvailable, 0, 0, availableUARTSpace, availableHandlerSpace, bytesToSend, 0, + bufferOverruns); + } - // For the SD card, we need to write everything we've got - // to prevent the ARP Write and Events from gatecrashing... - - int32_t sendTheseBytes = bytesToSend; + // For the SD card, we need to write everything we've got + // to prevent the ARP Write and Events from gatecrashing... + + int32_t sendTheseBytes = bytesToSend; - // Reduce bytes to record if we have more then the end of the buffer - if ((sdRingBufferTail + sendTheseBytes) > settings.gnssHandlerBufferSize) - sendTheseBytes = settings.gnssHandlerBufferSize - sdRingBufferTail; + // Reduce bytes to record if we have more then the end of the buffer + if ((sdRingBufferTail + sendTheseBytes) > settings.gnssHandlerBufferSize) + sendTheseBytes = settings.gnssHandlerBufferSize - sdRingBufferTail; - startMillis = millis(); + startMillis = millis(); - // Write the data to the file - int32_t bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); + // Write the data to the file + int32_t bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); - if (bytesSent != sendTheseBytes) - systemPrintf("SD write mismatch: wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + // Account for the sent data or dropped + sdRingBufferTail += bytesSent; + if (sdRingBufferTail >= settings.gnssHandlerBufferSize) + sdRingBufferTail -= settings.gnssHandlerBufferSize; - // Account for the sent data or dropped - sdRingBufferTail += sendTheseBytes; // Use sendTheseBytes, not bytesSent. It's the best we can do - if (sdRingBufferTail >= settings.gnssHandlerBufferSize) - sdRingBufferTail -= settings.gnssHandlerBufferSize; + if (bytesSent != sendTheseBytes) + { + systemPrintf("SD write mismatch (1): wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + break; // Exit the do loop + } - // If we have more data to write - if (bytesToSend > sendTheseBytes) - { - sendTheseBytes = bytesToSend - sendTheseBytes; + // If we have more data to write - and the first write was successful + if (bytesToSend > sendTheseBytes) + { + sendTheseBytes = bytesToSend - sendTheseBytes; - bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); + bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); - if (bytesSent != sendTheseBytes) - systemPrintf("SD write mismatch: wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + // Account for the sent data or dropped + sdRingBufferTail += bytesSent; + if (sdRingBufferTail >= settings.gnssHandlerBufferSize) // Should be redundant + sdRingBufferTail -= settings.gnssHandlerBufferSize; - // Account for the sent data or dropped - sdRingBufferTail += sendTheseBytes; // Use sendTheseBytes, not bytesSent. It's the best we can do - if (sdRingBufferTail >= settings.gnssHandlerBufferSize) // Should be redundant - sdRingBufferTail -= settings.gnssHandlerBufferSize; + if (bytesSent != sendTheseBytes) + { + systemPrintf("SD write mismatch (2): wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + break; // Exit the do loop + } + } - } + if (PERIODIC_DISPLAY(PD_SD_LOG_WRITE) && (bytesSent > 0)) + { + PERIODIC_CLEAR(PD_SD_LOG_WRITE); + systemPrintf("SD %d bytes written to log file\r\n", bytesToSend); + } - if (PERIODIC_DISPLAY(PD_SD_LOG_WRITE) && (bytesSent > 0)) - { - PERIODIC_CLEAR(PD_SD_LOG_WRITE); - systemPrintf("SD %d bytes written to log file\r\n", bytesToSend); - } + sdFreeSpace -= bytesToSend; // Update remaining space on SD - logFileSize = logFile->fileSize(); // Update file size + // Record any pending trigger events + if (newEventToRecord == true) + { + newEventToRecord = false; - sdFreeSpace -= bytesToSend; // Update remaining space on SD + if (settings.enablePrintLogFileStatus) + systemPrintln("Log file: recording event"); - // Force file sync every 60s - if (millis() - lastUBXLogSyncTime > 60000) - { - baseStatusLedBlink(); // Blink LED to indicate logging activity + // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of + // rising edge (ns), and accuracy estimate (ns) + char eventData[82]; // Max NMEA sentence length is 82 + snprintf(eventData, sizeof(eventData), "%d,%d,%d,%d", triggerCount, triggerTowMsR, triggerTowSubMsR, + triggerAccEst); - logFile->sync(); - sdUpdateFileAccessTimestamp(logFile); // Update the file access time & date + char nmeaMessage[82]; // Max NMEA sentence length is 82 + createNMEASentence(CUSTOM_NMEA_TYPE_EVENT, nmeaMessage, sizeof(nmeaMessage), + eventData); // textID, buffer, sizeOfBuffer, text - baseStatusLedBlink(); // Blink LED to indicate logging activity + logFile->write(nmeaMessage, strlen(nmeaMessage)); + const char *crlf = "\r\n"; + logFile->write(crlf, 2); - lastUBXLogSyncTime = millis(); - } + sdFreeSpace -= strlen(nmeaMessage) + 2; // Update remaining space on SD + } - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_SD_CARD] < deltaMillis) - maxMillis[RBC_SD_CARD] = deltaMillis; + // Record the Antenna Reference Position - if available + if (newARPAvailable == true && settings.enableARPLogging && + ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) + { + lastARPLog = millis(); + newARPAvailable = false; // Clear flag. It doesn't matter if the ARP cannot be logged + + double x = ARPECEFX; + x /= 10000.0; // Convert to m + double y = ARPECEFY; + y /= 10000.0; // Convert to m + double z = ARPECEFZ; + z /= 10000.0; // Convert to m + double h = ARPECEFH; + h /= 10000.0; // Convert to m + char ARPData[82]; // Max NMEA sentence length is 82 + snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); + + if (settings.enablePrintLogFileStatus) + systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); + + char nmeaMessage[82]; // Max NMEA sentence length is 82 + createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), + ARPData); // textID, buffer, sizeOfBuffer, text + + logFile->write(nmeaMessage, strlen(nmeaMessage)); + const char *crlf = "\r\n"; + logFile->write(crlf, 2); + + sdFreeSpace -= strlen(nmeaMessage) + 2; // Update remaining space on SD + } - if (settings.enablePrintBufferOverrun) - { - if (deltaMillis > 150) - systemPrintf("Long Write! Time: %ld ms / Location: %ld / Recorded %d bytes / " - "spaceRemaining %d bytes\r\n", - deltaMillis, logFileSize, bytesToSend, combinedSpaceRemaining); - } + logFileSize = logFile->fileSize(); // Update file size + + // Force file sync every 60s + if (millis() - lastUBXLogSyncTime > 60000) + { + baseStatusLedBlink(); // Blink LED to indicate logging activity + + logFile->sync(); + sdUpdateFileAccessTimestamp(logFile); // Update the file access time & date + + baseStatusLedBlink(); // Blink LED to indicate logging activity + + lastUBXLogSyncTime = millis(); + } + + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_SD_CARD] < deltaMillis) + maxMillis[RBC_SD_CARD] = deltaMillis; + + if (settings.enablePrintBufferOverrun) + { + if (deltaMillis > 150) + systemPrintf("Long Write! Time: %ld ms / Location: %ld / Recorded %d bytes / " + "spaceRemaining %d bytes\r\n", + deltaMillis, logFileSize, bytesToSend, combinedSpaceRemaining); + } + } while(0); xSemaphoreGive(sdCardSemaphore); } // End sdCardSemaphore From 979309df3412b9e16abd0faab3e02a481785029b Mon Sep 17 00:00:00 2001 From: PaulZC Date: Thu, 24 Jul 2025 15:52:41 +0100 Subject: [PATCH 29/39] Remove redundant RTCM parsing --- Firmware/RTK_Everywhere/MQTT_Client.ino | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Firmware/RTK_Everywhere/MQTT_Client.ino b/Firmware/RTK_Everywhere/MQTT_Client.ino index f3e7062fd..71081001c 100644 --- a/Firmware/RTK_Everywhere/MQTT_Client.ino +++ b/Firmware/RTK_Everywhere/MQTT_Client.ino @@ -514,7 +514,7 @@ int mqttClientProcessZedMessage(uint8_t *mqttData, uint16_t mqttCount, int bytes zed->updateCorrectionsSource(0); // Set SOURCE to 0 (IP) if needed gnss->pushRawData(mqttData, mqttCount); - sempParseNextBytes(rtcmParse, mqttData, mqttCount); // Parse the data for RTCM1005/1006 + // Corrections are SPARTN. No point in pushing them to rtcmParse bytesPushed += mqttCount; mqttClientDataReceived = true; @@ -543,7 +543,7 @@ int mqttClientProcessZedMessage(uint8_t *mqttData, uint16_t mqttCount, int bytes } gnss->pushRawData(mqttData, mqttCount); - sempParseNextBytes(rtcmParse, mqttData, mqttCount); // Parse the data for RTCM1005/1006 + // No point in pushing keys / MGA to rtcmParse bytesPushed += mqttCount; } #endif // COMPILE_ZED From 3af1b8fa6828d8f66caccd120523fc82da36e231 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Fri, 25 Jul 2025 11:11:00 +0100 Subject: [PATCH 30/39] ringBuffer improvements Prevent 'empty' (zero length) messages from filling up rbOffsetArray Update availableHandlerSpace with a single write Update dataHead in a single write Strictly, the ringBuffer should be controlled by a semaphore. updateRingBufferTails especially can caused badness if handleGnssDataTask executes at the same time --- Firmware/RTK_Everywhere/Tasks.ino | 72 +++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index c72a1e141..fd5f36116 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -786,16 +786,29 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) parse->length = 0; } + // If parse->length is zero, we should exit now. + // Previously, the code would continue past here and fill rbOffsetArray with 'empty' entries. + // E.g. RTCM1019/1042/1046 suppressed above + if (parse->length == 0) + return; + + // We need to be careful with availableHandlerSpace because it is possible for handleGnssDataTask + // to execute and update availableHandlerSpace while this code is executing... + // To see this happening, enable printing of the ring buffer offsets (s d 10) and + // the SD buffer sizes (s h 7). You will see the "SD Incoming Serial" from handleGnssDataTask + // gatecrash the "DH:" prints here. + // Strictly, we should have a sempahore controlling access to the ring buffer. + // Determine if this message will fit into the ring buffer + int32_t discardedBytes = 0; bytesToCopy = parse->length; - space = availableHandlerSpace; + space = availableHandlerSpace; // Take a copy of availableHandlerSpace here use = settings.gnssHandlerBufferSize - space; consumer = (char *)slowConsumer; - if ((bytesToCopy > space) && (!inMainMenu)) + if (bytesToCopy > space) // Paul removed the && (!inMainMenu)) check 7-25-25 { int32_t bufferedData; int32_t bytesToDiscard; - int32_t discardedBytes; int32_t listEnd; int32_t messageLength; int32_t previousTail; @@ -909,7 +922,6 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) // +-----------+ <-- rbOffsetHead // | | // - discardedBytes = 0; if (bufferedData < use) discardedBytes = use - bufferedData; @@ -961,31 +973,44 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) // Printing the slow consumer is not that useful as any consumer will be // considered 'slow' if its data wraps over the end of the buffer and // needs a second write to clear... - if (consumer) - systemPrintf("Ring buffer full: discarding %d bytes, %s could be slow\r\n", discardedBytes, consumer); - else - systemPrintf("Ring buffer full: discarding %d bytes\r\n", discardedBytes); + if (!inMainMenu) + { + if (consumer) + systemPrintf("Ring buffer full: discarding %d bytes, %s could be slow\r\n", discardedBytes, consumer); + else + systemPrintf("Ring buffer full: discarding %d bytes\r\n", discardedBytes); + } + + // Update the tails. Strictly this needs semaphore protection updateRingBufferTails(previousTail, rbOffsetArray[rbOffsetTail]); - availableHandlerSpace = availableHandlerSpace + discardedBytes; } + if (bytesToCopy > (space + discardedBytes - 1)) // Sanity check + systemPrintf("Ring buffer update error: bytesToCopy (%d) is > space (%d) + discardedBytes (%d) - 1\r\n", + bytesToCopy, space, discardedBytes); + // Add another message to the ring buffer // Account for this message - availableHandlerSpace = availableHandlerSpace - bytesToCopy; + // Diagnostic prints are provided by settings.enablePrintSDBuffers and the handleGnssDataTask + // availableHandlerSpace may have been updated by handleGnssDataTask + availableHandlerSpace = availableHandlerSpace + discardedBytes - bytesToCopy; + + // Copy dataHead so we can update with a single write + RING_BUFFER_OFFSET newDataHead = dataHead; // Fill the buffer to the end and then start at the beginning - if ((dataHead + bytesToCopy) > settings.gnssHandlerBufferSize) - bytesToCopy = settings.gnssHandlerBufferSize - dataHead; + if ((newDataHead + bytesToCopy) > settings.gnssHandlerBufferSize) + bytesToCopy = settings.gnssHandlerBufferSize - newDataHead; // Display the dataHead offset if (settings.enablePrintRingBufferOffsets && (!inMainMenu)) - systemPrintf("DH: %4d --> ", dataHead); + systemPrintf("DH: %4d --> ", newDataHead); // Copy the data into the ring buffer - memcpy(&ringBuffer[dataHead], parse->buffer, bytesToCopy); - dataHead = dataHead + bytesToCopy; - if (dataHead >= settings.gnssHandlerBufferSize) - dataHead = dataHead - settings.gnssHandlerBufferSize; + memcpy(&ringBuffer[newDataHead], parse->buffer, bytesToCopy); + newDataHead = newDataHead + bytesToCopy; + if (newDataHead >= settings.gnssHandlerBufferSize) + newDataHead = newDataHead - settings.gnssHandlerBufferSize; // Determine the remaining bytes remainingBytes = parse->length - bytesToCopy; @@ -993,18 +1018,21 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) { // Copy the remaining bytes into the beginning of the ring buffer memcpy(ringBuffer, &parse->buffer[bytesToCopy], remainingBytes); - dataHead = dataHead + remainingBytes; - if (dataHead >= settings.gnssHandlerBufferSize) - dataHead = dataHead - settings.gnssHandlerBufferSize; + newDataHead = newDataHead + remainingBytes; + if (newDataHead >= settings.gnssHandlerBufferSize) + newDataHead = newDataHead - settings.gnssHandlerBufferSize; } // Add the head offset to the offset array WRAP_OFFSET(rbOffsetHead, 1, rbOffsetEntries); - rbOffsetArray[rbOffsetHead] = dataHead; + rbOffsetArray[rbOffsetHead] = newDataHead; // Display the dataHead offset if (settings.enablePrintRingBufferOffsets && (!inMainMenu)) - systemPrintf("%4d\r\n", dataHead); + systemPrintf("%4d\r\n", newDataHead); + + // Update dataHead in a single write - handleGnssDataTask will use it as soon as it updates + dataHead = newDataHead; } // Remove previous messages from the ring buffer From f2efc25eb326af217483bea1a5adc7e5531e2af6 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sat, 26 Jul 2025 09:41:37 +0100 Subject: [PATCH 31/39] More improvements: Add getTimeStamp. Clean up timestamp printing Add ringBufferSemaphore --- Firmware/RTK_Everywhere/Begin.ino | 6 +- Firmware/RTK_Everywhere/NtripServer.ino | 9 +- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 6 + Firmware/RTK_Everywhere/SD.ino | 6 +- Firmware/RTK_Everywhere/States.ino | 10 +- Firmware/RTK_Everywhere/Tasks.ino | 1132 ++++++++++---------- Firmware/RTK_Everywhere/WebServer.ino | 10 +- Firmware/RTK_Everywhere/menuFirmware.ino | 10 +- Firmware/RTK_Everywhere/menuMessages.ino | 2 +- Firmware/RTK_Everywhere/menuSystem.ino | 2 +- Firmware/RTK_Everywhere/support.ino | 28 +- 11 files changed, 624 insertions(+), 597 deletions(-) diff --git a/Firmware/RTK_Everywhere/Begin.ino b/Firmware/RTK_Everywhere/Begin.ino index 2798e62c3..46e48a420 100644 --- a/Firmware/RTK_Everywhere/Begin.ino +++ b/Firmware/RTK_Everywhere/Begin.ino @@ -862,7 +862,7 @@ void beginSD() break; // Give up on loop // If an SD card is present, allow SdFat to take over - log_d("SD card detected"); + systemPrintf("SD card detected @ %s\r\n", getTimeStamp()); // Allocate the data structure that manages the microSD card if (!sd) @@ -930,7 +930,7 @@ void beginSD() sdCardSize = 0; outOfSDSpace = true; - systemPrintln("microSD: Online"); + systemPrintf("microSD: Online @ %s\r\n", getTimeStamp()); online.microSD = true; break; } @@ -951,7 +951,7 @@ void endSD(bool alreadyHaveSemaphore, bool releaseSemaphore) sd->end(); online.microSD = false; - systemPrintln("microSD: Offline"); + systemPrintf("microSD: Offline @ %s\r\n", getTimeStamp()); } // Free the caches for the microSD card diff --git a/Firmware/RTK_Everywhere/NtripServer.ino b/Firmware/RTK_Everywhere/NtripServer.ino index 85a5417e5..467ed4883 100644 --- a/Firmware/RTK_Everywhere/NtripServer.ino +++ b/Firmware/RTK_Everywhere/NtripServer.ino @@ -425,14 +425,7 @@ void ntripServerProcessRTCM(int serverIndex, uint8_t incoming) (!settings.enableRtcmMessageChecking) && (!inMainMenu) && ntripServer->bytesSent) { PERIODIC_CLEAR(PD_NTRIP_SERVER_DATA); - printTimeStamp(); - // 1 2 3 - // 123456789012345678901234567890 - // YYYY-mm-dd HH:MM:SS.xxxrn0 - struct tm timeinfo = rtc.getTimeStruct(); - char timestamp[30]; - strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", &timeinfo); - systemPrintf(" Tx%d RTCM: %s.%03ld, %d bytes sent\r\n", serverIndex, timestamp, rtc.getMillis(), + systemPrintf(" Tx%d RTCM: %s, %d bytes sent\r\n", serverIndex, getTimeStamp(), ntripServer->rtcmBytesSent); ntripServer->rtcmBytesSent = 0; } diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 7a763a6dc..c8a05894c 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -307,6 +307,7 @@ const int COMMON_COORDINATES_MAX_STATIONS = 50; // Record up to 50 ECEF and Geod #include //http://librarymanager/All#ESP32Time by FBiego ESP32Time rtc; unsigned long syncRTCInterval = 1000; // To begin, sync RTC every second. Interval can be increased once sync'd. +void printTimeStamp(bool always = false); // Header //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- // microSD Interface @@ -336,6 +337,11 @@ TickType_t loggingSemaphoreWait_ms = 10 / portTICK_PERIOD_MS; const TickType_t fatSemaphore_shortWait_ms = 10 / portTICK_PERIOD_MS; const TickType_t fatSemaphore_longWait_ms = 200 / portTICK_PERIOD_MS; +// ringBuffer semaphore - prevent processUart1Message (gnssReadTask) and handleGnssDataTask +// from gatecrashing each other. +SemaphoreHandle_t ringBufferSemaphore = NULL; +const char *ringBufferSemaphoreHolder = "None"; + // Display used/free space in menu and config page uint64_t sdCardSize; uint64_t sdFreeSpace; diff --git a/Firmware/RTK_Everywhere/SD.ino b/Firmware/RTK_Everywhere/SD.ino index e37d25f49..09d485d75 100644 --- a/Firmware/RTK_Everywhere/SD.ino +++ b/Firmware/RTK_Everywhere/SD.ino @@ -27,7 +27,7 @@ void sdUpdate() } else if (sdCardPresent() == true) // Poll card to see if a card is inserted { - systemPrintln("SD inserted"); + systemPrintf("SD inserted @ %s\r\n", getTimeStamp()); beginSD(); // Attempt to start SD } } @@ -35,7 +35,7 @@ void sdUpdate() if (online.logging == true && sdCardSize > 0 && sdFreeSpace < sdMinAvailableSpace) // Stop logging if we are below the min { - log_d("Logging stopped. SD full."); + systemPrintf("Logging stopped. SD full @ %s\r\n", getTimeStamp()); outOfSDSpace = true; endSD(false, true); //(alreadyHaveSemaphore, releaseSemaphore) Close down file. return; @@ -50,7 +50,7 @@ void sdUpdate() // Check if SD card is still present if (sdCardPresent() == false) { - systemPrintln("SD removed"); + systemPrintf("SD removed @ %s\r\n", getTimeStamp()); endSD(false, true); //(alreadyHaveSemaphore, releaseSemaphore) Close down SD. } } diff --git a/Firmware/RTK_Everywhere/States.ino b/Firmware/RTK_Everywhere/States.ino index 3154afb01..0cc059062 100644 --- a/Firmware/RTK_Everywhere/States.ino +++ b/Firmware/RTK_Everywhere/States.ino @@ -764,16 +764,8 @@ void changeState(SystemState newState) if (!online.rtc) systemPrintf("%s%s%s%s\r\n", asterisk, initialState, arrow, endingState); else - { // Timestamp the state change - // 1 2 - // 12345678901234567890123456 - // YYYY-mm-dd HH:MM:SS.xxxrn0 - struct tm timeinfo = rtc.getTimeStruct(); - char s[30]; - strftime(s, sizeof(s), "%Y-%m-%d %H:%M:%S", &timeinfo); - systemPrintf("%s%s%s%s, %s.%03ld\r\n", asterisk, initialState, arrow, endingState, s, rtc.getMillis()); - } + systemPrintf("%s%s%s%s, %s\r\n", asterisk, initialState, arrow, endingState, getTimeStamp()); } } diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index fd5f36116..bdf959c71 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -792,247 +792,265 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) if (parse->length == 0) return; - // We need to be careful with availableHandlerSpace because it is possible for handleGnssDataTask - // to execute and update availableHandlerSpace while this code is executing... - // To see this happening, enable printing of the ring buffer offsets (s d 10) and - // the SD buffer sizes (s h 7). You will see the "SD Incoming Serial" from handleGnssDataTask - // gatecrash the "DH:" prints here. - // Strictly, we should have a sempahore controlling access to the ring buffer. - - // Determine if this message will fit into the ring buffer - int32_t discardedBytes = 0; - bytesToCopy = parse->length; - space = availableHandlerSpace; // Take a copy of availableHandlerSpace here - use = settings.gnssHandlerBufferSize - space; - consumer = (char *)slowConsumer; - if (bytesToCopy > space) // Paul removed the && (!inMainMenu)) check 7-25-25 + // Use a semaphore to prevent handleGnssDataTask from gatecrashing + if (ringBufferSemaphore == NULL) + ringBufferSemaphore = xSemaphoreCreateMutex(); // Create the mutex + + // Take the semaphore. Long wait. handleGnssDataTask could block + // Enable printing of the ring buffer offsets (s d 10) and the SD buffer sizes (s h 7) + // to see this in action. No more gatecrashing! + if (xSemaphoreTake(ringBufferSemaphore, fatSemaphore_longWait_ms) == pdPASS) { - int32_t bufferedData; - int32_t bytesToDiscard; - int32_t listEnd; - int32_t messageLength; - int32_t previousTail; - int32_t rbOffsetTail; - - // Determine the tail of the ring buffer - previousTail = dataHead + space + 1; - if (previousTail >= settings.gnssHandlerBufferSize) - previousTail -= settings.gnssHandlerBufferSize; - - /* The rbOffsetArray holds the offsets into the ring buffer of the - * start of each of the parsed messages. A head (rbOffsetHead) and - * tail (rbOffsetTail) offsets are used for this array to insert and - * remove entries. Typically this task only manipulates the head as - * new messages are placed into the ring buffer. The handleGnssDataTask - * normally manipulates the tail as data is removed from the buffer. - * However this task will manipulate the tail under two conditions: - * - * 1. The ring buffer gets full and data must be discarded - * - * 2. The rbOffsetArray is too small to hold all of the message - * offsets for the data in the ring buffer. The array is full - * when (Head + 1) == Tail - * - * Notes: - * The rbOffsetArray is allocated along with the ring buffer in - * Begin.ino - * - * The first entry rbOffsetArray[0] is initialized to zero (0) - * in Begin.ino - * - * The array always has one entry in it containing the head offset - * which contains a valid offset into the ringBuffer, handled below - * - * The empty condition is Tail == Head - * - * The amount of data described by the rbOffsetArray is - * rbOffsetArray[Head] - rbOffsetArray[Tail] - * - * rbOffsetArray ringBuffer - * .-----------------. .-----------------. - * | | | | - * +-----------------+ | | - * Tail --> | Msg 1 Offset |---------->+-----------------+ <-- Tail n - * +-----------------+ | Msg 1 | - * | Msg 2 Offset |--------. | | - * +-----------------+ | | | - * | Msg 3 Offset |------. '->+-----------------+ - * +-----------------+ | | Msg 2 | - * Head --> | Head Offset |--. | | | - * +-----------------+ | | | | - * | | | | | | - * +-----------------+ | | | | - * | | | '--->+-----------------+ - * +-----------------+ | | Msg 3 | - * | | | | | - * +-----------------+ '------->+-----------------+ <-- dataHead - * | | | | - */ - - // Determine the index for the end of the circular ring buffer - // offset list - listEnd = rbOffsetHead; - WRAP_OFFSET(listEnd, 1, rbOffsetEntries); - - // Update the tail, walk newest message to oldest message - rbOffsetTail = rbOffsetHead; - bufferedData = 0; - messageLength = 0; - while ((rbOffsetTail != listEnd) && (bufferedData < use)) + ringBufferSemaphoreHolder = "processUart1Message"; + + // Determine if this message will fit into the ring buffer + int32_t discardedBytes = 0; + bytesToCopy = parse->length; + space = availableHandlerSpace; // Take a copy of availableHandlerSpace here + use = settings.gnssHandlerBufferSize - space; + consumer = (char *)slowConsumer; + if (bytesToCopy > space) // Paul removed the && (!inMainMenu)) check 7-25-25 { - // Determine the amount of data in the ring buffer up until - // either the tail or the end of the rbOffsetArray + int32_t bufferedData; + int32_t bytesToDiscard; + int32_t listEnd; + int32_t messageLength; + int32_t previousTail; + int32_t rbOffsetTail; + + // Determine the tail of the ring buffer + previousTail = dataHead + space + 1; + if (previousTail >= settings.gnssHandlerBufferSize) + previousTail -= settings.gnssHandlerBufferSize; + + /* The rbOffsetArray holds the offsets into the ring buffer of the + * start of each of the parsed messages. A head (rbOffsetHead) and + * tail (rbOffsetTail) offsets are used for this array to insert and + * remove entries. Typically this task only manipulates the head as + * new messages are placed into the ring buffer. The handleGnssDataTask + * normally manipulates the tail as data is removed from the buffer. + * However this task will manipulate the tail under two conditions: + * + * 1. The ring buffer gets full and data must be discarded + * + * 2. The rbOffsetArray is too small to hold all of the message + * offsets for the data in the ring buffer. The array is full + * when (Head + 1) == Tail + * + * Notes: + * The rbOffsetArray is allocated along with the ring buffer in + * Begin.ino + * + * The first entry rbOffsetArray[0] is initialized to zero (0) + * in Begin.ino + * + * The array always has one entry in it containing the head offset + * which contains a valid offset into the ringBuffer, handled below + * + * The empty condition is Tail == Head + * + * The amount of data described by the rbOffsetArray is + * rbOffsetArray[Head] - rbOffsetArray[Tail] + * + * rbOffsetArray ringBuffer + * .-----------------. .-----------------. + * | | | | + * +-----------------+ | | + * Tail --> | Msg 1 Offset |---------->+-----------------+ <-- Tail n + * +-----------------+ | Msg 1 | + * | Msg 2 Offset |--------. | | + * +-----------------+ | | | + * | Msg 3 Offset |------. '->+-----------------+ + * +-----------------+ | | Msg 2 | + * Head --> | Head Offset |--. | | | + * +-----------------+ | | | | + * | | | | | | + * +-----------------+ | | | | + * | | | '--->+-----------------+ + * +-----------------+ | | Msg 3 | + * | | | | | + * +-----------------+ '------->+-----------------+ <-- dataHead + * | | | | + */ + + // Determine the index for the end of the circular ring buffer + // offset list + listEnd = rbOffsetHead; + WRAP_OFFSET(listEnd, 1, rbOffsetEntries); + + // Update the tail, walk newest message to oldest message + rbOffsetTail = rbOffsetHead; + bufferedData = 0; + messageLength = 0; + while ((rbOffsetTail != listEnd) && (bufferedData < use)) + { + // Determine the amount of data in the ring buffer up until + // either the tail or the end of the rbOffsetArray + // + // | | + // | | Valid, still in ring buffer + // | Newest | + // +-----------+ <-- rbOffsetHead + // | | + // | | free space + // | | + // rbOffsetTail --> +-----------+ <-- bufferedData + // | ring | + // | buffer | <-- used + // | data | + // +-----------+ Valid, still in ring buffer + // | | + // + messageLength = rbOffsetArray[rbOffsetTail]; + WRAP_OFFSET(rbOffsetTail, rbOffsetEntries - 1, rbOffsetEntries); + messageLength -= rbOffsetArray[rbOffsetTail]; + if (messageLength < 0) + messageLength += settings.gnssHandlerBufferSize; + bufferedData += messageLength; + } + + // Account for any data in the ring buffer not described by the array // // | | - // | | Valid, still in ring buffer + // +-----------+ + // | Oldest | + // | | + // | ring | + // | buffer | <-- used + // | data | + // +-----------+ Valid, still in ring buffer + // | | + // rbOffsetTail --> +-----------+ <-- bufferedData + // | | // | Newest | // +-----------+ <-- rbOffsetHead // | | - // | | free space + // + if (bufferedData < use) + discardedBytes = use - bufferedData; + + // Writing to the SD card, the network or Bluetooth, a partial + // message may be written leaving the tail pointer mid-message + // + // | | + // rbOffsetTail --> +-----------+ + // | Oldest | // | | - // rbOffsetTail --> +-----------+ <-- bufferedData // | ring | // | buffer | <-- used - // | data | - // +-----------+ Valid, still in ring buffer + // | data | Valid, still in ring buffer + // +-----------+ <-- + // | | + // +-----------+ + // | | + // | Newest | + // +-----------+ <-- rbOffsetHead // | | // - messageLength = rbOffsetArray[rbOffsetTail]; - WRAP_OFFSET(rbOffsetTail, rbOffsetEntries - 1, rbOffsetEntries); - messageLength -= rbOffsetArray[rbOffsetTail]; - if (messageLength < 0) - messageLength += settings.gnssHandlerBufferSize; - bufferedData += messageLength; - } + else if (bufferedData > use) + { + // Remove the remaining portion of the oldest entry in the array + discardedBytes = messageLength + use - bufferedData; + WRAP_OFFSET(rbOffsetTail, 1, rbOffsetEntries); + } - // Account for any data in the ring buffer not described by the array - // - // | | - // +-----------+ - // | Oldest | - // | | - // | ring | - // | buffer | <-- used - // | data | - // +-----------+ Valid, still in ring buffer - // | | - // rbOffsetTail --> +-----------+ <-- bufferedData - // | | - // | Newest | - // +-----------+ <-- rbOffsetHead - // | | - // - if (bufferedData < use) - discardedBytes = use - bufferedData; + // rbOffsetTail now points to the beginning of a message in the + // ring buffer + // Determine the amount of data to discard + bytesToDiscard = discardedBytes; + if (bytesToDiscard < bytesToCopy) + bytesToDiscard = bytesToCopy; + if (bytesToDiscard < AMOUNT_OF_RING_BUFFER_DATA_TO_DISCARD) + bytesToDiscard = AMOUNT_OF_RING_BUFFER_DATA_TO_DISCARD; + + // Walk the ring buffer messages from oldest to newest + while ((discardedBytes < bytesToDiscard) && (rbOffsetTail != rbOffsetHead)) + { + // Determine the length of the oldest message + WRAP_OFFSET(rbOffsetTail, 1, rbOffsetEntries); + discardedBytes = rbOffsetArray[rbOffsetTail] - previousTail; + if (discardedBytes < 0) + discardedBytes += settings.gnssHandlerBufferSize; + } - // Writing to the SD card, the network or Bluetooth, a partial - // message may be written leaving the tail pointer mid-message - // - // | | - // rbOffsetTail --> +-----------+ - // | Oldest | - // | | - // | ring | - // | buffer | <-- used - // | data | Valid, still in ring buffer - // +-----------+ <-- - // | | - // +-----------+ - // | | - // | Newest | - // +-----------+ <-- rbOffsetHead - // | | - // - else if (bufferedData > use) - { - // Remove the remaining portion of the oldest entry in the array - discardedBytes = messageLength + use - bufferedData; - WRAP_OFFSET(rbOffsetTail, 1, rbOffsetEntries); + // Discard the oldest data from the ring buffer + // Printing the slow consumer is not that useful as any consumer will be + // considered 'slow' if its data wraps over the end of the buffer and + // needs a second write to clear... + if (!inMainMenu) + { + if (consumer) + systemPrintf("Ring buffer full: discarding %d bytes, %s could be slow\r\n", discardedBytes, consumer); + else + systemPrintf("Ring buffer full: discarding %d bytes\r\n", discardedBytes); + Serial.flush(); // TODO - delete me! + } + + // Update the tails. This needs semaphore protection + updateRingBufferTails(previousTail, rbOffsetArray[rbOffsetTail]); } - // rbOffsetTail now points to the beginning of a message in the - // ring buffer - // Determine the amount of data to discard - bytesToDiscard = discardedBytes; - if (bytesToDiscard < bytesToCopy) - bytesToDiscard = bytesToCopy; - if (bytesToDiscard < AMOUNT_OF_RING_BUFFER_DATA_TO_DISCARD) - bytesToDiscard = AMOUNT_OF_RING_BUFFER_DATA_TO_DISCARD; - - // Walk the ring buffer messages from oldest to newest - while ((discardedBytes < bytesToDiscard) && (rbOffsetTail != rbOffsetHead)) + if (bytesToCopy > (space + discardedBytes - 1)) // Sanity check { - // Determine the length of the oldest message - WRAP_OFFSET(rbOffsetTail, 1, rbOffsetEntries); - discardedBytes = rbOffsetArray[rbOffsetTail] - previousTail; - if (discardedBytes < 0) - discardedBytes += settings.gnssHandlerBufferSize; + systemPrintf("Ring buffer update error %s: bytesToCopy (%d) is > space (%d) + discardedBytes (%d) - 1\r\n", + getTimeStamp(), bytesToCopy, space, discardedBytes); + Serial.flush(); // Flush Serial - the code is about to go bang...! } + + // Add another message to the ring buffer + // Account for this message + // Diagnostic prints are provided by settings.enablePrintSDBuffers and the handleGnssDataTask + // The semaphore prevents badness here. Previously availableHandlerSpace may have been updated + // by handleGnssDataTask + availableHandlerSpace = availableHandlerSpace + discardedBytes - bytesToCopy; + + // Copy dataHead so we can update with a single write - redundant with the semaphore + RING_BUFFER_OFFSET newDataHead = dataHead; + + // Fill the buffer to the end and then start at the beginning + if ((newDataHead + bytesToCopy) > settings.gnssHandlerBufferSize) + bytesToCopy = settings.gnssHandlerBufferSize - newDataHead; + + // Display the dataHead offset + if (settings.enablePrintRingBufferOffsets && (!inMainMenu)) + systemPrintf("DH: %4d --> ", newDataHead); + + // Copy the data into the ring buffer + memcpy(&ringBuffer[newDataHead], parse->buffer, bytesToCopy); + newDataHead = newDataHead + bytesToCopy; + if (newDataHead >= settings.gnssHandlerBufferSize) + newDataHead = newDataHead - settings.gnssHandlerBufferSize; - // Discard the oldest data from the ring buffer - // Printing the slow consumer is not that useful as any consumer will be - // considered 'slow' if its data wraps over the end of the buffer and - // needs a second write to clear... - if (!inMainMenu) + // Determine the remaining bytes + remainingBytes = parse->length - bytesToCopy; + if (remainingBytes) { - if (consumer) - systemPrintf("Ring buffer full: discarding %d bytes, %s could be slow\r\n", discardedBytes, consumer); - else - systemPrintf("Ring buffer full: discarding %d bytes\r\n", discardedBytes); + // Copy the remaining bytes into the beginning of the ring buffer + memcpy(ringBuffer, &parse->buffer[bytesToCopy], remainingBytes); + newDataHead = newDataHead + remainingBytes; + if (newDataHead >= settings.gnssHandlerBufferSize) + newDataHead = newDataHead - settings.gnssHandlerBufferSize; } - // Update the tails. Strictly this needs semaphore protection - updateRingBufferTails(previousTail, rbOffsetArray[rbOffsetTail]); - } - - if (bytesToCopy > (space + discardedBytes - 1)) // Sanity check - systemPrintf("Ring buffer update error: bytesToCopy (%d) is > space (%d) + discardedBytes (%d) - 1\r\n", - bytesToCopy, space, discardedBytes); - - // Add another message to the ring buffer - // Account for this message - // Diagnostic prints are provided by settings.enablePrintSDBuffers and the handleGnssDataTask - // availableHandlerSpace may have been updated by handleGnssDataTask - availableHandlerSpace = availableHandlerSpace + discardedBytes - bytesToCopy; - - // Copy dataHead so we can update with a single write - RING_BUFFER_OFFSET newDataHead = dataHead; - - // Fill the buffer to the end and then start at the beginning - if ((newDataHead + bytesToCopy) > settings.gnssHandlerBufferSize) - bytesToCopy = settings.gnssHandlerBufferSize - newDataHead; - - // Display the dataHead offset - if (settings.enablePrintRingBufferOffsets && (!inMainMenu)) - systemPrintf("DH: %4d --> ", newDataHead); - - // Copy the data into the ring buffer - memcpy(&ringBuffer[newDataHead], parse->buffer, bytesToCopy); - newDataHead = newDataHead + bytesToCopy; - if (newDataHead >= settings.gnssHandlerBufferSize) - newDataHead = newDataHead - settings.gnssHandlerBufferSize; - - // Determine the remaining bytes - remainingBytes = parse->length - bytesToCopy; - if (remainingBytes) - { - // Copy the remaining bytes into the beginning of the ring buffer - memcpy(ringBuffer, &parse->buffer[bytesToCopy], remainingBytes); - newDataHead = newDataHead + remainingBytes; - if (newDataHead >= settings.gnssHandlerBufferSize) - newDataHead = newDataHead - settings.gnssHandlerBufferSize; - } + // Add the head offset to the offset array + WRAP_OFFSET(rbOffsetHead, 1, rbOffsetEntries); + rbOffsetArray[rbOffsetHead] = newDataHead; - // Add the head offset to the offset array - WRAP_OFFSET(rbOffsetHead, 1, rbOffsetEntries); - rbOffsetArray[rbOffsetHead] = newDataHead; + // Display the dataHead offset + if (settings.enablePrintRingBufferOffsets && (!inMainMenu)) + systemPrintf("%4d @ %s\r\n", newDataHead, getTimeStamp()); - // Display the dataHead offset - if (settings.enablePrintRingBufferOffsets && (!inMainMenu)) - systemPrintf("%4d\r\n", newDataHead); + // Update dataHead in a single write - redundant with the semaphore + // handleGnssDataTask will use it as soon as it updates + dataHead = newDataHead; - // Update dataHead in a single write - handleGnssDataTask will use it as soon as it updates - dataHead = newDataHead; + // handleGnssDataTask will be chomping at the bit. Let it fly! + xSemaphoreGive(ringBufferSemaphore); + } + else + { + systemPrintf("processUart1Message could not get ringBuffer semaphore - held by %s\r\n", ringBufferSemaphoreHolder); + } } // Remove previous messages from the ring buffer @@ -1124,423 +1142,445 @@ void handleGnssDataTask(void *e) usedSpace = 0; - //---------------------------------------------------------------------- - // Send data over Bluetooth - //---------------------------------------------------------------------- - - startMillis = millis(); - - // Determine BT connection state - bool connected = (bluetoothGetState() == BT_CONNECTED); - - if (!connected) - // Discard the data - btRingBufferTail = dataHead; - else + // Use a semaphore to prevent handleGnssDataTask from gatecrashing + if (ringBufferSemaphore == NULL) + ringBufferSemaphore = xSemaphoreCreateMutex(); // Create the mutex + + // Take the semaphore. Short wait. processUart1Message shouldn't block + if (xSemaphoreTake(ringBufferSemaphore, fatSemaphore_shortWait_ms) == pdPASS) { - // Determine the amount of Bluetooth data in the buffer - bytesToSend = dataHead - btRingBufferTail; - if (bytesToSend < 0) - bytesToSend += settings.gnssHandlerBufferSize; - if (bytesToSend > 0) - { - // Reduce bytes to send if we have more to send then the end of - // the buffer, we'll wrap next loop - if ((btRingBufferTail + bytesToSend) > settings.gnssHandlerBufferSize) - bytesToSend = settings.gnssHandlerBufferSize - btRingBufferTail; + ringBufferSemaphoreHolder = "handleGnssDataTask"; - // If we are in the config menu, suppress data flowing from GNSS to cell phone - if (btPrintEcho == false) - { - // Push new data over Bluetooth - bytesToSend = bluetoothWrite(&ringBuffer[btRingBufferTail], bytesToSend); - } + //---------------------------------------------------------------------- + // Send data over Bluetooth + //---------------------------------------------------------------------- - // Account for the data that was sent - if (bytesToSend > 0) - { - // If we are in base mode, assume part of the outgoing data is RTCM - if (inBaseMode() == true) - bluetoothOutgoingRTCM = true; - - // Account for the sent or dropped data - btRingBufferTail += bytesToSend; - if (btRingBufferTail >= settings.gnssHandlerBufferSize) - btRingBufferTail -= settings.gnssHandlerBufferSize; - - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_BLUETOOTH] < deltaMillis) - maxMillis[RBC_BLUETOOTH] = deltaMillis; - - // Display the data movement - if (PERIODIC_DISPLAY(PD_BLUETOOTH_DATA_TX)) - { - PERIODIC_CLEAR(PD_BLUETOOTH_DATA_TX); - systemPrintf("Bluetooth: %d bytes written\r\n", bytesToSend); - } - } - else - log_w("BT failed to send"); + startMillis = millis(); + + // Determine BT connection state + bool connected = (bluetoothGetState() == BT_CONNECTED); - // Determine the amount of data that remains in the buffer + if (!connected) + // Discard the data + btRingBufferTail = dataHead; + else + { + // Determine the amount of Bluetooth data in the buffer bytesToSend = dataHead - btRingBufferTail; if (bytesToSend < 0) bytesToSend += settings.gnssHandlerBufferSize; - if (usedSpace < bytesToSend) + if (bytesToSend > 0) { - usedSpace = bytesToSend; - slowConsumer = "Bluetooth"; - } - } - } + // Reduce bytes to send if we have more to send then the end of + // the buffer, we'll wrap next loop + if ((btRingBufferTail + bytesToSend) > settings.gnssHandlerBufferSize) + bytesToSend = settings.gnssHandlerBufferSize - btRingBufferTail; - //---------------------------------------------------------------------- - // Send data over USB serial - //---------------------------------------------------------------------- + // If we are in the config menu, suppress data flowing from GNSS to cell phone + if (btPrintEcho == false) + { + // Push new data over Bluetooth + bytesToSend = bluetoothWrite(&ringBuffer[btRingBufferTail], bytesToSend); + } - startMillis = millis(); + // Account for the data that was sent + if (bytesToSend > 0) + { + // If we are in base mode, assume part of the outgoing data is RTCM + if (inBaseMode() == true) + bluetoothOutgoingRTCM = true; - // Determine USB serial connection state - if (!forwardGnssDataToUsbSerial) - // Discard the data - usbRingBufferTail = dataHead; - else - { - // Determine the amount of USB serial data in the buffer - bytesToSend = dataHead - usbRingBufferTail; - if (bytesToSend < 0) - bytesToSend += settings.gnssHandlerBufferSize; - if (bytesToSend > 0) - { - // Reduce bytes to send if we have more to send then the end of - // the buffer, we'll wrap next loop - if ((usbRingBufferTail + bytesToSend) > settings.gnssHandlerBufferSize) - bytesToSend = settings.gnssHandlerBufferSize - usbRingBufferTail; + // Account for the sent or dropped data + btRingBufferTail += bytesToSend; + if (btRingBufferTail >= settings.gnssHandlerBufferSize) + btRingBufferTail -= settings.gnssHandlerBufferSize; - // Send data over USB serial to the PC - bytesToSend = systemWriteGnssDataToUsbSerial(&ringBuffer[usbRingBufferTail], bytesToSend); + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_BLUETOOTH] < deltaMillis) + maxMillis[RBC_BLUETOOTH] = deltaMillis; - // Account for the data that was sent - if (bytesToSend > 0) - { - // Account for the sent or dropped data - usbRingBufferTail += bytesToSend; - if (usbRingBufferTail >= settings.gnssHandlerBufferSize) - usbRingBufferTail -= settings.gnssHandlerBufferSize; - - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_USB_SERIAL] < deltaMillis) - maxMillis[RBC_USB_SERIAL] = deltaMillis; + // Display the data movement + if (PERIODIC_DISPLAY(PD_BLUETOOTH_DATA_TX)) + { + PERIODIC_CLEAR(PD_BLUETOOTH_DATA_TX); + systemPrintf("Bluetooth: %d bytes written\r\n", bytesToSend); + } + } + else + log_w("BT failed to send"); + + // Determine the amount of data that remains in the buffer + bytesToSend = dataHead - btRingBufferTail; + if (bytesToSend < 0) + bytesToSend += settings.gnssHandlerBufferSize; + if (usedSpace < bytesToSend) + { + usedSpace = bytesToSend; + slowConsumer = "Bluetooth"; + } } + } + + //---------------------------------------------------------------------- + // Send data over USB serial + //---------------------------------------------------------------------- + + startMillis = millis(); - // Determine the amount of data that remains in the buffer + // Determine USB serial connection state + if (!forwardGnssDataToUsbSerial) + // Discard the data + usbRingBufferTail = dataHead; + else + { + // Determine the amount of USB serial data in the buffer bytesToSend = dataHead - usbRingBufferTail; if (bytesToSend < 0) bytesToSend += settings.gnssHandlerBufferSize; - if (usedSpace < bytesToSend) + if (bytesToSend > 0) { - usedSpace = bytesToSend; - slowConsumer = "USB Serial"; + // Reduce bytes to send if we have more to send then the end of + // the buffer, we'll wrap next loop + if ((usbRingBufferTail + bytesToSend) > settings.gnssHandlerBufferSize) + bytesToSend = settings.gnssHandlerBufferSize - usbRingBufferTail; + + // Send data over USB serial to the PC + bytesToSend = systemWriteGnssDataToUsbSerial(&ringBuffer[usbRingBufferTail], bytesToSend); + + // Account for the data that was sent + if (bytesToSend > 0) + { + // Account for the sent or dropped data + usbRingBufferTail += bytesToSend; + if (usbRingBufferTail >= settings.gnssHandlerBufferSize) + usbRingBufferTail -= settings.gnssHandlerBufferSize; + + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_USB_SERIAL] < deltaMillis) + maxMillis[RBC_USB_SERIAL] = deltaMillis; + } + + // Determine the amount of data that remains in the buffer + bytesToSend = dataHead - usbRingBufferTail; + if (bytesToSend < 0) + bytesToSend += settings.gnssHandlerBufferSize; + if (usedSpace < bytesToSend) + { + usedSpace = bytesToSend; + slowConsumer = "USB Serial"; + } } } - } - //---------------------------------------------------------------------- - // Send data to the network clients - //---------------------------------------------------------------------- + //---------------------------------------------------------------------- + // Send data to the network clients + //---------------------------------------------------------------------- - startMillis = millis(); + startMillis = millis(); - // Update space available for use in UART task - bytesToSend = tcpClientSendData(dataHead); - if (usedSpace < bytesToSend) - { - usedSpace = bytesToSend; - slowConsumer = "TCP client"; - } + // Update space available for use in UART task + bytesToSend = tcpClientSendData(dataHead); + if (usedSpace < bytesToSend) + { + usedSpace = bytesToSend; + slowConsumer = "TCP client"; + } - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_TCP_CLIENT] < deltaMillis) - maxMillis[RBC_TCP_CLIENT] = deltaMillis; + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_TCP_CLIENT] < deltaMillis) + maxMillis[RBC_TCP_CLIENT] = deltaMillis; - startMillis = millis(); + startMillis = millis(); - // Update space available for use in UART task - bytesToSend = tcpServerSendData(dataHead); - if (usedSpace < bytesToSend) - { - usedSpace = bytesToSend; - slowConsumer = "TCP server"; - } + // Update space available for use in UART task + bytesToSend = tcpServerSendData(dataHead); + if (usedSpace < bytesToSend) + { + usedSpace = bytesToSend; + slowConsumer = "TCP server"; + } - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_TCP_SERVER] < deltaMillis) - maxMillis[RBC_TCP_SERVER] = deltaMillis; + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_TCP_SERVER] < deltaMillis) + maxMillis[RBC_TCP_SERVER] = deltaMillis; - startMillis = millis(); + startMillis = millis(); - // Update space available for use in UART task - bytesToSend = udpServerSendData(dataHead); - if (usedSpace < bytesToSend) - { - usedSpace = bytesToSend; - slowConsumer = "UDP server"; - } + // Update space available for use in UART task + bytesToSend = udpServerSendData(dataHead); + if (usedSpace < bytesToSend) + { + usedSpace = bytesToSend; + slowConsumer = "UDP server"; + } - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_UDP_SERVER] < deltaMillis) - maxMillis[RBC_UDP_SERVER] = deltaMillis; + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_UDP_SERVER] < deltaMillis) + maxMillis[RBC_UDP_SERVER] = deltaMillis; - //---------------------------------------------------------------------- - // Log data to the SD card - //---------------------------------------------------------------------- + //---------------------------------------------------------------------- + // Log data to the SD card + //---------------------------------------------------------------------- - // Determine if the SD card is enabled for logging - connected = online.logging && (!logTimeExceeded()); + // Determine if the SD card is enabled for logging + connected = online.logging && (!logTimeExceeded()); - // Block logging during Web Config to avoid SD collisions - // See issue: https://github.com/sparkfun/SparkFun_RTK_Everywhere_Firmware/issues/693 - if(webServerIsRunning() == true) - connected = false; + // Block logging during Web Config to avoid SD collisions + // See issue: https://github.com/sparkfun/SparkFun_RTK_Everywhere_Firmware/issues/693 + if(webServerIsRunning() == true) + connected = false; - // If user wants to log, record to SD - if (!connected) - // Discard the data - sdRingBufferTail = dataHead; - else - { - // Determine the amount of microSD card logging data in the buffer - bytesToSend = dataHead - sdRingBufferTail; - if (bytesToSend < 0) - bytesToSend += settings.gnssHandlerBufferSize; - if (bytesToSend > 0) + // If user wants to log, record to SD + if (!connected) + // Discard the data + sdRingBufferTail = dataHead; + else { - // Attempt to gain access to the SD card, avoids collisions with file - // writing from other functions like recordSystemSettingsToFile() - if (xSemaphoreTake(sdCardSemaphore, loggingSemaphoreWait_ms) == pdPASS) + // Determine the amount of microSD card logging data in the buffer + bytesToSend = dataHead - sdRingBufferTail; + if (bytesToSend < 0) + bytesToSend += settings.gnssHandlerBufferSize; + if (bytesToSend > 0) { - markSemaphore(FUNCTION_WRITESD); - - do // Do the SD write in a do loop so we can break out if needed + // Attempt to gain access to the SD card, avoids collisions with file + // writing from other functions like recordSystemSettingsToFile() + if (xSemaphoreTake(sdCardSemaphore, loggingSemaphoreWait_ms) == pdPASS) { - if (settings.enablePrintSDBuffers && (!inMainMenu)) - { - int bufferAvailable = serialGNSS->available(); + markSemaphore(FUNCTION_WRITESD); - int availableUARTSpace = settings.uartReceiveBufferSize - bufferAvailable; - - systemPrintf("SD Incoming Serial: %04d\tToRead: %04d\tMovedToBuffer: %04d\tavailableUARTSpace: " - "%04d\tavailableHandlerSpace: %04d\tToRecord: %04d\tRecorded: %04d\tBO: %d\r\n", - bufferAvailable, 0, 0, availableUARTSpace, availableHandlerSpace, bytesToSend, 0, - bufferOverruns); - } - - // For the SD card, we need to write everything we've got - // to prevent the ARP Write and Events from gatecrashing... - - int32_t sendTheseBytes = bytesToSend; - - // Reduce bytes to record if we have more then the end of the buffer - if ((sdRingBufferTail + sendTheseBytes) > settings.gnssHandlerBufferSize) - sendTheseBytes = settings.gnssHandlerBufferSize - sdRingBufferTail; + do // Do the SD write in a do loop so we can break out if needed + { + if (settings.enablePrintSDBuffers && (!inMainMenu)) + { + int bufferAvailable = serialGNSS->available(); - startMillis = millis(); + int availableUARTSpace = settings.uartReceiveBufferSize - bufferAvailable; - // Write the data to the file - int32_t bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); + systemPrintf("SD Incoming Serial @ %s: %04d\tToRead: %04d\tMovedToBuffer: %04d\tavailableUARTSpace: " + "%04d\tavailableHandlerSpace: %04d\tToRecord: %04d\tRecorded: %04d\tBO: %d\r\n", + getTimeStamp(), bufferAvailable, 0, 0, availableUARTSpace, availableHandlerSpace, + bytesToSend, 0, bufferOverruns); + } - // Account for the sent data or dropped - sdRingBufferTail += bytesSent; - if (sdRingBufferTail >= settings.gnssHandlerBufferSize) - sdRingBufferTail -= settings.gnssHandlerBufferSize; + // For the SD card, we need to write everything we've got + // to prevent the ARP Write and Events from gatecrashing... + + int32_t sendTheseBytes = bytesToSend; - if (bytesSent != sendTheseBytes) - { - systemPrintf("SD write mismatch (1): wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); - break; // Exit the do loop - } + // Reduce bytes to record if we have more then the end of the buffer + if ((sdRingBufferTail + sendTheseBytes) > settings.gnssHandlerBufferSize) + sendTheseBytes = settings.gnssHandlerBufferSize - sdRingBufferTail; - // If we have more data to write - and the first write was successful - if (bytesToSend > sendTheseBytes) - { - sendTheseBytes = bytesToSend - sendTheseBytes; + startMillis = millis(); - bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); + // Write the data to the file + int32_t bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); // Account for the sent data or dropped sdRingBufferTail += bytesSent; - if (sdRingBufferTail >= settings.gnssHandlerBufferSize) // Should be redundant + if (sdRingBufferTail >= settings.gnssHandlerBufferSize) sdRingBufferTail -= settings.gnssHandlerBufferSize; if (bytesSent != sendTheseBytes) { - systemPrintf("SD write mismatch (2): wrote %d bytes of %d\r\n", bytesSent, sendTheseBytes); + systemPrintf("SD write mismatch (1) @ %s: wrote %d bytes of %d\r\n", + getTimeStamp(), bytesSent, sendTheseBytes); break; // Exit the do loop } - } - if (PERIODIC_DISPLAY(PD_SD_LOG_WRITE) && (bytesSent > 0)) - { - PERIODIC_CLEAR(PD_SD_LOG_WRITE); - systemPrintf("SD %d bytes written to log file\r\n", bytesToSend); - } + // If we have more data to write - and the first write was successful + if (bytesToSend > sendTheseBytes) + { + sendTheseBytes = bytesToSend - sendTheseBytes; - sdFreeSpace -= bytesToSend; // Update remaining space on SD + bytesSent = logFile->write(&ringBuffer[sdRingBufferTail], sendTheseBytes); - // Record any pending trigger events - if (newEventToRecord == true) - { - newEventToRecord = false; + // Account for the sent data or dropped + sdRingBufferTail += bytesSent; + if (sdRingBufferTail >= settings.gnssHandlerBufferSize) // Should be redundant + sdRingBufferTail -= settings.gnssHandlerBufferSize; - if (settings.enablePrintLogFileStatus) - systemPrintln("Log file: recording event"); + if (bytesSent != sendTheseBytes) + { + systemPrintf("SD write mismatch (2) @ %s: wrote %d bytes of %d\r\n", + getTimeStamp(), bytesSent, sendTheseBytes); + break; // Exit the do loop + } + } - // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of - // rising edge (ns), and accuracy estimate (ns) - char eventData[82]; // Max NMEA sentence length is 82 - snprintf(eventData, sizeof(eventData), "%d,%d,%d,%d", triggerCount, triggerTowMsR, triggerTowSubMsR, - triggerAccEst); + if (PERIODIC_DISPLAY(PD_SD_LOG_WRITE) && (bytesSent > 0)) + { + PERIODIC_CLEAR(PD_SD_LOG_WRITE); + systemPrintf("SD %d bytes written to log file\r\n", bytesToSend); + } - char nmeaMessage[82]; // Max NMEA sentence length is 82 - createNMEASentence(CUSTOM_NMEA_TYPE_EVENT, nmeaMessage, sizeof(nmeaMessage), - eventData); // textID, buffer, sizeOfBuffer, text + sdFreeSpace -= bytesToSend; // Update remaining space on SD - logFile->write(nmeaMessage, strlen(nmeaMessage)); - const char *crlf = "\r\n"; - logFile->write(crlf, 2); + // Record any pending trigger events + if (newEventToRecord == true) + { + newEventToRecord = false; - sdFreeSpace -= strlen(nmeaMessage) + 2; // Update remaining space on SD - } + if (settings.enablePrintLogFileStatus) + systemPrintln("Log file: recording event"); - // Record the Antenna Reference Position - if available - if (newARPAvailable == true && settings.enableARPLogging && - ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) - { - lastARPLog = millis(); - newARPAvailable = false; // Clear flag. It doesn't matter if the ARP cannot be logged - - double x = ARPECEFX; - x /= 10000.0; // Convert to m - double y = ARPECEFY; - y /= 10000.0; // Convert to m - double z = ARPECEFZ; - z /= 10000.0; // Convert to m - double h = ARPECEFH; - h /= 10000.0; // Convert to m - char ARPData[82]; // Max NMEA sentence length is 82 - snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); - - if (settings.enablePrintLogFileStatus) - systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); - - char nmeaMessage[82]; // Max NMEA sentence length is 82 - createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), - ARPData); // textID, buffer, sizeOfBuffer, text - - logFile->write(nmeaMessage, strlen(nmeaMessage)); - const char *crlf = "\r\n"; - logFile->write(crlf, 2); - - sdFreeSpace -= strlen(nmeaMessage) + 2; // Update remaining space on SD - } + // Record trigger count with Time Of Week of rising edge (ms), Millisecond fraction of Time Of Week of + // rising edge (ns), and accuracy estimate (ns) + char eventData[82]; // Max NMEA sentence length is 82 + snprintf(eventData, sizeof(eventData), "%d,%d,%d,%d", triggerCount, triggerTowMsR, triggerTowSubMsR, + triggerAccEst); - logFileSize = logFile->fileSize(); // Update file size + char nmeaMessage[82]; // Max NMEA sentence length is 82 + createNMEASentence(CUSTOM_NMEA_TYPE_EVENT, nmeaMessage, sizeof(nmeaMessage), + eventData); // textID, buffer, sizeOfBuffer, text - // Force file sync every 60s - if (millis() - lastUBXLogSyncTime > 60000) - { - baseStatusLedBlink(); // Blink LED to indicate logging activity + logFile->write(nmeaMessage, strlen(nmeaMessage)); + const char *crlf = "\r\n"; + logFile->write(crlf, 2); - logFile->sync(); - sdUpdateFileAccessTimestamp(logFile); // Update the file access time & date + sdFreeSpace -= strlen(nmeaMessage) + 2; // Update remaining space on SD + } - baseStatusLedBlink(); // Blink LED to indicate logging activity + // Record the Antenna Reference Position - if available + if (newARPAvailable == true && settings.enableARPLogging && + ((millis() - lastARPLog) > (settings.ARPLoggingInterval_s * 1000))) + { + lastARPLog = millis(); + newARPAvailable = false; // Clear flag. It doesn't matter if the ARP cannot be logged + + double x = ARPECEFX; + x /= 10000.0; // Convert to m + double y = ARPECEFY; + y /= 10000.0; // Convert to m + double z = ARPECEFZ; + z /= 10000.0; // Convert to m + double h = ARPECEFH; + h /= 10000.0; // Convert to m + char ARPData[82]; // Max NMEA sentence length is 82 + snprintf(ARPData, sizeof(ARPData), "%.4f,%.4f,%.4f,%.4f", x, y, z, h); + + if (settings.enablePrintLogFileStatus) + systemPrintf("Log file: recording Antenna Reference Position %s\r\n", ARPData); + + char nmeaMessage[82]; // Max NMEA sentence length is 82 + createNMEASentence(CUSTOM_NMEA_TYPE_ARP_ECEF_XYZH, nmeaMessage, sizeof(nmeaMessage), + ARPData); // textID, buffer, sizeOfBuffer, text + + logFile->write(nmeaMessage, strlen(nmeaMessage)); + const char *crlf = "\r\n"; + logFile->write(crlf, 2); + + sdFreeSpace -= strlen(nmeaMessage) + 2; // Update remaining space on SD + } - lastUBXLogSyncTime = millis(); - } + logFileSize = logFile->fileSize(); // Update file size - // Remember the maximum transfer time - deltaMillis = millis() - startMillis; - if (maxMillis[RBC_SD_CARD] < deltaMillis) - maxMillis[RBC_SD_CARD] = deltaMillis; + // Force file sync every 60s + if (millis() - lastUBXLogSyncTime > 60000) + { + baseStatusLedBlink(); // Blink LED to indicate logging activity - if (settings.enablePrintBufferOverrun) - { - if (deltaMillis > 150) - systemPrintf("Long Write! Time: %ld ms / Location: %ld / Recorded %d bytes / " - "spaceRemaining %d bytes\r\n", - deltaMillis, logFileSize, bytesToSend, combinedSpaceRemaining); - } - } while(0); + logFile->sync(); + sdUpdateFileAccessTimestamp(logFile); // Update the file access time & date - xSemaphoreGive(sdCardSemaphore); - } // End sdCardSemaphore - else - { - char semaphoreHolder[50]; - getSemaphoreFunction(semaphoreHolder); - log_w("sdCardSemaphore failed to yield for SD write, held by %s, Tasks.ino line %d", - semaphoreHolder, __LINE__); + baseStatusLedBlink(); // Blink LED to indicate logging activity - delay(1); // Needed to prevent WDT resets during long Record Settings locks - taskYIELD(); - } + lastUBXLogSyncTime = millis(); + } - // Update space available for use in UART task - bytesToSend = dataHead - sdRingBufferTail; - if (bytesToSend < 0) - bytesToSend += settings.gnssHandlerBufferSize; - if (usedSpace < bytesToSend) - { - usedSpace = bytesToSend; - slowConsumer = "SD card"; - } - } // bytesToSend - } // End connected + // Remember the maximum transfer time + deltaMillis = millis() - startMillis; + if (maxMillis[RBC_SD_CARD] < deltaMillis) + maxMillis[RBC_SD_CARD] = deltaMillis; - //---------------------------------------------------------------------- - // Update the available space in the ring buffer - //---------------------------------------------------------------------- + if (settings.enablePrintBufferOverrun) + { + if (deltaMillis > 150) + systemPrintf("Long Write! Time: %ld ms / Location: %ld / Recorded %d bytes / " + "spaceRemaining %d bytes\r\n", + deltaMillis, logFileSize, bytesToSend, combinedSpaceRemaining); + } + } while(0); - freeSpace = settings.gnssHandlerBufferSize - usedSpace; + xSemaphoreGive(sdCardSemaphore); + } // End sdCardSemaphore + else + { + char semaphoreHolder[50]; + getSemaphoreFunction(semaphoreHolder); + log_w("sdCardSemaphore failed to yield for SD write, held by %s, Tasks.ino line %d", + semaphoreHolder, __LINE__); - // Don't fill the last byte to prevent buffer overflow - if (freeSpace) - freeSpace -= 1; - availableHandlerSpace = freeSpace; + feedWdt(); + taskYIELD(); + } - //---------------------------------------------------------------------- - // Display the millisecond values for the different ring buffer consumers - //---------------------------------------------------------------------- + // Update space available for use in UART task + bytesToSend = dataHead - sdRingBufferTail; + if (bytesToSend < 0) + bytesToSend += settings.gnssHandlerBufferSize; + if (usedSpace < bytesToSend) + { + usedSpace = bytesToSend; + slowConsumer = "SD card"; + } + } // bytesToSend + } // End connected - if (PERIODIC_DISPLAY(PD_RING_BUFFER_MILLIS)) - { - int milliseconds; - int seconds; + //---------------------------------------------------------------------- + // Update the available space in the ring buffer + //---------------------------------------------------------------------- + + freeSpace = settings.gnssHandlerBufferSize - usedSpace; - PERIODIC_CLEAR(PD_RING_BUFFER_MILLIS); - for (int index = 0; index < RBC_MAX; index++) + // Don't fill the last byte to prevent buffer overflow + if (freeSpace) + freeSpace -= 1; + availableHandlerSpace = freeSpace; + + //---------------------------------------------------------------------- + // Display the millisecond values for the different ring buffer consumers + //---------------------------------------------------------------------- + + if (PERIODIC_DISPLAY(PD_RING_BUFFER_MILLIS)) { - milliseconds = maxMillis[index]; - if (milliseconds > 1) + int milliseconds; + int seconds; + + PERIODIC_CLEAR(PD_RING_BUFFER_MILLIS); + for (int index = 0; index < RBC_MAX; index++) { - seconds = milliseconds / MILLISECONDS_IN_A_SECOND; - milliseconds %= MILLISECONDS_IN_A_SECOND; - systemPrintf("%s: %d:%03d Sec\r\n", ringBufferConsumer[index], seconds, milliseconds); + milliseconds = maxMillis[index]; + if (milliseconds > 1) + { + seconds = milliseconds / MILLISECONDS_IN_A_SECOND; + milliseconds %= MILLISECONDS_IN_A_SECOND; + systemPrintf("%s: %d:%03d Sec\r\n", ringBufferConsumer[index], seconds, milliseconds); + } } } + + //---------------------------------------------------------------------- + // processUart1Message will be chomping at the bit. Let it fly! + //---------------------------------------------------------------------- + + xSemaphoreGive(ringBufferSemaphore); + } + else + { + systemPrintf("handleGnssDataTask could not get ringBuffer semaphore - held by %s\r\n", ringBufferSemaphoreHolder); } //---------------------------------------------------------------------- // Let other tasks run, prevent watch dog timer (WDT) resets //---------------------------------------------------------------------- - delay(1); + feedWdt(); taskYIELD(); } @@ -2249,7 +2289,7 @@ void sdSizeCheckTask(void *e) } } - delay(1); + feedWdt(); taskYIELD(); // Let other tasks run } @@ -2338,6 +2378,8 @@ void beginRtcmParse() if (!rtcmParse) reportFatalError("Failed to initialize the RTCM parser"); + //sempEnableDebugOutput(rtcmParse); + //sempPrintParserConfiguration(rtcmParse); } // Check and record the base location in RTCM1005/1006 diff --git a/Firmware/RTK_Everywhere/WebServer.ino b/Firmware/RTK_Everywhere/WebServer.ino index 666c0b74a..b225b680e 100644 --- a/Firmware/RTK_Everywhere/WebServer.ino +++ b/Firmware/RTK_Everywhere/WebServer.ino @@ -1093,16 +1093,8 @@ void webServerSetState(uint8_t newState) if (!online.rtc) systemPrintf("%s%s%s%s\r\n", asterisk, initialState, arrow, endingState); else - { // Timestamp the state change - // 1 2 - // 12345678901234567890123456 - // YYYY-mm-dd HH:MM:SS.xxxrn0 - struct tm timeinfo = rtc.getTimeStruct(); - char s[30]; - strftime(s, sizeof(s), "%Y-%m-%d %H:%M:%S", &timeinfo); - systemPrintf("%s%s%s%s, %s.%03ld\r\n", asterisk, initialState, arrow, endingState, s, rtc.getMillis()); - } + systemPrintf("%s%s%s%s, %s\r\n", asterisk, initialState, arrow, endingState, getTimeStamp()); } // Validate the state diff --git a/Firmware/RTK_Everywhere/menuFirmware.ino b/Firmware/RTK_Everywhere/menuFirmware.ino index 2f7382cc1..9f5b399c7 100644 --- a/Firmware/RTK_Everywhere/menuFirmware.ino +++ b/Firmware/RTK_Everywhere/menuFirmware.ino @@ -754,16 +754,8 @@ void otaSetState(uint8_t newState) if (!online.rtc) systemPrintf("%s%s%s%s\r\n", asterisk, initialState, arrow, endingState); else - { // Timestamp the state change - // 1 2 - // 12345678901234567890123456 - // YYYY-mm-dd HH:MM:SS.xxxrn0 - struct tm timeinfo = rtc.getTimeStruct(); - char s[30]; - strftime(s, sizeof(s), "%Y-%m-%d %H:%M:%S", &timeinfo); - systemPrintf("%s%s%s%s, %s.%03ld\r\n", asterisk, initialState, arrow, endingState, s, rtc.getMillis()); - } + systemPrintf("%s%s%s%s, %s\r\n", asterisk, initialState, arrow, endingState, getTimeStamp()); } // Validate the firmware update state diff --git a/Firmware/RTK_Everywhere/menuMessages.ino b/Firmware/RTK_Everywhere/menuMessages.ino index fe45c6d64..cb8a6cf53 100644 --- a/Firmware/RTK_Everywhere/menuMessages.ino +++ b/Firmware/RTK_Everywhere/menuMessages.ino @@ -434,7 +434,7 @@ void endLogging(bool gotSemaphore, bool releaseSemaphore) delete logFile; logFile = nullptr; - systemPrintln("Log file closed"); + systemPrintf("Log file closed @ %s\r\n", getTimeStamp()); // Release the semaphore if requested if (releaseSemaphore) diff --git a/Firmware/RTK_Everywhere/menuSystem.ino b/Firmware/RTK_Everywhere/menuSystem.ino index df4f8e020..b6dbe71af 100644 --- a/Firmware/RTK_Everywhere/menuSystem.ino +++ b/Firmware/RTK_Everywhere/menuSystem.ino @@ -8,7 +8,7 @@ void menuSystem() systemPrintln(); systemPrintln("System Status"); - printTimeStamp(); + printTimeStamp(true); systemPrint("GNSS: "); if (online.gnss == true) diff --git a/Firmware/RTK_Everywhere/support.ino b/Firmware/RTK_Everywhere/support.ino index 8d0f67a84..55314efa4 100644 --- a/Firmware/RTK_Everywhere/support.ino +++ b/Firmware/RTK_Everywhere/support.ino @@ -475,23 +475,33 @@ void printElapsedTime(const char *title) #define TIMESTAMP_INTERVAL 1000 // Milliseconds +// Get the timestamp +const char *getTimeStamp() +{ + static char theTime[30]; + + // 1 2 3 + // 123456789012345678901234567890 + // YYYY-mm-dd HH:MM:SS.xxxrn0 + struct tm timeinfo = rtc.getTimeStruct(); + char timestamp[30]; + strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", &timeinfo); + snprintf(theTime, sizeof(theTime), "%s.%03ld", timestamp, rtc.getMillis()); + + return (const char *)theTime; +} + // Print the timestamp -void printTimeStamp() +void printTimeStamp(bool always) { uint32_t currentMilliseconds; static uint32_t previousMilliseconds; // Timestamp the messages currentMilliseconds = millis(); - if ((currentMilliseconds - previousMilliseconds) >= TIMESTAMP_INTERVAL) + if (always || ((currentMilliseconds - previousMilliseconds) >= TIMESTAMP_INTERVAL)) { - // 1 2 3 - // 123456789012345678901234567890 - // YYYY-mm-dd HH:MM:SS.xxxrn0 - struct tm timeinfo = rtc.getTimeStruct(); - char timestamp[30]; - strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", &timeinfo); - systemPrintf("%s.%03ld\r\n", timestamp, rtc.getMillis()); + systemPrintln(getTimeStamp()); // Select the next time to display the timestamp previousMilliseconds = currentMilliseconds; From 11712418711951fe4696065845ef7357fc9f14a8 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sat, 26 Jul 2025 12:18:42 +0100 Subject: [PATCH 32/39] Resolve #695 ???!!! Restructure pushGPGGA. Add reentry protection Only push GPGGA from ntripClientUpdate, NOT from processUart1Message Revert to ntripClient->read Add settings.haltOnPanic --- Firmware/RTK_Everywhere/GNSS.ino | 55 +++++++++++----- Firmware/RTK_Everywhere/NtripClient.ino | 73 +++++++++++----------- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 7 ++- Firmware/RTK_Everywhere/Tasks.ino | 13 ++-- Firmware/RTK_Everywhere/menuSystem.ino | 4 ++ Firmware/RTK_Everywhere/settings.h | 17 +++++ 6 files changed, 111 insertions(+), 58 deletions(-) diff --git a/Firmware/RTK_Everywhere/GNSS.ino b/Firmware/RTK_Everywhere/GNSS.ino index e3776b075..09719b908 100644 --- a/Firmware/RTK_Everywhere/GNSS.ino +++ b/Firmware/RTK_Everywhere/GNSS.ino @@ -72,29 +72,52 @@ bool GNSS::supportsAntennaShortOpen() } // Periodically push GGA sentence over NTRIP Client, to Caster, if enabled -void pushGPGGA(char *ggaData) +// We must not push to the Caster while we are reading data from the Caster +// See #695 +// pushGPGGA is called by processUart1Message from gnssReadTask +// ntripClient->read is called by ntripClientUpdate and ntripClientResponse from networkUpdate from loop +// We need to make sure processUart1Message doesn't gatecrash +// If ggaData is provided, store it. If ggaData is nullptr, try to push it +static void pushGPGGA(char *ggaData) { -#ifdef COMPILE_NETWORK - // Wait until the client has been created - if (ntripClient != nullptr) + static char storedGPGGA[100]; + + static SemaphoreHandle_t reentrant = xSemaphoreCreateMutex(); // Create the mutex + + if (xSemaphoreTake(reentrant, 10 / portTICK_PERIOD_MS) == pdPASS) { - // Provide the caster with our current position as needed - if (ntripClient->connected() && settings.ntripClient_TransmitGGA == true) + if (ggaData) { - if (millis() - lastGGAPush > NTRIPCLIENT_MS_BETWEEN_GGA) - { - lastGGAPush = millis(); + snprintf(storedGPGGA, sizeof(storedGPGGA), "%s", ggaData); + xSemaphoreGive(reentrant); + return; + } - if ((settings.debugNtripClientRtcm || PERIODIC_DISPLAY(PD_NTRIP_CLIENT_GGA)) && !inMainMenu) +#ifdef COMPILE_NETWORK + // Wait until the client has been created + if (ntripClient != nullptr) + { + // Provide the caster with our current position as needed + if (ntripClient->connected() && settings.ntripClient_TransmitGGA == true) + { + if (millis() - lastGGAPush > NTRIPCLIENT_MS_BETWEEN_GGA) { - PERIODIC_CLEAR(PD_NTRIP_CLIENT_GGA); - systemPrintf("NTRIP Client pushing GGA to server: %s", (const char *)ggaData); - } + lastGGAPush = millis(); - // Push our current GGA sentence to caster - ntripClient->print((const char *)ggaData); + if ((settings.debugNtripClientRtcm || PERIODIC_DISPLAY(PD_NTRIP_CLIENT_GGA)) && !inMainMenu) + { + PERIODIC_CLEAR(PD_NTRIP_CLIENT_GGA); + systemPrintf("NTRIP Client pushing GGA to server: %s", (const char *)storedGPGGA); + } + + // Push our current GGA sentence to caster + if (strlen(storedGPGGA) > 0) + ntripClient->write((const uint8_t *)storedGPGGA, strlen(storedGPGGA)); + } } } - } #endif // COMPILE_NETWORK + + xSemaphoreGive(reentrant); + } } diff --git a/Firmware/RTK_Everywhere/NtripClient.ino b/Firmware/RTK_Everywhere/NtripClient.ino index f00ca3b74..8bc8d8097 100644 --- a/Firmware/RTK_Everywhere/NtripClient.ino +++ b/Firmware/RTK_Everywhere/NtripClient.ino @@ -871,8 +871,7 @@ void ntripClientUpdate() } // Check for timeout receiving NTRIP data - int avail = ntripClientReceiveDataAvailable(); - if (avail <= 0) + if (ntripClientReceiveDataAvailable() == 0) { // Don't fail during retransmission attempts if ((millis() - ntripClientTimer) > NTRIP_CLIENT_RECEIVE_DATA_TIMEOUT) @@ -901,51 +900,55 @@ void ntripClientUpdate() { // Receive data from the NTRIP Caster uint8_t rtcmData[RTCM_DATA_SIZE]; + size_t rtcmCount = 0; - // See #695 - // Rare LWIP ASSERT "(pbuf_free: p->ref > 0)" errors have been seen here - // - which trigger an esp_system_abort (unless CONFIG_LWIP_ESP_LWIP_ASSERT is disabled) - // readBytes seems to perform better and cause fewer crashes than ->read - int rtcmCount = ntripClient->readBytes(rtcmData, avail < RTCM_DATA_SIZE ? avail : RTCM_DATA_SIZE); - if (rtcmCount > 0) + // Collect any available RTCM data + if (ntripClientReceiveDataAvailable() > 0) { - // Restart the NTRIP receive data timer - ntripClientTimer = millis(); - - // Record the arrival of RTCM from the WiFi connection. This resets the RTCM timeout used on the - // L-Band. - rtcmLastPacketReceived = millis(); + rtcmCount = ntripClient->read(rtcmData, sizeof(rtcmData)); + if (rtcmCount) + { + // Restart the NTRIP receive data timer + ntripClientTimer = millis(); - netIncomingRTCM = true; + // Record the arrival of RTCM from the WiFi connection. This resets the RTCM timeout used on the + // L-Band. + rtcmLastPacketReceived = millis(); - if (correctionLastSeen(CORR_TCP)) - { - // Push RTCM to GNSS module over I2C / SPI - gnss->pushRawData(rtcmData, rtcmCount); - sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); // Parse the data for RTCM1005/1006 + netIncomingRTCM = true; - if ((settings.debugCorrections || settings.debugNtripClientRtcm || - PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && - (!inMainMenu)) + if (correctionLastSeen(CORR_TCP)) { - PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); - systemPrintf("NTRIP Client received %d RTCM bytes, pushed to GNSS\r\n", rtcmCount); + // Push RTCM to GNSS module over I2C / SPI + gnss->pushRawData(rtcmData, rtcmCount); + sempParseNextBytes(rtcmParse, rtcmData, rtcmCount); // Parse the data for RTCM1005/1006 + + if ((settings.debugCorrections || settings.debugNtripClientRtcm || + PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && + (!inMainMenu)) + { + PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); + systemPrintf("NTRIP Client received %d RTCM bytes, pushed to GNSS\r\n", rtcmCount); + } } - } - else - { - if ((settings.debugCorrections || settings.debugNtripClientRtcm || - PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && - (!inMainMenu)) + else { - PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); - systemPrintf( - "NTRIP Client received %d RTCM bytes, NOT pushed to GNSS due to priority\r\n", - rtcmCount); + if ((settings.debugCorrections || settings.debugNtripClientRtcm || + PERIODIC_DISPLAY(PD_NTRIP_CLIENT_DATA)) && + (!inMainMenu)) + { + PERIODIC_CLEAR(PD_NTRIP_CLIENT_DATA); + systemPrintf( + "NTRIP Client received %d RTCM bytes, NOT pushed to GNSS due to priority\r\n", + rtcmCount); + } } } } } + + // Now that the ntripClient->read is complete, write GPGGA if needed and available. See #695 + pushGPGGA(nullptr); } break; } diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index c8a05894c..3c08c41bf 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -336,6 +336,8 @@ SemaphoreHandle_t sdCardSemaphore = NULL; TickType_t loggingSemaphoreWait_ms = 10 / portTICK_PERIOD_MS; const TickType_t fatSemaphore_shortWait_ms = 10 / portTICK_PERIOD_MS; const TickType_t fatSemaphore_longWait_ms = 200 / portTICK_PERIOD_MS; +const TickType_t ringBuffer_shortWait_ms = 20 / portTICK_PERIOD_MS; +const TickType_t ringBuffer_longWait_ms = 300 / portTICK_PERIOD_MS; // ringBuffer semaphore - prevent processUart1Message (gnssReadTask) and handleGnssDataTask // from gatecrashing each other. @@ -481,6 +483,7 @@ unsigned long rtcmLastPacketReceived; // Time stamp of RTCM coming in (from BT, bool usbSerialIncomingRtcm; // Incoming RTCM over the USB serial port #define RTCM_CORRECTION_INPUT_TIMEOUT (2 * 1000) + //=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= // Extensible Message Parser @@ -1232,8 +1235,8 @@ void setup() beginVersion(); // Assemble platform name. Requires settings/LFS. - //if (esp_reset_reason() == ESP_RST_PANIC) // Halt on PANIC - to trap rare crashes - // reportFatalError("ESP_RST_PANIC"); + if ((settings.haltOnPanic) && (esp_reset_reason() == ESP_RST_PANIC)) // Halt on PANIC - to trap rare crashes + reportFatalError("ESP_RST_PANIC"); DMW_b("beginGnssUart"); beginGnssUart(); // Requires settings. Start the UART connected to the GNSS receiver on core 0. Start before diff --git a/Firmware/RTK_Everywhere/Tasks.ino b/Firmware/RTK_Everywhere/Tasks.ino index bdf959c71..9e330b2e8 100644 --- a/Firmware/RTK_Everywhere/Tasks.ino +++ b/Firmware/RTK_Everywhere/Tasks.ino @@ -799,7 +799,7 @@ void processUart1Message(SEMP_PARSE_STATE *parse, uint16_t type) // Take the semaphore. Long wait. handleGnssDataTask could block // Enable printing of the ring buffer offsets (s d 10) and the SD buffer sizes (s h 7) // to see this in action. No more gatecrashing! - if (xSemaphoreTake(ringBufferSemaphore, fatSemaphore_longWait_ms) == pdPASS) + if (xSemaphoreTake(ringBufferSemaphore, ringBuffer_longWait_ms) == pdPASS) { ringBufferSemaphoreHolder = "processUart1Message"; @@ -1146,8 +1146,8 @@ void handleGnssDataTask(void *e) if (ringBufferSemaphore == NULL) ringBufferSemaphore = xSemaphoreCreateMutex(); // Create the mutex - // Take the semaphore. Short wait. processUart1Message shouldn't block - if (xSemaphoreTake(ringBufferSemaphore, fatSemaphore_shortWait_ms) == pdPASS) + // Take the semaphore. Short wait. processUart1Message shouldn't block for long + if (xSemaphoreTake(ringBufferSemaphore, ringBuffer_shortWait_ms) == pdPASS) { ringBufferSemaphoreHolder = "handleGnssDataTask"; @@ -2378,8 +2378,11 @@ void beginRtcmParse() if (!rtcmParse) reportFatalError("Failed to initialize the RTCM parser"); - //sempEnableDebugOutput(rtcmParse); - //sempPrintParserConfiguration(rtcmParse); + if (settings.debugNtripClientRtcm) + { + sempEnableDebugOutput(rtcmParse); + sempPrintParserConfiguration(rtcmParse); + } } // Check and record the base location in RTCM1005/1006 diff --git a/Firmware/RTK_Everywhere/menuSystem.ino b/Firmware/RTK_Everywhere/menuSystem.ino index b6dbe71af..adad8513a 100644 --- a/Firmware/RTK_Everywhere/menuSystem.ino +++ b/Firmware/RTK_Everywhere/menuSystem.ino @@ -801,6 +801,8 @@ void menuDebugSoftware() systemPrintf("40) Print LittleFS and settings management: %s\r\n", settings.debugSettings ? "Enabled" : "Disabled"); + systemPrintf("41) Halt on ESP_RST_PANIC: %s\r\n", + settings.haltOnPanic ? "Enabled" : "Disabled"); // Tasks systemPrint("50) Task Highwater Reporting: "); @@ -856,6 +858,8 @@ void menuDebugSoftware() else if (incoming == 40) settings.debugSettings ^= 1; + else if (incoming == 40) + settings.haltOnPanic ^= 1; else if (incoming == 50) settings.enableTaskReports ^= 1; diff --git a/Firmware/RTK_Everywhere/settings.h b/Firmware/RTK_Everywhere/settings.h index 4807dbcfb..31df0ccdf 100644 --- a/Firmware/RTK_Everywhere/settings.h +++ b/Firmware/RTK_Everywhere/settings.h @@ -833,6 +833,7 @@ struct Settings 1; // Read from GNSS and Write to circular buffer (SD, TCP, BT). 3 being the highest, and 0 being the lowest uint8_t gnssUartInterruptsCore = 1; // Core where hardware is started and interrupts are assigned to, 0=core, 1=Arduino + bool haltOnPanic = false; // Halt after beginVersion if the reset reason was panic uint8_t handleGnssDataTaskCore = 1; // Core where task should run, 0=core, 1=Arduino uint8_t handleGnssDataTaskPriority = 1; // Read from the circular buffer and dole out to end points (SD, TCP, BT). uint8_t i2cInterruptsCore = 1; // Core where hardware is started and interrupts are assigned to, 0=core, 1=Arduino @@ -1420,6 +1421,21 @@ const RTK_Settings_Entry rtkSettingsEntries[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, tNSMtPt, NTRIP_SERVER_MAX, & settings.ntripServer_MountPoint[0], "ntripServerMountPoint_", }, { 1, 1, 1, 1, 1, 1, 1, 1, 1, tNSMtPtPw, NTRIP_SERVER_MAX, & settings.ntripServer_MountPointPW[0], "ntripServerMountPointPW_", }, +// F +// a +// F c +// i a e +// n i c t +// W n u e +// e C s F t V P +// b o e a 2 o +// C m S c M s +// o m u e o T L t +// n a f t s o B c +// f n f E a r a a +// i d i v V i c n r +// g s x k 2 c h d d Type Qual Variable Name + // OS { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.bluetoothInterruptsCore, "bluetoothInterruptsCore", }, { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.btReadTaskCore, "btReadTaskCore", }, @@ -1432,6 +1448,7 @@ const RTK_Settings_Entry rtkSettingsEntries[] = { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.gnssReadTaskCore, "gnssReadTaskCore", }, { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.gnssReadTaskPriority, "gnssReadTaskPriority", }, { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.gnssUartInterruptsCore, "gnssUartInterruptsCore", }, + { 0, 1, 0, 1, 1, 1, 1, 1, 1, _bool, 0, & settings.haltOnPanic, "haltOnPanic", }, { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.handleGnssDataTaskCore, "handleGnssDataTaskCore", }, { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.handleGnssDataTaskPriority, "handleGnssDataTaskPriority", }, { 0, 0, 0, 1, 1, 1, 1, 1, 1, _uint8_t, 0, & settings.i2cInterruptsCore, "i2cInterruptsCore", }, From 916a41e83fd3081751785b3b300e1990dd1b8b83 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sat, 26 Jul 2025 12:47:16 +0100 Subject: [PATCH 33/39] Keep the ntripClient alive by pushing GPGGA when the menu is open --- Firmware/RTK_Everywhere/menuSystem.ino | 2 +- Firmware/RTK_Everywhere/support.ino | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/menuSystem.ino b/Firmware/RTK_Everywhere/menuSystem.ino index adad8513a..52b8ec7f1 100644 --- a/Firmware/RTK_Everywhere/menuSystem.ino +++ b/Firmware/RTK_Everywhere/menuSystem.ino @@ -858,7 +858,7 @@ void menuDebugSoftware() else if (incoming == 40) settings.debugSettings ^= 1; - else if (incoming == 40) + else if (incoming == 41) settings.haltOnPanic ^= 1; else if (incoming == 50) diff --git a/Firmware/RTK_Everywhere/support.ino b/Firmware/RTK_Everywhere/support.ino index 55314efa4..ef61e6625 100644 --- a/Firmware/RTK_Everywhere/support.ino +++ b/Firmware/RTK_Everywhere/support.ino @@ -302,6 +302,14 @@ InputResponse getUserInputString(char *userString, uint16_t stringSize, bool loc gnss->update(); // Regularly poll to get latest data + // Keep the ntripClient alive by pushing GPGGA. + // I'm not sure if we want / need to do this, but that's how it was when + // the GPGGA push was performed from processUart1Message from gnssReadTask. + // Doing it here keeps the user experience the same. It is safe to do it here + // because the loop is suspended and networkUpdate / ntripClientUpdate aren't + // being called. Maybe we _should_ call networkUpdate() here? Just sayin'... + pushGPGGA(nullptr); + // Keep processing NTP requests if (online.ethernetNTPServer) { From fbb5e7196e0e0072d3587867a3995bba91815b02 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sat, 26 Jul 2025 14:28:48 +0100 Subject: [PATCH 34/39] Update tpISR comment --- Firmware/RTK_Everywhere/Begin.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Firmware/RTK_Everywhere/Begin.ino b/Firmware/RTK_Everywhere/Begin.ino index 46e48a420..c5ad98578 100644 --- a/Firmware/RTK_Everywhere/Begin.ino +++ b/Firmware/RTK_Everywhere/Begin.ino @@ -1699,7 +1699,7 @@ void tpISR() { if (online.rtc) // Only sync if the RTC has been set via PVT first { - if (timTpUpdated) // Only sync if timTpUpdated is true + if (timTpUpdated) // Only sync if timTpUpdated is true - set by storeTIMTPdata on ZED platforms only { if (millisNow - lastRTCSync > syncRTCInterval) // Only sync if it is more than syncRTCInterval since the last sync From d66c96d6fddeb6ce4466a84106ce0846f8a6e73e Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sat, 26 Jul 2025 17:54:36 +0100 Subject: [PATCH 35/39] Add a semaphore for tcpServer - UNTESTED --- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 5 + Firmware/RTK_Everywhere/TcpServer.ino | 130 +++++++++++++-------- 2 files changed, 86 insertions(+), 49 deletions(-) diff --git a/Firmware/RTK_Everywhere/RTK_Everywhere.ino b/Firmware/RTK_Everywhere/RTK_Everywhere.ino index 3c08c41bf..7faa4139a 100644 --- a/Firmware/RTK_Everywhere/RTK_Everywhere.ino +++ b/Firmware/RTK_Everywhere/RTK_Everywhere.ino @@ -344,6 +344,11 @@ const TickType_t ringBuffer_longWait_ms = 300 / portTICK_PERIOD_MS; SemaphoreHandle_t ringBufferSemaphore = NULL; const char *ringBufferSemaphoreHolder = "None"; +// tcpServer semaphore - prevent tcpServerClientSendData (handleGnssDataTask) and tcpServerUpdate +// from gatecrashing each other. See #695 for why this is needed. +SemaphoreHandle_t tcpServerSemaphore = NULL; +const char *tcpServerSemaphoreHolder = "None"; + // Display used/free space in menu and config page uint64_t sdCardSize; uint64_t sdFreeSpace; diff --git a/Firmware/RTK_Everywhere/TcpServer.ino b/Firmware/RTK_Everywhere/TcpServer.ino index 170f86979..7f4385570 100644 --- a/Firmware/RTK_Everywhere/TcpServer.ino +++ b/Firmware/RTK_Everywhere/TcpServer.ino @@ -139,26 +139,42 @@ int32_t tcpServerClientSendData(int index, uint8_t *data, uint16_t length) { if (tcpServerClient[index]) { - length = tcpServerClient[index]->write(data, length); - if (length > 0) + // Use a semaphore to prevent tcpServerUpdate from gatecrashing + if (tcpServerSemaphore == NULL) + tcpServerSemaphore = xSemaphoreCreateMutex(); // Create the mutex + + // Take the semaphore + if (xSemaphoreTake(tcpServerSemaphore, 50 / portTICK_PERIOD_MS) == pdPASS) { - // Update the data sent flag when data successfully sent + tcpServerSemaphoreHolder = "tcpServerClientSendData"; + + length = tcpServerClient[index]->write(data, length); if (length > 0) - tcpServerClientDataSent = tcpServerClientDataSent | (1 << index); - if ((settings.debugTcpServer || PERIODIC_DISPLAY(PD_TCP_SERVER_CLIENT_DATA)) && (!inMainMenu)) { - PERIODIC_CLEAR(PD_TCP_SERVER_CLIENT_DATA); - systemPrintf("TCP server wrote %d bytes to %s\r\n", length, - tcpServerClientIpAddress[index].toString().c_str()); + // Update the data sent flag when data successfully sent + if (length > 0) + tcpServerClientDataSent = tcpServerClientDataSent | (1 << index); + if ((settings.debugTcpServer || PERIODIC_DISPLAY(PD_TCP_SERVER_CLIENT_DATA)) && (!inMainMenu)) + { + PERIODIC_CLEAR(PD_TCP_SERVER_CLIENT_DATA); + systemPrintf("TCP server wrote %d bytes to %s\r\n", length, + tcpServerClientIpAddress[index].toString().c_str()); + } } - } - // Failed to write the data + // Failed to write the data + else + { + // Done with this client connection + tcpServerStopClient(index); + length = 0; + } + + xSemaphoreGive(tcpServerSemaphore); + } else { - // Done with this client connection - tcpServerStopClient(index); - length = 0; + systemPrintf("tcpServerClientSendData could not get semaphore - held by %s\r\n", tcpServerSemaphoreHolder); } } return length; @@ -553,46 +569,62 @@ void tcpServerUpdate() // and respond accordingly if (settings.enableNtripCaster || settings.baseCasterOverride) { - // Read response from client - char response[512]; - int spot = 0; - while (tcpServerClient[index]->available()) - { - response[spot++] = tcpServerClient[index]->read(); - if (spot == sizeof(response)) - spot = 0; // Wrap - } - response[spot] = '\0'; // Terminate string - - if (strnstr(response, "GET / ", sizeof(response)) != NULL) // No mount point in header - { - if (settings.debugTcpServer) - systemPrintln("Mount point table requested."); - - // Respond with a single mountpoint - const char fakeSourceTable[] = - "SOURCETABLE 200 OK\r\nServer: SparkPNT Caster/1.0\r\nContent-Type: " - "text/plain\r\nContent-Length: 96\r\n\r\nSTR;SparkBase;none;RTCM " - "3.0;none;none;none;none;none;none;none;none;none;none;none;B;N;none;none"; - - tcpServerClient[index]->write(fakeSourceTable, strlen(fakeSourceTable)); - - tcpServerStopClient(index); // Disconnect from client - } - else if (strnstr(response, "GET /", sizeof(response)) != NULL) // Mount point in header + // Use a semaphore to prevent tcpServerClientSendData from gatecrashing + if (tcpServerSemaphore == NULL) + tcpServerSemaphore = xSemaphoreCreateMutex(); // Create the mutex + + // Take the semaphore + if (xSemaphoreTake(tcpServerSemaphore, 50 / portTICK_PERIOD_MS) == pdPASS) { - // NTRIP Client is sending us their mount point. Begin sending RTCM. - if (settings.debugTcpServer) - systemPrintln("NTRIP Client connected - Sending ICY 200 OK"); - - char confirmConnection[] = "ICY 200 OK\r\n"; - tcpServerClient[index]->write(confirmConnection, strlen(confirmConnection)); + tcpServerSemaphoreHolder = "tcpServerUpdate"; + + // Read response from client + char response[512]; + int spot = 0; + while (tcpServerClient[index]->available()) + { + response[spot++] = tcpServerClient[index]->read(); + if (spot == sizeof(response)) + spot = 0; // Wrap + } + response[spot] = '\0'; // Terminate string + + if (strnstr(response, "GET / ", sizeof(response)) != NULL) // No mount point in header + { + if (settings.debugTcpServer) + systemPrintln("Mount point table requested."); + + // Respond with a single mountpoint + const char fakeSourceTable[] = + "SOURCETABLE 200 OK\r\nServer: SparkPNT Caster/1.0\r\nContent-Type: " + "text/plain\r\nContent-Length: 96\r\n\r\nSTR;SparkBase;none;RTCM " + "3.0;none;none;none;none;none;none;none;none;none;none;none;B;N;none;none"; + + tcpServerClient[index]->write(fakeSourceTable, strlen(fakeSourceTable)); + + tcpServerStopClient(index); // Disconnect from client + } + else if (strnstr(response, "GET /", sizeof(response)) != NULL) // Mount point in header + { + // NTRIP Client is sending us their mount point. Begin sending RTCM. + if (settings.debugTcpServer) + systemPrintln("NTRIP Client connected - Sending ICY 200 OK"); + + char confirmConnection[] = "ICY 200 OK\r\n"; + tcpServerClient[index]->write(confirmConnection, strlen(confirmConnection)); + } + else + { + // Unknown response + if (settings.debugTcpServer) + systemPrintf("Unknown response: %s\r\n", response); + } + + xSemaphoreGive(tcpServerSemaphore); } else { - // Unknown response - if (settings.debugTcpServer) - systemPrintf("Unknown response: %s\r\n", response); + systemPrintf("tcpServerUpdate could not get semaphore - held by %s\r\n", tcpServerSemaphoreHolder); } } // settings.enableNtripCaster == true || settings.baseCasterOverride == true From f5204669ad0b11fbcf7b54dc39ced5485821ff04 Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sun, 27 Jul 2025 10:37:22 +0100 Subject: [PATCH 36/39] Notify incomingSettings wrap-around --- Firmware/RTK_Everywhere/WebServer.ino | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Firmware/RTK_Everywhere/WebServer.ino b/Firmware/RTK_Everywhere/WebServer.ino index b225b680e..7588866b0 100644 --- a/Firmware/RTK_Everywhere/WebServer.ino +++ b/Firmware/RTK_Everywhere/WebServer.ino @@ -1335,6 +1335,8 @@ static esp_err_t ws_handler(httpd_req_t *req) for (int i = 0; i < ws_pkt.len; i++) { incomingSettings[incomingSettingsSpot++] = ws_pkt.payload[i]; + if (incomingSettingsSpot == AP_CONFIG_SETTING_SIZE) + systemPrintln("incomingSettings wrap-around. Increase AP_CONFIG_SETTING_SIZE"); incomingSettingsSpot %= AP_CONFIG_SETTING_SIZE; } timeSinceLastIncomingSetting = millis(); From 56de3d221cd501ad6cf77e494fc2d4065c6a100f Mon Sep 17 00:00:00 2001 From: PaulZC Date: Sun, 27 Jul 2025 14:17:20 +0100 Subject: [PATCH 37/39] Add settings.alignedLogFiles - resolves #630 --- Firmware/RTK_Everywhere/AP-Config/index.html | 14 +++++++++--- Firmware/RTK_Everywhere/AP-Config/src/main.js | 4 ++-- Firmware/RTK_Everywhere/RTK_Everywhere.ino | 10 +++++---- Firmware/RTK_Everywhere/menuMessages.ino | 22 ++++++++++++++++++- Firmware/RTK_Everywhere/settings.h | 4 +++- 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/Firmware/RTK_Everywhere/AP-Config/index.html b/Firmware/RTK_Everywhere/AP-Config/index.html index 240705c4f..5ca7044ad 100644 --- a/Firmware/RTK_Everywhere/AP-Config/index.html +++ b/Firmware/RTK_Everywhere/AP-Config/index.html @@ -2016,7 +2016,7 @@ @@ -2027,15 +2027,23 @@
-

+
+ + + + + +
+