diff --git a/include/GTID_Server_Data.h b/include/GTID_Server_Data.h index 0c4ca00cb5..39833e0e87 100644 --- a/include/GTID_Server_Data.h +++ b/include/GTID_Server_Data.h @@ -31,6 +31,6 @@ class GTID_Server_Data { void dump(); }; -bool addGtidInterval(gtid_set_t& gtid_executed, std::string server_uuid, int64_t txid_start, int64_t txid_end); +bool addGtidInterval(const std::string& uuid, const gtid_interval_t &iv, gtid_set_t& gtid_executed); #endif // CLASS_GTID_Server_Data_H diff --git a/include/proxysql_gtid.h b/include/proxysql_gtid.h index 836e95c7a7..2daedd2e68 100644 --- a/include/proxysql_gtid.h +++ b/include/proxysql_gtid.h @@ -2,13 +2,38 @@ #define PROXYSQL_GTID // highly inspired by libslave // https://github.com/vozbu/libslave/ +#include #include #include -#include #include typedef std::pair gtid_t; -typedef std::pair gtid_interval_t; + +class Gtid_Interval { + public: + int64_t start; + int64_t end; + + public: + explicit Gtid_Interval(const int64_t gtid); + explicit Gtid_Interval(const int64_t _start, const int64_t _end); + explicit Gtid_Interval(const char* s); + explicit Gtid_Interval(const std::string& s); + + const std::string to_string(void); + const bool contains(const Gtid_Interval& other); + const bool contains(int64_t gtid); + const bool append(const Gtid_Interval& other); + const bool merge(const Gtid_Interval& other); + + const int cmp(const Gtid_Interval& other); + const bool operator<(const Gtid_Interval& other); + const bool operator==(const Gtid_Interval& other); + const bool operator!=(const Gtid_Interval& other); +}; +typedef Gtid_Interval gtid_interval_t; + +// TODO: make me a proper class. typedef std::unordered_map> gtid_set_t; /* @@ -31,4 +56,4 @@ class Gtid_Server_Info { }; */ -#endif /* PROXYSQL_GTID */ +#endif /* PROXYSQL_GTID */ \ No newline at end of file diff --git a/lib/GTID_Server_Data.cpp b/lib/GTID_Server_Data.cpp index 6fdb3ebc2e..706b01fed9 100644 --- a/lib/GTID_Server_Data.cpp +++ b/lib/GTID_Server_Data.cpp @@ -269,7 +269,7 @@ bool GTID_Server_Data::gtid_exists(char *gtid_uuid, uint64_t gtid_trxid) { return false; } for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) { - if ((int64_t)gtid_trxid >= itr->first && (int64_t)gtid_trxid <= itr->second) { + if (itr->contains((int64_t)gtid_trxid)) { // fprintf(stderr,"YES\n"); return true; } @@ -358,10 +358,9 @@ bool GTID_Server_Data::read_next_gtid() { } } } else { // we are reading the trxids - uint64_t trx_from; - uint64_t trx_to; - sscanf(subtoken,"%lu-%lu",&trx_from,&trx_to); - updated = addGtidInterval(gtid_executed, uuid_server, trx_from, trx_to) || updated; + std::string s = uuid_server; + gtid_interval_t iv = Gtid_Interval(subtoken); + updated = addGtidInterval(s, iv, gtid_executed) || updated; } } } @@ -394,8 +393,8 @@ bool GTID_Server_Data::read_next_gtid() { break; } std::string s = uuid_server; - gtid_t new_gtid = std::make_pair(s,rec_trxid); - addGtid(new_gtid,gtid_executed); + gtid_interval_t iv = Gtid_Interval(rec_trxid); + addGtidInterval(s, iv, gtid_executed); events_read++; } } @@ -412,12 +411,7 @@ std::string gtid_executed_to_string(gtid_set_t& gtid_executed) { s.insert(23,"-"); s = s + ":"; for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) { - std::string s2 = s; - s2 = s2 + std::to_string(itr->first); - s2 = s2 + "-"; - s2 = s2 + std::to_string(itr->second); - s2 = s2 + ","; - gtid_set = gtid_set + s2; + gtid_set += s + itr->to_string() + ","; } } // Extract latest comma only in case 'gtid_executed' isn't empty @@ -428,112 +422,52 @@ std::string gtid_executed_to_string(gtid_set_t& gtid_executed) { } - -void addGtid(const gtid_t& gtid, gtid_set_t& gtid_executed) { - auto it = gtid_executed.find(gtid.first); - if (it == gtid_executed.end()) - { - gtid_executed[gtid.first].emplace_back(gtid.second, gtid.second); - return; +// Merges a GTID interval into a gitd_executed instance. Returns true if gtid_executed was updated, false otherwise. +bool addGtidInterval(const std::string& uuid, const gtid_interval_t &iv, gtid_set_t& gtid_executed) { + auto it = gtid_executed.find(uuid); + if (it == gtid_executed.end()) { + // new UUID entry + gtid_executed[uuid].emplace_back(iv); + return true; } - bool flag = true; - for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) - { - if (gtid.second >= itr->first && gtid.second <= itr->second) - return; - if (gtid.second + 1 == itr->first) - { - --itr->first; - flag = false; - break; - } - else if (gtid.second == itr->second + 1) - { - ++itr->second; - flag = false; - break; - } - else if (gtid.second < itr->first) - { - it->second.emplace(itr, gtid.second, gtid.second); - return; + if (!it->second.empty()) { + if (it->second.back().append(iv)) { + // if appending to the last GTID range succeded, gtid_executed was modified, but remains optimized - nothing else to do + return true; } } - if (flag) - it->second.emplace_back(gtid.second, gtid.second); - - for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) - { - auto next_itr = std::next(itr); - if (next_itr != it->second.end() && itr->second + 1 == next_itr->first) - { - itr->second = next_itr->second; - it->second.erase(next_itr); - break; + // insert/merge GTID interval... + auto pos = it->second.begin(); + for (; pos != it->second.end(); ++pos) { + if (pos->contains(iv)) { + // GTID interval is already present, nothing to do + return false; } + if (pos->merge(iv)) + break; } -} - -/** - * @brief Adds or updates a GTID interval in the executed set - * - * This function intelligently merges GTID intervals to prevent events_count reset - * when a binlog reader reconnects and provides updated GTID sets. It handles - * reconnection scenarios where the server provides updated transaction ID ranges. - * - * For example, during reconnection: - * - Before disconnection: server_UUID:1-10 - * - After reconnection: server_UUID:1-19 - * - * This function will update the existing interval rather than replacing it, - * preserving the events_count metric accuracy. - * - * @param gtid_executed Reference to the GTID set to update - * @param server_uuid The server UUID string - * @param txid_start Starting transaction ID of the interval - * @param txid_end Ending transaction ID of the interval - * @return bool True if the GTID set was updated, false if interval already existed - * - * @note This function is critical for maintaining accurate GTID metrics across - * binlog reader reconnections and preventing events_count resets. - */ -bool addGtidInterval(gtid_set_t& gtid_executed, std::string server_uuid, int64_t txid_start, int64_t txid_end) { - bool updated = true; - - auto it = gtid_executed.find(server_uuid); - if (it == gtid_executed.end()) { - gtid_executed[server_uuid].emplace_back(txid_start, txid_end); - return updated; + if (pos == it->second.end()) { + it->second.emplace_back(iv); } - bool insert = true; - - // When ProxySQL reconnects with binlog reader, it might - // receive updated txid intervals in the bootstrap message. - // For example, - // before disconnection -> server_UUID:1-10 - // after reconnection -> server_UUID:1-19 - auto &txid_intervals = it->second; - for (auto &interval : txid_intervals) { - if (interval.first == txid_start) { - if(interval.second == txid_end) { - updated = false; - } else { - interval.second = txid_end; - } - insert = false; + // ...and merge overlapping GTID ranges, if any + it->second.sort(); + auto a = it->second.begin(); + while (a != it->second.end()) { + auto b = std::next(a); + if (b == it->second.end()) { break; } + if (a->merge(*b)) { + it->second.erase(b); + continue; + } + a++; } - if (insert) { - txid_intervals.emplace_back(txid_start, txid_end); - - } - - return updated; + return true; } void * GTID_syncer_run() { diff --git a/lib/Makefile b/lib/Makefile index 8653118f00..891db8ea4b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -71,6 +71,7 @@ _OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo ProxySQL_Admin_Tests.oo ProxySQL_Admin_Tests2.oo ProxySQL_Admin_Scheduler.oo ProxySQL_Admin_Disk_Upgrade.oo ProxySQL_Admin_Stats.oo \ Admin_Handler.oo Admin_FlushVariables.oo Admin_Bootstrap.oo \ Base_Session.oo Base_Thread.oo \ + proxysql_gtid.oo \ proxy_protocol_info.oo \ proxysql_find_charset.oo ProxySQL_Poll.oo \ PgSQL_Protocol.oo PgSQL_Thread.oo PgSQL_Data_Stream.oo PgSQL_Session.oo PgSQL_Variables.oo PgSQL_HostGroups_Manager.oo PgSQL_Connection.oo PgSQL_Backend.oo PgSQL_Logger.oo PgSQL_Authentication.oo PgSQL_Error_Helper.oo \ diff --git a/lib/proxysql_gtid.cpp b/lib/proxysql_gtid.cpp new file mode 100644 index 0000000000..2262d6c305 --- /dev/null +++ b/lib/proxysql_gtid.cpp @@ -0,0 +1,127 @@ +#include +#include +#include +#include + +#include "proxysql_gtid.h" + + +// Initializes a GTID interval from a single GTID. +Gtid_Interval::Gtid_Interval(const int64_t gtid) { + start = gtid; + end = gtid; +} + +// Initializes a GTID interval from a range. +Gtid_Interval::Gtid_Interval(const int64_t _start, const int64_t _end) { + start = _start; + end = _end; + + if (start > end) { + std::swap(start, end); + } +} + +// Initializes a GTID interval from a string buffer, in [gtid]{-[gtid]} format. +Gtid_Interval::Gtid_Interval(const char *s) { + uint64_t _start = 0, _end = 0; + + if (sscanf(s, "%lu-%lu", &_start, &_end) == 2) { + start = _start; + end = _end; + } else if (sscanf(s, "%lu", &_start) == 1) { + start = _start; + end = _start; + } + + if (start > end) { + std::swap(start, end); + } +} + +Gtid_Interval::Gtid_Interval(const std::string& s) : Gtid_Interval(s.c_str()) { +} + +// Checks if another GTID interval is contained in this one, +const bool Gtid_Interval::contains(const Gtid_Interval& other) { + return (other.start >= start && other.end <= end); +} + +// Checks if a given GTID is contained in this interval. +const bool Gtid_Interval::contains(int64_t gtid) { + return (gtid >= start && gtid <= end); +} + +// Yields a string representation for a GTID interval. +const std::string Gtid_Interval::to_string(void) { + if (start == end) { + return std::to_string(start); + } + return std::to_string(start) + "-" + std::to_string(end); +} + +// Attempts to append a new interval to this interval's end. Returns true if the append succeded, false otherwise. +const bool Gtid_Interval::append(const Gtid_Interval& other) { + if (other.end >= end && other.start <= (end+1)) { + // other overlaps interval at end + end = other.end; + return true; + } + + return false; +} + +// Attempts to merge two GTID intervals. Returns true if the intervals were merged (and potentially modified), false otherwise. +const bool Gtid_Interval::merge(const Gtid_Interval& other) { + if (other.start >= start && other.end <= end) { + // other is contained by interval + return true; + } + if (other.start <= start && other.end >= end) { + // other contains whole of existing interval + start = other.start; + end = other.end; + return true; + } + if (other.start <= start && other.end >= (start-1)) { + // other overlaps interval at start + start = other.start; + return true; + } + if (other.end >= end && other.start <= (end+1)) { + // other overlaps interval at end + end = other.end; + return true; + } + + return false; +} + +// Compares two GTID intervals, by strict weak ordering. +const int Gtid_Interval::cmp(const Gtid_Interval& other) { + if (start < other.start) { + return -1; + } + if (start > other.start) { + return 1; + } + if (end < other.end) { + return -1; + } + if (end > other.end) { + return 1; + } + return 0; +} + +const bool Gtid_Interval::operator<(const Gtid_Interval& other) { + return cmp(other) == -1; +} + +const bool Gtid_Interval::operator==(const Gtid_Interval& other) { + return cmp(other) == 0; +} + +const bool Gtid_Interval::operator!=(const Gtid_Interval& other) { + return cmp(other) != 0; +} diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index 801013cf3a..9c5050fba4 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -93,11 +93,7 @@ CUSTOMARGS += -Wl,-Bdynamic -lcpp_dotenv -lcurl -lssl -lcrypto -lre2 -lpthread - .PHONY: all all: tests -debug: OPT := $(STDCPP) -O0 -DDEBUG -ggdb -Wl,--no-as-needed -Wl,-rpath,$(TAP_LDIR) $(WGCOV) $(WASAN) -DGITVERSION=\"$(GIT_VERSION)\" -debug: tests - -tests: CUSTOMARGS += $(OPT) -tests: tests-cpp \ +tap-tests: tests-cpp \ tests-php \ tests-py \ tests-sh \ @@ -125,6 +121,15 @@ tests: tests-cpp \ fast_forward_switch_replication_deprecate_eof_libmysql-t \ reg_test_mariadb_stmt_store_result_libmysql-t \ reg_test_mariadb_stmt_store_result_async-t + +unit-tests: \ + unit-proxysql_gtid-t + +debug: OPT := $(STDCPP) -O0 -DDEBUG -ggdb -Wl,--no-as-needed -Wl,-rpath,$(TAP_LDIR) $(WGCOV) $(WASAN) -DGITVERSION=\"$(GIT_VERSION)\" +debug: tests + +tests: CUSTOMARGS += $(OPT) +tests: tap-tests unit-tests tests: @echo "Removing empty .gcno files ..." find -L . -type f -name '*.gcno' -empty -ls -delete diff --git a/test/tap/tests/unit-proxysql_gtid-t.cpp b/test/tap/tests/unit-proxysql_gtid-t.cpp new file mode 100644 index 0000000000..07a64ba3e1 --- /dev/null +++ b/test/tap/tests/unit-proxysql_gtid-t.cpp @@ -0,0 +1,114 @@ +#include + +#include "tap.h" +#include "unit_test.h" +#include "proxysql_gtid.h" + +using std::string; + +int testGtidIntervalFromString_Count() { + return 2; +} +void testGtidIntervalFromString() { + ok(gtid_interval_t("123-456") == gtid_interval_t(123, 456), "GTID interval from range string"); + ok(gtid_interval_t("111") == gtid_interval_t(111, 111), "GTID interval from single GTID string"); +} + +int testGtidIntervalContains_Count() { + return 8; +} +void testGtidIntervalContains() { + auto iv = gtid_interval_t(123, 456); + + ok(iv.contains(123), "GTID interval contains start"); + ok(iv.contains(456), "GTID interval contains end"); + ok(iv.contains(300), "GTID interval contains middle"); + ok(!iv.contains(100), "GTID interval doesn't contain before start"); + ok(!iv.contains(500), "GTID interval doesn't contain past end"); + ok(!iv.contains(gtid_interval_t(100, 300)), "GTID interval doesn't contain range before start"); + ok(!iv.contains(gtid_interval_t(300, 500)), "GTID interval doesn't contain range past end"); + ok(iv.contains(gtid_interval_t(150, 310)), "GTID interval contains range"); +} + +int testGtidIntervalAppend_Count() { + return 7; +} +void testGtidIntervalAppend() { + auto iv = gtid_interval_t(123, 456); + + ok(!iv.append(gtid_interval_t(90, 100)), "cannot append before range start"); + ok(!iv.append(gtid_interval_t(100, 200)), "cannot append at start"); + ok(!iv.append(gtid_interval_t(500, 600)), "cannot append past end"); + ok(iv.append(gtid_interval_t(457, 490)), "append"); + ok(iv.to_string() == "123-490", "append result"); + + iv = gtid_interval_t(123, 456); + ok(iv.append(gtid_interval_t(200, 600)), "append with overlap"); + ok(iv.to_string() == "123-600", "append with overlap result"); +} + +int testGtidIntervalMerge_Count() { + return 14; +} +void testGtidIntervalMerge() { + auto iv = gtid_interval_t(123, 456); + ok(!iv.merge(gtid_interval_t(90, 100)), "cannot merge before range start"); + ok(!iv.merge(gtid_interval_t(500, 600)), "cannot merge past range end"); + ok(iv.merge(gtid_interval_t(90, 200)), "merge at start"); + auto want = gtid_interval_t(90, 456); + ok(iv == want, "merge at start result"); + + iv = gtid_interval_t(123, 456); + ok(iv.merge(gtid_interval_t(300, 500)), "merge at end"); + want = gtid_interval_t(123, 500); + ok(iv == want, "merge at end result"); + + iv = gtid_interval_t(123, 456); + ok(iv.merge(gtid_interval_t(200, 300)), "merge at middle"); + want = gtid_interval_t(123, 456); + ok(iv == want, "merge at middle result"); + + iv = gtid_interval_t(123, 456); + ok(iv.merge(gtid_interval_t(100, 500)), "merge overlap"); + want = gtid_interval_t(100, 500); + ok(iv == want, "merge overlap result"); + + iv = gtid_interval_t(123, 456); + ok(iv.merge(gtid_interval_t(100, 122)), "merge append at start"); + want = gtid_interval_t(100, 456); + ok(iv == want, "merge append at start result"); + + iv = gtid_interval_t(123, 456); + want = gtid_interval_t(123, 600); + ok(iv.merge(gtid_interval_t(457, 600)), "merge append at end"); + ok(iv == want, "merge append at end result"); +} + +std::function testFunctionCounts[] = { + testGtidIntervalFromString_Count, + testGtidIntervalContains_Count, + testGtidIntervalAppend_Count, + testGtidIntervalMerge_Count, +}; +std::function testFunctions[] = { + testGtidIntervalFromString, + testGtidIntervalContains, + testGtidIntervalAppend, + testGtidIntervalMerge, +}; + +int main(int argc, char** argv) { + // Set up unit tests... + int n = 0; + for (auto f : testFunctionCounts) { + n += f(); + } + plan(n); + + // ...and run them. + for (auto f : testFunctions) { + f(); + } + + return exit_status(); +} \ No newline at end of file