diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 3f07a7f991a9a..c5a82a33d6060 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -288,7 +288,6 @@ Jolokia JSONAs jsonify jsonlines -jszwedko kebabcase kernelmode keyclock @@ -306,6 +305,7 @@ landingpad leebenson leveldb libcrypto +libmaodbc librdkafka libvector lld @@ -329,6 +329,7 @@ majorly makecache Makefiles mallocs +mariadb markdownify marketo maxbin @@ -402,6 +403,7 @@ nullishness NUMA numbackends NXRR +odbcinst OIDC OKD oneof @@ -530,6 +532,9 @@ spencergilbert spinlock SPOF spog +SQLLEN +SQLSETPOSIROW +SQLULEN sqlx srcaddr srcport @@ -613,6 +618,7 @@ underutilizing unevictable unflatten unioning +unixodbc unnested upgradable urql @@ -672,5 +678,4 @@ zieme zoog zork zorp -zsherman zulip diff --git a/Cargo.lock b/Cargo.lock index c04a3cce92cfe..2c8d757bd25fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,33 @@ dependencies = [ "url", ] +[[package]] +name = "android-activity" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6978589202a00cd7e118380c448a08b6ed394c3a8df3a430d0898e3a42d046" +dependencies = [ + "android-properties", + "bitflags 2.9.0", + "cc", + "cesu8", + "jni", + "jni-sys", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum 0.7.3", + "thiserror 1.0.68", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -1791,6 +1818,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2", +] + [[package]] name = "blocking" version = "1.4.1" @@ -1848,7 +1884,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tokio-util", "tower-service", @@ -2020,6 +2056,20 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5c434ae3cf0089ca203e9019ebe529c47ff45cefe8af7c85ecb734ef541822f" +[[package]] +name = "calloop" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b99da2f8558ca23c71f4fd15dc57c906239752dd27ff3c00a1d56b685b7cbfec" +dependencies = [ + "bitflags 2.9.0", + "log", + "polling 3.7.4", + "rustix 0.38.40", + "slab", + "thiserror 1.0.68", +] + [[package]] name = "cargo-lock" version = "10.1.0" @@ -2146,7 +2196,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -2453,7 +2503,7 @@ checksum = "f29222b549d4e3ded127989d523da9e928918d0d0d7f7c1690b439d0d538bae9" dependencies = [ "directories", "serde", - "thiserror 2.0.3", + "thiserror 2.0.17", "toml 0.8.23", ] @@ -2612,6 +2662,30 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970a29baf4110c26fedbc7f82107d42c23f7e88e404c4577ed73fe99ff85a212" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.3", + "core-graphics-types", + "foreign-types 0.5.0", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb142d41022986c1d8ff29103a1411c8a3dfad3552f87a4f8dc50d61d4f4e33" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.3", + "libc", +] + [[package]] name = "core2" version = "0.4.0" @@ -2712,6 +2786,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" +[[package]] +name = "cron" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" +dependencies = [ + "chrono", + "once_cell", + "serde", + "winnow 0.6.26", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -2893,6 +2979,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "cursor-icon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -3221,6 +3313,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + [[package]] name = "displaydoc" version = "0.2.5" @@ -3232,6 +3330,15 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "dlib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" +dependencies = [ + "libloading", +] + [[package]] name = "dns-lookup" version = "2.0.4" @@ -3341,6 +3448,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dpi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" + [[package]] name = "duct" version = "0.13.6" @@ -3923,7 +4036,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -3932,6 +4066,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -4692,7 +4832,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustls-pki-types", - "thiserror 2.0.3", + "thiserror 2.0.17", "time", "tinyvec", "tracing 0.1.41", @@ -5436,7 +5576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ "socket2 0.5.10", - "widestring 1.0.2", + "widestring 1.2.1", "windows-sys 0.48.0", "winreg", ] @@ -5929,6 +6069,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libm" version = "0.2.8" @@ -5943,6 +6093,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", + "redox_syscall 0.5.12", ] [[package]] @@ -6064,9 +6215,9 @@ checksum = "3a69c0481fc2424cb55795de7da41add33372ea75a94f9b6588ab6a2826dfebc" [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "loki-logproto" @@ -6221,7 +6372,7 @@ dependencies = [ "memchr", "serde", "simdutf8", - "thiserror 2.0.3", + "thiserror 2.0.17", ] [[package]] @@ -6557,12 +6708,36 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.9.0", + "jni-sys", + "log", + "ndk-sys", + "num_enum 0.7.3", + "raw-window-handle 0.6.2", + "thiserror 1.0.68", +] + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys", +] + [[package]] name = "netlink-packet-core" version = "0.7.0" @@ -7029,6 +7204,74 @@ dependencies = [ "malloc_buf", ] +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.9.0", + "block2", + "libc", + "objc2", + "objc2-core-data", + "objc2-core-image", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" +dependencies = [ + "bitflags 2.9.0", + "block2", + "objc2", + "objc2-core-location", + "objc2-foundation", +] + +[[package]] +name = "objc2-contacts" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.9.0", + "block2", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-core-foundation" version = "0.3.1" @@ -7038,6 +7281,49 @@ dependencies = [ "bitflags 2.9.0", ] +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "objc2-core-location" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781" +dependencies = [ + "block2", + "objc2", + "objc2-contacts", + "objc2-foundation", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.9.0", + "block2", + "dispatch", + "libc", + "objc2", +] + [[package]] name = "objc2-io-kit" version = "0.3.1" @@ -7048,6 +7334,98 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-link-presentation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" +dependencies = [ + "block2", + "objc2", + "objc2-app-kit", + "objc2-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.9.0", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.9.0", + "block2", + "objc2", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "objc2-symbols" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a684efe3dec1b305badae1a28f6555f6ddd3bb2c2267896782858d5a78404dc" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-ui-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" +dependencies = [ + "bitflags 2.9.0", + "block2", + "objc2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-image", + "objc2-core-location", + "objc2-foundation", + "objc2-link-presentation", + "objc2-quartz-core", + "objc2-symbols", + "objc2-uniform-type-identifiers", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-uniform-type-identifiers" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" +dependencies = [ + "bitflags 2.9.0", + "block2", + "objc2", + "objc2-core-location", + "objc2-foundation", +] + [[package]] name = "object" version = "0.36.7" @@ -7068,6 +7446,26 @@ dependencies = [ "smallvec", ] +[[package]] +name = "odbc-api" +version = "19.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f017d3949731e436bc1bb9a1fbc34197c2f39c588cdcb60d21adb1f8dd3b8514" +dependencies = [ + "atoi", + "log", + "odbc-sys", + "thiserror 2.0.17", + "widestring 1.2.1", + "winit", +] + +[[package]] +name = "odbc-sys" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7e3c4b5b7bbd3e7bd01dc00cb4614f2445591cad1f6f18a7e16d7f98c392e9" + [[package]] name = "ofb" version = "0.6.1" @@ -7187,7 +7585,7 @@ checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ "bitflags 2.9.0", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -7257,6 +7655,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "orbclient" +version = "0.3.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba0b26cec2e24f08ed8bb31519a9333140a6599b867dac464bb150bdb796fd43" +dependencies = [ + "libredox", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -8274,7 +8681,7 @@ dependencies = [ "newtype-uuid", "quick-xml 0.37.4", "strip-ansi-escapes", - "thiserror 2.0.3", + "thiserror 2.0.17", "uuid", ] @@ -8333,7 +8740,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.23", "socket2 0.5.10", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tracing 0.1.41", ] @@ -8352,7 +8759,7 @@ dependencies = [ "rustls 0.23.23", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.17", "tinyvec", "tracing 0.1.41", "web-time", @@ -8568,6 +8975,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + [[package]] name = "rawpointer" version = "0.2.1" @@ -8713,7 +9126,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror 2.0.3", + "thiserror 2.0.17", ] [[package]] @@ -10048,6 +10461,15 @@ dependencies = [ "futures-lite 1.13.0", ] +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + [[package]] name = "smpl_jwt" version = "0.8.0" @@ -10220,7 +10642,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing 0.1.41", @@ -10303,7 +10725,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.3", + "thiserror 2.0.17", "tracing 0.1.41", "whoami", ] @@ -10341,7 +10763,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.3", + "thiserror 2.0.17", "tracing 0.1.41", "whoami", ] @@ -10366,7 +10788,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.3", + "thiserror 2.0.17", "tracing 0.1.41", "url", ] @@ -10755,11 +11177,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.17", ] [[package]] @@ -10775,9 +11197,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2 1.0.101", "quote 1.0.40", @@ -12101,6 +12523,7 @@ dependencies = [ "colored", "console-subscriber", "criterion", + "cron", "csv", "databend-client", "deadpool", @@ -12165,6 +12588,7 @@ dependencies = [ "nkeys", "nom 8.0.0", "notify", + "odbc-api", "opendal", "openssl", "openssl-probe", @@ -12728,7 +13152,7 @@ dependencies = [ "strip-ansi-escapes", "syslog_loose 0.22.0", "termcolor", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tracing 0.1.41", "ua-parser", @@ -12874,12 +13298,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.38" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] @@ -12976,7 +13401,7 @@ dependencies = [ "log", "ndk-context", "objc", - "raw-window-handle", + "raw-window-handle 0.5.2", "url", "web-sys", ] @@ -13040,9 +13465,9 @@ checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" [[package]] name = "widestring" -version = "1.0.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -13094,7 +13519,7 @@ dependencies = [ "windows-collections", "windows-core 0.60.1", "windows-future", - "windows-link", + "windows-link 0.1.0", "windows-numerics", ] @@ -13124,7 +13549,7 @@ checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" dependencies = [ "windows-implement", "windows-interface", - "windows-link", + "windows-link 0.1.0", "windows-result 0.3.1", "windows-strings 0.3.1", ] @@ -13136,7 +13561,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" dependencies = [ "windows-core 0.60.1", - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13167,6 +13592,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-numerics" version = "0.1.1" @@ -13174,7 +13605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" dependencies = [ "windows-core 0.60.1", - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13203,7 +13634,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" dependencies = [ - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13213,7 +13644,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "193cae8e647981c35bc947fdd57ba7928b1fa0d4a79305f6dd2dc55221ac35ac" dependencies = [ "bitflags 2.9.0", - "widestring 1.0.2", + "widestring 1.2.1", "windows-sys 0.59.0", ] @@ -13233,7 +13664,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13523,6 +13954,46 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winit" +version = "0.30.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66d4b9ed69c4009f6321f762d6e61ad8a2389cd431b97cb1e146812e9e6c732" +dependencies = [ + "android-activity", + "atomic-waker", + "bitflags 2.9.0", + "block2", + "calloop", + "cfg_aliases", + "concurrent-queue", + "core-foundation 0.9.3", + "core-graphics", + "cursor-icon", + "dpi", + "js-sys", + "libc", + "ndk", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "objc2-ui-kit", + "orbclient", + "pin-project", + "raw-window-handle 0.6.2", + "redox_syscall 0.4.1", + "rustix 0.38.40", + "smol_str", + "tracing 0.1.41", + "unicode-segmentation", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "web-time", + "windows-sys 0.52.0", + "xkbcommon-dl", +] + [[package]] name = "winnow" version = "0.5.18" @@ -13532,6 +14003,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.10" @@ -13614,6 +14094,25 @@ dependencies = [ "tap", ] +[[package]] +name = "xkbcommon-dl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" +dependencies = [ + "bitflags 2.9.0", + "dlib", + "log", + "once_cell", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 14e1e219f1297..dcacb706879a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -328,6 +328,10 @@ deadpool = { version = "0.12.2", default-features = false, features = ["managed" async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] } async-graphql-warp = { version = "7.0.17", default-features = false, optional = true } +# ODBC +odbc-api = { version = "19.1.0", optional = true } +cron = { version = "0.15.0", features = ["serde"], optional = true } + # Opentelemetry hex = { version = "0.4.3", default-features = false, optional = true } @@ -612,6 +616,7 @@ sources-logs = [ "sources-logstash", "sources-mqtt", "sources-nats", + "sources-odbc", "sources-okta", "sources-opentelemetry", "sources-pulsar", @@ -672,6 +677,7 @@ sources-mongodb_metrics = ["dep:mongodb"] sources-mqtt = ["dep:rumqttc"] sources-nats = ["dep:async-nats", "dep:nkeys"] sources-nginx_metrics = ["dep:nom"] +sources-odbc = ["dep:odbc-api", "dep:cron"] sources-okta = ["sources-utils-http-client"] sources-opentelemetry = [ "dep:hex", @@ -922,6 +928,7 @@ all-integration-tests = [ "mqtt-integration-tests", "nats-integration-tests", "nginx-integration-tests", + "odbc-integration-tests", "opentelemetry-integration-tests", "postgresql_metrics-integration-tests", "postgres_sink-integration-tests", @@ -987,6 +994,7 @@ mongodb_metrics-integration-tests = ["sources-mongodb_metrics"] mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"] nats-integration-tests = ["sinks-nats", "sources-nats"] nginx-integration-tests = ["sources-nginx_metrics"] +odbc-integration-tests = ["sources-odbc"] opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"] postgresql_metrics-integration-tests = ["sources-postgresql_metrics"] postgres_sink-integration-tests = ["sinks-postgres"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 6967fbd458500..f06c705da6fcf 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -16,6 +16,8 @@ amq-protocol,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine P amq-protocol-tcp,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> amq-protocol-types,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> amq-protocol-uri,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> +android-activity,https://github.com/rust-mobile/android-activity,MIT OR Apache-2.0,The android-activity Authors +android-properties,https://github.com/miklelappo/android-properties,MIT,Mikhail Lappo android-tzdata,https://github.com/RumovZ/android-tzdata,MIT OR Apache-2.0,RumovZ android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva ansi_term,https://github.com/ogham/rust-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett " @@ -113,6 +115,7 @@ bitmask-enum,https://github.com/Lukas3674/rust-bitmask-enum,MIT OR Apache-2.0,Lu bitvec,https://github.com/bitvecto-rs/bitvec,MIT,The bitvec Authors block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +block2,https://github.com/madsmtm/objc2,MIT,"Steven Sheldon, Mads Marquart " blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors @@ -132,6 +135,7 @@ bytes,https://github.com/carllerche/bytes,MIT,Carl Lerche bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " bytes-utils,https://github.com/vorner/bytes-utils,Apache-2.0 OR MIT,Michal 'vorner' Vaner bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,"Hyunsik Choi , MrCroxx , Rob Ede " +calloop,https://github.com/Smithay/calloop,MIT,Elinor Berger cassowary,https://github.com/dylanede/cassowary-rs,MIT OR Apache-2.0,Dylan Ede castaway,https://github.com/sagebind/castaway,MIT,Stephen M. Coakley cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers @@ -172,6 +176,8 @@ cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick F core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-graphics,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-graphics-types,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core2,https://github.com/bbqsrc/core2,Apache-2.0 OR MIT,Brendan Molloy cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -180,6 +186,7 @@ crc32c,https://github.com/zowens/crc32c,Apache-2.0 OR MIT,Zack Owens crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs , Alex Crichton " crc64fast-nvme,https://github.com/awesomized/crc64fast-nvme,MIT OR Apache-2.0,"The TiKV Project Developers, Don MacAskill" critical-section,https://github.com/rust-embedded/critical-section,MIT OR Apache-2.0,The critical-section Authors +cron,https://github.com/zslayton/cron,MIT OR Apache-2.0,Zack Slayton crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors crossbeam-epoch,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-epoch Authors crossbeam-queue,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-queue Authors @@ -194,6 +201,7 @@ csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton +cursor-icon,https://github.com/rust-windowing/cursor-icon,MIT OR Apache-2.0 OR Zlib,Kirill Chibisov curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs @@ -218,13 +226,16 @@ derive_more,https://github.com/JelteF/derive_more,MIT,Jelte Fennema +dlib,https://github.com/elinorbgr/dlib,MIT,Elinor Berger dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh Driver doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez document-features,https://github.com/slint-ui/document-features,MIT OR Apache-2.0,Slint Developers domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs domain-macros,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " +dpi,https://github.com/rust-windowing/winit,Apache-2.0 AND MIT,The dpi Authors dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 OR MIT,RustCrypto Developers @@ -268,6 +279,7 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +foreign-types-macros,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 @@ -405,7 +417,9 @@ lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2. libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers libflate,https://github.com/sile/libflate,MIT,Takeru Ohta libflate_lz77,https://github.com/sile/libflate,MIT,Takeru Ohta +libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio +libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com> libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers libz-rs-sys,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The libz-rs-sys Authors libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett , Sebastian Thiel " @@ -450,7 +464,9 @@ moka,https://github.com/moka-rs/moka,MIT OR Apache-2.0,The moka Authors mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar " multer,https://github.com/rousan/multer-rs,MIT,Rousan Ali native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler +ndk,https://github.com/rust-mobile/ndk,MIT OR Apache-2.0,The Rust Mobile contributors ndk-context,https://github.com/rust-windowing/android-ndk-rs,MIT OR Apache-2.0,The Rust Windowing contributors +ndk-sys,https://github.com/rust-mobile/ndk,MIT OR Apache-2.0,The Rust Windowing contributors netlink-packet-core,https://github.com/rust-netlink/netlink-packet-core,MIT,Corentin Henry netlink-packet-sock-diag,https://github.com/rust-netlink/netlink-packet-sock-diag,MIT,"Flier Lu , Corentin Henry " netlink-packet-utils,https://github.com/rust-netlink/netlink-packet-utils,MIT,Corentin Henry @@ -488,10 +504,29 @@ num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt number_prefix,https://github.com/ogham/rust-number-prefix,MIT,Benjamin Sago oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " objc,http://github.com/SSheldon/rust-objc,MIT,Steven Sheldon +objc-sys,https://github.com/madsmtm/objc2,MIT,Mads Marquart +objc2,https://github.com/madsmtm/objc2,MIT,"Steven Sheldon, Mads Marquart " +objc2-app-kit,https://github.com/madsmtm/objc2,MIT,The objc2-app-kit Authors +objc2-cloud-kit,https://github.com/madsmtm/objc2,MIT,The objc2-cloud-kit Authors +objc2-contacts,https://github.com/madsmtm/objc2,MIT,The objc2-contacts Authors +objc2-core-data,https://github.com/madsmtm/objc2,MIT,The objc2-core-data Authors objc2-core-foundation,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-core-foundation Authors +objc2-core-image,https://github.com/madsmtm/objc2,MIT,The objc2-core-image Authors +objc2-core-location,https://github.com/madsmtm/objc2,MIT,The objc2-core-location Authors +objc2-encode,https://github.com/madsmtm/objc2,MIT,Mads Marquart +objc2-foundation,https://github.com/madsmtm/objc2,MIT,The objc2-foundation Authors objc2-io-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-io-kit Authors +objc2-link-presentation,https://github.com/madsmtm/objc2,MIT,The objc2-link-presentation Authors +objc2-metal,https://github.com/madsmtm/objc2,MIT,The objc2-metal Authors +objc2-quartz-core,https://github.com/madsmtm/objc2,MIT,The objc2-quartz-core Authors +objc2-symbols,https://github.com/madsmtm/objc2,MIT,The objc2-symbols Authors +objc2-ui-kit,https://github.com/madsmtm/objc2,MIT,The objc2-ui-kit Authors +objc2-uniform-type-identifiers,https://github.com/madsmtm/objc2,MIT,The objc2-uniform-type-identifiers Authors +objc2-user-notifications,https://github.com/madsmtm/objc2,MIT,The objc2-user-notifications Authors object,https://github.com/gimli-rs/object,Apache-2.0 OR MIT,The object Authors octseq,https://github.com/NLnetLabs/octets/,BSD-3-Clause,NLnet Labs +odbc-api,https://github.com/pacman82/odbc-api,MIT,Markus Klein +odbc-sys,https://github.com/pacman82/odbc-sys,MIT,Markus Klein ofb,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " @@ -503,6 +538,7 @@ openssl,https://github.com/sfackler/rust-openssl,Apache-2.0,Steven Fackler openssl-sys,https://github.com/sfackler/rust-openssl,MIT,"Alex Crichton , Steven Fackler " +orbclient,https://gitlab.redox-os.org/redox-os/orbclient,MIT,Jeremy Soller ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem , Matt Brubeck " outref,https://github.com/Nugine/outref,MIT,The outref Authors owo-colors,https://github.com/owo-colors/owo-colors,MIT,jam1garner <8260240+jam1garner@users.noreply.github.com> @@ -687,6 +723,7 @@ sketches-ddsketch,https://github.com/mheffner/rust-sketches-ddsketch,Apache-2.0, slab,https://github.com/tokio-rs/slab,MIT,Carl Lerche smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Project Developers smol,https://github.com/smol-rs/smol,Apache-2.0 OR MIT,Stjepan Glavina +smol_str,https://github.com/rust-analyzer/smol_str,MIT OR Apache-2.0,Aleksey Kladov smpl_jwt,https://github.com/durch/rust-jwt,MIT,Drazen Urch snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding @@ -840,8 +877,8 @@ web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Aut webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors +widestring,https://github.com/VoidStarKat/widestring-rs,MIT OR Apache-2.0,The widestring Authors widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long -widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The widestring Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant @@ -853,6 +890,7 @@ windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The win windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-link Authors windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors windows-registry,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-result,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft @@ -868,6 +906,7 @@ windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Micr windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +winit,https://github.com/rust-windowing/winit,Apache-2.0,"The winit contributors, Pierre Krieger " winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula wit-bindgen-rt,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wit-bindgen-rt Authors @@ -875,6 +914,8 @@ woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto +xkbcommon-dl,https://github.com/rust-windowing/xkbcommon-dl,MIT,Francesca Frangipane +xkeysym,https://github.com/notgull/xkeysym,MIT OR Apache-2.0 OR Zlib,John Nunley xmlparser,https://github.com/RazrFalcon/xmlparser,MIT OR Apache-2.0,Yevhenii Reizner xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yoke,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar diff --git a/changelog.d/24044_odbc_source.feature.md b/changelog.d/24044_odbc_source.feature.md new file mode 100644 index 0000000000000..e933806cfae20 --- /dev/null +++ b/changelog.d/24044_odbc_source.feature.md @@ -0,0 +1,3 @@ +Add a new ODBC(Open Database Connectivity) source. + +authors: powerumc \ No newline at end of file diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index a77c6ce409a5e..8bd7c8048ff75 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -14,8 +14,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsasl2-dev \ libssl-dev \ libxxhash-dev \ + odbcinst \ + odbc-mariadb \ + odbc-postgresql \ zlib1g-dev \ zlib1g \ + unixodbc \ unzip \ mold \ && rm -rf /var/lib/apt/lists/* diff --git a/scripts/integration/odbc-mariadb/compose.yaml b/scripts/integration/odbc-mariadb/compose.yaml new file mode 100644 index 0000000000000..44344631a2763 --- /dev/null +++ b/scripts/integration/odbc-mariadb/compose.yaml @@ -0,0 +1,17 @@ +version: '3' + +services: + mariadb: + image: docker.io/mariadb:${CONFIG_VERSION} + ports: + - "3306:3306" + environment: + - MYSQL_USER=vector + - MYSQL_PASSWORD=vector + - MYSQL_ROOT_PASSWORD=vector + - MYSQL_DATABASE=vector_db + +networks: + default: + name: ${VECTOR_NETWORK} + external: true diff --git a/scripts/integration/odbc-mariadb/test.yaml b/scripts/integration/odbc-mariadb/test.yaml new file mode 100644 index 0000000000000..04943957f8ddc --- /dev/null +++ b/scripts/integration/odbc-mariadb/test.yaml @@ -0,0 +1,17 @@ +features: + - odbc-integration-tests + +test_filter: '::odbc::' + +env: + ODBC_DB_TYPE: "mariadb" + ODBC_CONN_STRING: "driver={MariaDB Unicode};server=mariadb;port=3306;database=vector_db;uid=vector;pwd=vector;" + +matrix: + version: [ '11-jammy' ] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: + - "src/sources/odbc/**" + - "scripts/integration/odbc/**" diff --git a/scripts/integration/odbc-postgresql/compose.yaml b/scripts/integration/odbc-postgresql/compose.yaml new file mode 100644 index 0000000000000..556a709aac59f --- /dev/null +++ b/scripts/integration/odbc-postgresql/compose.yaml @@ -0,0 +1,16 @@ +version: '3' + +services: + postgresql: + image: docker.io/postgres:${CONFIG_VERSION} + ports: + - "5432:5432" + environment: + - POSTGRES_USER=vector + - POSTGRES_PASSWORD=vector + - POSTGRES_DB=vector_db + +networks: + default: + name: ${VECTOR_NETWORK} + external: true diff --git a/scripts/integration/odbc-postgresql/test.yaml b/scripts/integration/odbc-postgresql/test.yaml new file mode 100644 index 0000000000000..8fb2cdcffc507 --- /dev/null +++ b/scripts/integration/odbc-postgresql/test.yaml @@ -0,0 +1,17 @@ +features: + - odbc-integration-tests + +test_filter: '::odbc::' + +env: + ODBC_DB_TYPE: "postgresql" + ODBC_CONN_STRING: "driver={PostgreSQL Unicode};server=postgresql;port=5432;database=vector_db;uid=vector;pwd=vector;" + +matrix: + version: [ '16' ] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: + - "src/sources/odbc/**" + - "scripts/integration/odbc/**" diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 1647e2b8ff0ca..94d483282f74a 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -152,6 +152,9 @@ mod file; #[cfg(windows)] mod windows; +#[cfg(feature = "sources-odbc")] +mod odbc_metrics; + pub mod config; #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] mod expansion; @@ -248,6 +251,9 @@ pub(crate) use self::metric_to_log::*; pub(crate) use self::mqtt::*; #[cfg(feature = "sources-nginx_metrics")] pub(crate) use self::nginx_metrics::*; +#[cfg(feature = "sources-odbc")] +pub(crate) use self::odbc_metrics::*; +#[allow(unused_imports)] #[cfg(any( feature = "sources-kubernetes_logs", feature = "transforms-log_to_metric", diff --git a/src/internal_events/odbc_metrics.rs b/src/internal_events/odbc_metrics.rs new file mode 100644 index 0000000000000..5ec38826d8af6 --- /dev/null +++ b/src/internal_events/odbc_metrics.rs @@ -0,0 +1,64 @@ +use metrics::counter; +use vector_common::internal_event::{InternalEvent, error_type}; + +#[derive(Debug)] +pub struct OdbcEventsReceived { + pub count: usize, +} + +impl InternalEvent for OdbcEventsReceived { + fn emit(self) { + trace!( + message = "Events received.", + count = %self.count, + ); + counter!( + "component_received_events_total", + "protocol" => "odbc" + ) + .increment(self.count as u64); + counter!( + "component_received_event_bytes_total", + "protocol" => "odbc" + ) + .increment(0); + } +} + +#[derive(Debug)] +pub struct OdbcFailedError<'a> { + pub statement: &'a str, +} + +impl InternalEvent for OdbcFailedError<'_> { + fn emit(self) { + error!( + message = "Unable to execute statement.", + statement = %self.statement, + error = error_type::COMMAND_FAILED + ); + counter!( + "component_errors_total", + "statement" => self.statement.to_owned(), + "error_type" => error_type::COMMAND_FAILED + ) + .increment(1); + } +} + +#[derive(Debug)] +pub struct OdbcQueryExecuted<'a> { + pub statement: &'a str, + pub elapsed: u128, +} + +impl InternalEvent for OdbcQueryExecuted<'_> { + fn emit(self) { + trace!( + message = "Executed statement.", + statement = %self.statement, + elapsedMs = %self.elapsed + ); + counter!("component_executed_events_total").increment(1); + } +} diff --git a/src/sources/mod.rs b/src/sources/mod.rs index f7b2b6bb534e8..ec5570cd489e3 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -64,6 +64,8 @@ pub mod mqtt; pub mod nats; #[cfg(feature = "sources-nginx_metrics")] pub mod nginx_metrics; +#[cfg(feature = "sources-odbc")] +pub mod odbc; #[cfg(feature = "sources-okta")] pub mod okta; #[cfg(feature = "sources-opentelemetry")] diff --git a/src/sources/odbc/README.md b/src/sources/odbc/README.md new file mode 100644 index 0000000000000..9d7d8448a46d0 --- /dev/null +++ b/src/sources/odbc/README.md @@ -0,0 +1,46 @@ +# ODBC Development + +## Setup + +#### MacOS + +```shell +brew install unixodbc + +brew install mariadb-connector-odbc@3.2.6 # Install MariaDB ODBC, and you need to configure odbcinst.ini. +``` + +Refs + +- MySQL Connector/ODBC: +- MariaDB Connector/ODBC: + - Homebrew mariadb-connector-odbc: + - ODBC Configuration: + ```shell + cat << EOF >> /opt/homebrew/etc/odbcinst.ini + + [MariaDB ODBC 3.0 Driver] + Description = MariaDB Connector/ODBC v.3.0 + Driver = /opt/homebrew/Cellar/mariadb-connector-odbc/3.2.6/lib/mariadb/libmaodbc.dylib + EOF + ``` +- MSSQL + Connector/ODBC: + +## ODBC Tips + +Show ODBC configuration + +```shell +odbcinst -j + +### Output Example ### +# unixODBC 2.3.12 +# DRIVERS............: /opt/homebrew/etc/odbcinst.ini +# SYSTEM DATA SOURCES: /opt/homebrew/etc/odbc.ini +# FILE DATA SOURCES..: /opt/homebrew/etc/ODBCDataSources +# USER DATA SOURCES..: /Users//.odbc.ini +# SQLULEN Size.......: 8 +# SQLLEN Size........: 8 +# SQLSETPOSIROW Size.: 8 +``` diff --git a/src/sources/odbc/client.rs b/src/sources/odbc/client.rs new file mode 100644 index 0000000000000..092aa3ca84cc9 --- /dev/null +++ b/src/sources/odbc/client.rs @@ -0,0 +1,534 @@ +use crate::config::{LogNamespace, SourceConfig, SourceContext, SourceOutput, log_schema}; +use crate::internal_events::{OdbcEventsReceived, OdbcFailedError, OdbcQueryExecuted}; +use crate::serde::default_decoding; +use crate::sinks::prelude::*; +use crate::sources::Source; +use crate::sources::odbc::{ + ClosedSnafu, Column, Columns, ConfigSnafu, DbSnafu, OdbcError, OdbcSchedule, Rows, load_params, + map_value, order_params, save_params, +}; +use chrono::Utc; +use chrono_tz::Tz; +use futures::pin_mut; +use futures_util::StreamExt; +use itertools::Itertools; +use odbc_api::buffers::TextRowSet; +use odbc_api::parameter::VarCharBox; +use odbc_api::{ConnectionOptions, Cursor, Environment, ResultSetMetadata}; +use serde_with::DurationSeconds; +use serde_with::serde_as; +use snafu::{OptionExt, ResultExt}; +use std::fmt::Debug; +use std::fs; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::select; +use typetag::serde; +use vector_common::internal_event::{BytesReceived, Protocol}; +use vector_lib::codecs::decoding::DeserializerConfig; +use vector_lib::emit; +use vrl::prelude::*; + +/// Configuration for the `odbc` source. +#[serde_as] +#[configurable_component(source( + "odbc", + "Periodically pulls observability data from an ODBC interface by running a scheduled query." +))] +#[derive(Clone, Debug)] +pub struct OdbcConfig { + /// The connection string to use for ODBC. + /// If the `connection_string_filepath` is set, this value is ignored. + #[configurable(metadata( + docs::examples = "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=" + ))] + pub connection_string: String, + + /// The path to the file that contains the connection string. + /// If this is not set or the file at that path does not exist, the `connection_string` field is used instead. + #[configurable(metadata( + docs::examples = "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=" + ))] + pub connection_string_filepath: Option, + + /// The SQL statement to execute. + /// This SQL statement is executed periodically according to the `schedule`. + /// Defaults to `None`. If no SQL statement is provided, the source returns an error. + /// If the `statement_filepath` is set, this value is ignored. + #[configurable(metadata(docs::examples = "SELECT * FROM users WHERE id = ?"))] + pub statement: Option, + + /// The path to the file that contains the SQL statement. + /// If this is unset or the file cannot be read, the value from `statement` is used instead. + pub statement_filepath: Option, + + /// Maximum time to allow the SQL statement to run. + /// If the query does not finish within this window, it is canceled and retried at the next scheduled run. + /// The default is 3 seconds. + #[configurable(metadata(docs::examples = 3))] + #[configurable(metadata( + docs::additional_props_description = "Maximum time to wait for the SQL statement to execute" + ))] + #[serde(default = "default_query_timeout_sec")] + #[serde_as(as = "DurationSeconds")] + pub statement_timeout: Duration, + + /// Initial parameters for the first execution of the statement. + /// Used if `last_run_metadata_path` does not exist. + /// Values must be strings and follow the parameter order defined in the query. + /// + /// # Examples + /// + /// When the source runs for the first time, the file at `last_run_metadata_path` does not exist. + /// In that case, declare the initial values in `statement_init_params`. + /// + /// ```toml + /// [sources.odbc] + /// statement = "SELECT * FROM users WHERE id = ?" + /// statement_init_params = { "id": "0" } + /// tracking_columns = ["id"] + /// last_run_metadata_path = "/path/to/tracking.json" + /// # The rest of the fields are omitted + /// ``` + #[configurable(metadata( + docs::additional_props_description = "Initial value for the SQL statement parameters. The value is always a string." + ))] + pub statement_init_params: Option, + + /// Cron expression used to schedule database queries. + /// When omitted, the statement runs only once by default. + #[configurable(derived)] + pub schedule: Option, + + /// The timezone to use for the `schedule`. + /// Typically the timezone used when evaluating the cron expression. + /// The default is UTC. + /// + /// [Wikipedia]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + #[configurable(metadata(docs::examples = "UTC"))] + #[configurable(metadata( + docs::additional_props_description = "Timezone to use for the schedule" + ))] + #[serde(default = "default_schedule_timezone")] + pub schedule_timezone: Tz, + + /// Number of rows to fetch per batch from the ODBC driver. + /// The default is 100. + #[configurable(metadata(docs::examples = 100))] + #[serde(default = "default_odbc_batch_size")] + pub odbc_batch_size: usize, + + /// Maximum string length for ODBC driver operations. + /// The default is 4096. + #[configurable(metadata(docs::examples = 4096))] + #[serde(default = "default_odbc_batch_size")] + pub odbc_max_str_limit: usize, + + /// Timezone applied to database date/time columns that lack timezone information. + /// The default is UTC. + #[configurable(metadata(docs::examples = "UTC"))] + #[configurable(metadata( + docs::additional_props_description = "Timezone to use for the database date/time type without a timezone" + ))] + #[serde(default = "default_odbc_default_timezone")] + pub odbc_default_timezone: Tz, + + /// Specifies the columns to track from the last row of the statement result set. + /// Their values are passed as parameters to the SQL statement in the next scheduled run. + /// + /// # Examples + /// + /// ```toml + /// [sources.odbc] + /// statement = "SELECT * FROM users WHERE id = ?" + /// tracking_columns = ["id"] + /// # The rest of the fields are omitted + /// ``` + #[configurable(metadata(docs::examples = "id"))] + pub tracking_columns: Option>, + + /// The path to the file where the last row of the result set will be saved. + /// The last row of the result set is saved in JSON format. + /// This file provides parameters for the SQL query in the next scheduled run. + /// If the file does not exist or the path is not specified, the initial value from `statement_init_params` is used. + /// + /// # Examples + /// + /// If `tracking_columns = ["id", "name"]`, it is saved as the following JSON data. + /// + /// ```json + /// {"id":1, "name": "vector"} + /// ``` + #[configurable(metadata(docs::examples = "/path/to/tracking.json"))] + pub last_run_metadata_path: Option, + + /// Decoder to use for query results. + #[configurable(derived)] + #[serde(default = "default_decoding")] + pub decoding: DeserializerConfig, + + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + pub log_namespace: Option, + + #[cfg(test)] + #[configurable(derived)] + #[serde(default)] + pub iterations: Option, +} + +const fn default_query_timeout_sec() -> Duration { + Duration::from_secs(3) +} + +const fn default_schedule_timezone() -> Tz { + Tz::UTC +} + +const fn default_odbc_batch_size() -> usize { + 100 +} + +const fn default_odbc_max_str_limit() -> usize { + 4096 +} + +const fn default_odbc_default_timezone() -> Tz { + default_schedule_timezone() +} + +impl Default for OdbcConfig { + fn default() -> Self { + Self { + connection_string: "".to_string(), + connection_string_filepath: None, + schedule: None, + schedule_timezone: Tz::UTC, + statement: None, + statement_timeout: Duration::from_secs(3), + statement_init_params: None, + odbc_batch_size: default_odbc_batch_size(), + odbc_max_str_limit: default_odbc_max_str_limit(), + odbc_default_timezone: Tz::UTC, + tracking_columns: None, + last_run_metadata_path: None, + decoding: default_decoding(), + log_namespace: None, + statement_filepath: None, + #[cfg(test)] + iterations: None, + } + } +} + +impl OdbcConfig { + /// Returns the connection string to use for ODBC. + /// If the `connection_string_filepath` is set, read the file and return its content. + pub fn connection_string_or_file(&self) -> String { + self.connection_string_filepath + .as_ref() + .and_then(|path| fs::read_to_string(path).ok()) + .unwrap_or(self.connection_string.clone()) + } + + /// Returns the SQL statement to execute. + /// If the `statement_filepath` is set, read the file and return its content. + pub fn statement_or_file(&self) -> Option { + self.statement_filepath + .as_ref() + .map(|path| fs::read_to_string(path).ok()) + .unwrap_or(self.statement.clone()) + } +} + +impl_generate_config_from_default!(OdbcConfig); + +#[async_trait::async_trait] +#[typetag::serde(name = "odbc")] +impl SourceConfig for OdbcConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + let guard = Context::new(self.clone(), cx)?; + let context = Box::new(guard); + Ok(context.run_schedule().boxed()) + } + + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let log_namespace = global_log_namespace.merge(self.log_namespace); + + let mut schema_definition = self + .decoding + .schema_definition(log_namespace) + .with_standard_vector_source_metadata(); + + if let Some(timestamp_key) = log_schema().timestamp_key() { + schema_definition = schema_definition.optional_field( + timestamp_key, + Kind::timestamp(), + Some("timestamp"), + ) + } + + vec![SourceOutput::new_maybe_logs( + self.decoding.output_type(), + schema_definition, + )] + } + + fn can_acknowledge(&self) -> bool { + false + } +} + +struct Context { + cfg: OdbcConfig, + env: Arc, + cx: SourceContext, +} + +impl Context { + fn new(cfg: OdbcConfig, cx: SourceContext) -> Result { + let env = Environment::new().context(DbSnafu)?; + + Ok(Self { + cfg, + env: Arc::new(env), + cx, + }) + } + + async fn run_schedule(self: Box) -> Result<(), ()> { + let shutdown = self.cx.shutdown.clone(); + + let Some(ref schedule) = self.cfg.schedule else { + warn!(message = "No next schedule found. Retry in 10 seconds."); + return Err(()); + }; + + let schedule = schedule.clone().stream(self.cfg.schedule_timezone); + pin_mut!(schedule); + + let _ = register!(BytesReceived::from(Protocol::NONE)); + + #[cfg(test)] + let mut count = 0; + + let mut prev_result = self.cfg.statement_init_params.clone(); + + loop { + select! { + _ = shutdown.clone() => { + debug!(message = "Shutdown signal received. Shutting down ODBC source."); + break; + } + next = schedule.next() => { + emit!(OdbcEventsReceived { + count: 1, + }); + + let instant = Instant::now(); + if let Ok(result) = self.process(prev_result.clone()).await { + + // Update the cached result when the query returns rows. + if result.is_some() { + prev_result = result; + } + + emit!(OdbcQueryExecuted { + statement: &self.cfg.statement.clone().unwrap_or_default(), + elapsed: instant.elapsed().as_millis() + }) + } else { + emit!(OdbcFailedError { + statement: &self.cfg.statement.clone().unwrap_or_default(), + }) + } + + // When no further schedule is defined, run once and then stop. + if next.is_none() { + debug!(message = "No additional schedule configured. Shutting down ODBC source."); + break + } + + #[cfg(test)] + { + count += 1; + if let Some(iterations) = self.cfg.iterations + && count >= iterations { + debug!(message = "No additional schedule configured. Shutting down ODBC source."); + break; + } + } + } + } + } + + Ok(()) + } + + /// Executes the scheduled ODBC query, sends the result as an event, and updates tracking metadata. + async fn process(&self, map: Option) -> Result, OdbcError> { + let conn_str = self.cfg.connection_string_or_file(); + let stmt_str = self.cfg.statement_or_file().context(ConfigSnafu { + cause: "No statement", + })?; + let out = self.cx.out.clone(); + let log_schema = log_schema(); + let env = Arc::clone(&self.env); + + // Load the last-run metadata from disk when available. + // If the file is missing, fall back to the initial parameters or the latest query result. + let stmt_params = self + .cfg + .last_run_metadata_path + .as_ref() + .and_then(|path| load_params(path, self.cfg.tracking_columns.as_ref())) + .unwrap_or( + order_params(map.unwrap_or_default(), self.cfg.tracking_columns.as_ref()) + .unwrap_or_default(), + ); + let cfg = self.cfg.clone(); + + let rows = execute_query( + &env, + &conn_str, + &stmt_str, + stmt_params, + cfg.statement_timeout, + cfg.odbc_default_timezone, + cfg.odbc_batch_size, + cfg.odbc_max_str_limit, + )?; + + // Example with query results: `{"message":[{ ... }],"timestamp":"2025-10-21T00:00:00.05275Z"}` + // Example with no query results: `{"message":[],"timestamp":"2025-10-21T00:00:00.05275Z"}` + let mut event = LogEvent::default(); + event.maybe_insert(Some("timestamp"), Value::Timestamp(Utc::now())); + event.maybe_insert( + log_schema.message_key_target_path(), + Value::Array(rows.clone()), + ); + + let mut out = out.clone(); + out.send_event(event).await.context(ClosedSnafu)?; + + if let Some(last) = rows.last() { + let Some(tracking_columns) = cfg.tracking_columns else { + return Ok(None); + }; + let latest_result = extract_and_save_tracking( + cfg.last_run_metadata_path.as_deref(), + last.clone(), + tracking_columns, + ) + .await?; + return Ok(latest_result); + } + + Ok(None) + } +} + +/// Extracts specified tracking columns from the given object, +/// saves them to a file if a path is provided. +async fn extract_and_save_tracking( + path: Option<&str>, + obj: Value, + tracking_columns: Vec, +) -> Result, OdbcError> { + let tracking_columns = tracking_columns + .iter() + .map(|col| col.as_str()) + .collect_vec(); + + if let Value::Object(obj) = obj { + let save_obj = obj + .iter() + .filter(|item| tracking_columns.contains(&item.0.as_str())) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + if let Some(path) = path { + save_params(path, &save_obj)?; + } + return Ok(Some(save_obj)); + } + + Ok(None) +} + +/// Executes an ODBC SQL query with optional parameters, fetches rows in batches, +/// and returns the results as a vector of objects. +#[allow(clippy::too_many_arguments)] +pub fn execute_query( + env: &Environment, + conn_str: &str, + stmt_str: &str, + stmt_params: Vec, + timeout: Duration, + tz: Tz, + batch_size: usize, + max_str_limit: usize, +) -> Result { + let conn = env + .connect_with_connection_string(conn_str, ConnectionOptions::default()) + .context(DbSnafu)?; + let mut statement = conn.preallocate().context(DbSnafu)?; + statement + .set_query_timeout_sec(timeout.as_secs() as usize) + .context(DbSnafu)?; + + let result = if stmt_params.is_empty() { + statement.execute(stmt_str, ()) + } else { + statement.execute(stmt_str, &stmt_params[..]) + } + .context(DbSnafu)?; + + let Some(mut cursor) = result else { + return Ok(Rows::default()); + }; + + let names = cursor + .column_names() + .context(DbSnafu)? + .collect::, _>>() + .context(DbSnafu)?; + let types = (1..=names.len()) + .map(|col_index| cursor.col_data_type(col_index as u16).unwrap_or_default()) + .collect_vec(); + let columns = names + .into_iter() + .zip(types) + .map(|(column_name, column_type)| Column { + column_name, + column_type, + }) + .collect::(); + + let buffer = + TextRowSet::for_cursor(batch_size, &mut cursor, Some(max_str_limit)).context(DbSnafu)?; + let mut row_set_cursor = cursor.bind_buffer(buffer).context(DbSnafu)?; + let mut rows = Rows::with_capacity(batch_size); + + while let Some(batch) = row_set_cursor.fetch().context(DbSnafu)? { + let num_rows = batch.num_rows(); + + for row_index in 0..num_rows { + let mut cols = ObjectMap::new(); + + for (index, column) in columns.iter().enumerate() { + let data_name = &column.column_name; + let data_type = &column.column_type; + let data_value = batch.at(index, row_index); + let key = KeyString::from(data_name.as_str()); + let value = map_value(data_type, data_value, tz); + cols.insert(key, value); + } + + rows.push(Value::Object(cols)) + } + } + + Ok(rows) +} diff --git a/src/sources/odbc/integration_tests.rs b/src/sources/odbc/integration_tests.rs new file mode 100644 index 0000000000000..3112db314734c --- /dev/null +++ b/src/sources/odbc/integration_tests.rs @@ -0,0 +1,751 @@ +use crate::sources::odbc::client::{OdbcConfig, execute_query}; +use crate::test_util::components::SOURCE_TAGS; +use crate::test_util::components::run_and_assert_source_compliance; +use bytes::Bytes; +use chrono::TimeZone; +use chrono_tz::Tz; +use odbc_api::ConnectionOptions; +use ordered_float::NotNan; +use std::borrow::Cow; +use std::fs; +use std::time::Duration; +use vector_lib::event::Event; +use vrl::prelude::*; +use vrl::value::Value; + +enum DbType { + MariaDb, + Postgres, +} + +fn get_db_type() -> DbType { + match std::env::var("ODBC_DB_TYPE").as_deref() { + Ok("mariadb") => DbType::MariaDb, + Ok("postgresql") => DbType::Postgres, + _ => panic!("Required environment variable 'ODBC_DB_TYPE'"), + } +} + +fn get_conn_str() -> String { + std::env::var("ODBC_CONN_STRING").expect("Required environment variable 'ODBC_CONN_STRING'") +} + +const fn get_conn_opt() -> ConnectionOptions { + ConnectionOptions { + login_timeout_sec: Some(3), + packet_size: None, + } +} + +fn get_value_from_event<'a>(event: &'a Event, key: &str) -> Option> { + let log = event.as_log(); + let msg = log.get_message()?; + let arr_msg = msg.as_array_unwrap(); + let value = arr_msg[0].get(key); + value?.as_str() +} + +#[tokio::test] +async fn parse_odbc_config() { + let conn_str = get_conn_str(); + let config_str = format!( + r#" + connection_string = "{conn_str}" + statement = "SELECT * FROM odbc_table WHERE id > ? LIMIT 1;" + schedule = "*/5 * * * * *" + schedule_timezone = "UTC" + last_run_metadata_path = "odbc_tracking.json" + tracking_columns = ["id", "name", "datetime"] + statement_init_params = {{ id = "0", name = "test" }} + iterations = 1 + "# + ); + let config = toml::from_str::(&config_str); + assert!( + config.is_ok(), + "Failed to parse config: {}", + config.unwrap_err() + ); +} + +#[tokio::test] +async fn scheduled_query_executed() { + let conn_str = get_conn_str(); + run_and_assert_source_compliance( + OdbcConfig { + connection_string: conn_str, + schedule: Some("*/1 * * * * *".into()), + statement: Some("SELECT 1".to_string()), + iterations: Some(1), + ..Default::default() + }, + Duration::from_secs(3), + &SOURCE_TAGS, + ) + .await; +} + +#[tokio::test] +async fn query_executed_with_init_params() { + const LAST_RUN_METADATA_PATH: &str = "odbc_tracking-integration-tests.json"; + + let conn_str = get_conn_str(); + let env = odbc_api::Environment::new().unwrap(); + let conn = env + .connect_with_connection_string(&conn_str, get_conn_opt()) + .unwrap(); + let _ = conn + .execute("DROP TABLE IF EXISTS odbc_table;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE odbc_table +( + id int auto_increment primary key, + name varchar(255) null, + `datetime` datetime null +); + "# + } + DbType::Postgres => { + r#" +CREATE TABLE odbc_table +( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + "datetime" TIMESTAMP NULL +); +"# + } + }, + (), + Some(3), + ) + .unwrap(); + let _ = conn + .execute( + r#" +INSERT INTO odbc_table (name, datetime) VALUES +('test1', now()), +('test2', now()), +('test3', now()), +('test4', now()), +('test5', now()); + "#, + (), + Some(3), + ) + .unwrap(); + let params = ObjectMap::from([("id".into(), 0.into())]); + + let _ = fs::remove_file(LAST_RUN_METADATA_PATH); + + let events = run_and_assert_source_compliance( + OdbcConfig { + connection_string: conn_str, + schedule: Some("*/1 * * * * *".into()), + statement: Some("SELECT * FROM odbc_table WHERE id > ? LIMIT 1;".to_string()), + statement_init_params: Some(params), + tracking_columns: Some(vec!["id".to_string()]), + last_run_metadata_path: Some(LAST_RUN_METADATA_PATH.to_string()), + iterations: Some(5), + ..Default::default() + }, + Duration::from_secs(10), + &SOURCE_TAGS, + ) + .await; + + debug!("{}", serde_json::to_string_pretty(&events).unwrap()); + assert_eq!( + get_value_from_event(&events[0], "name"), + Some("test1".into()) + ); + assert_eq!( + get_value_from_event(&events[1], "name"), + Some("test2".into()) + ); + assert_eq!( + get_value_from_event(&events[2], "name"), + Some("test3".into()) + ); + assert_eq!( + get_value_from_event(&events[3], "name"), + Some("test4".into()) + ); + assert_eq!( + get_value_from_event(&events[4], "name"), + Some("test5".into()) + ); +} + +#[tokio::test] +async fn query_executed_with_filepath() { + const CONNECTION_STRING_FILE_PATH: &str = "odbc_connection_string.txt"; + const STATEMENT_FILE_PATH: &str = "odbc_statement.sql"; + const LAST_RUN_METADATA_PATH: &str = "odbc_tracking-integration-tests.json"; + + let conn_str = get_conn_str(); + let env = odbc_api::Environment::new().unwrap(); + let conn = env + .connect_with_connection_string(&conn_str, get_conn_opt()) + .unwrap(); + let _ = conn + .execute("DROP TABLE IF EXISTS odbc_table;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE odbc_table +( + id int auto_increment primary key, + name varchar(255) null, + `datetime` datetime null +);"# + } + DbType::Postgres => { + r#" +CREATE TABLE odbc_table +( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + "datetime" TIMESTAMP NULL +);"# + } + }, + (), + Some(3), + ) + .unwrap(); + let _ = conn + .execute( + r#" +INSERT INTO odbc_table (name, datetime) VALUES +('test1', now()), +('test2', now()), +('test3', now()), +('test4', now()), +('test5', now()); + "#, + (), + Some(3), + ) + .unwrap(); + let params = ObjectMap::from([("id".into(), 0.into())]); + + fs::write(CONNECTION_STRING_FILE_PATH, conn_str).unwrap(); + fs::write( + STATEMENT_FILE_PATH, + "SELECT * FROM odbc_table WHERE id > ? LIMIT 1;", + ) + .unwrap(); + let _ = fs::remove_file(LAST_RUN_METADATA_PATH); + + let events = run_and_assert_source_compliance( + OdbcConfig { + connection_string_filepath: Some(CONNECTION_STRING_FILE_PATH.to_string()), + schedule: Some("*/1 * * * * *".into()), + statement_filepath: Some(STATEMENT_FILE_PATH.to_string()), + statement_init_params: Some(params), + tracking_columns: Some(vec!["id".to_string()]), + last_run_metadata_path: Some(LAST_RUN_METADATA_PATH.to_string()), + iterations: Some(5), + ..Default::default() + }, + Duration::from_secs(10), + &SOURCE_TAGS, + ) + .await; + + debug!("{}", serde_json::to_string_pretty(&events).unwrap()); + assert_eq!( + get_value_from_event(&events[0], "name"), + Some("test1".into()) + ); + assert_eq!( + get_value_from_event(&events[1], "name"), + Some("test2".into()) + ); + assert_eq!( + get_value_from_event(&events[2], "name"), + Some("test3".into()) + ); + assert_eq!( + get_value_from_event(&events[3], "name"), + Some("test4".into()) + ); + assert_eq!( + get_value_from_event(&events[4], "name"), + Some("test5".into()) + ); +} + +#[tokio::test] +async fn query_number_types() { + let conn_str = get_conn_str(); + let env = odbc_api::Environment::new().unwrap(); + let conn = env + .connect_with_connection_string(&conn_str, get_conn_opt()) + .unwrap(); + let _ = conn + .execute("DROP TABLE IF EXISTS number_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +create table number_columns +( + int_col int(10) null, + bit_col bit null, + mediumint_col mediumint null, + middleint_col mediumint null, + smallint_col smallint null, + tinyint_col tinyint null, + bigint_col bigint null, + boolean_col tinyint(1) null, + double_col double null, + float_col float null, + decimal_col decimal(10, 2) null +); + "# + } + DbType::Postgres => { + r#" +CREATE TABLE number_columns +( + int_col INTEGER, -- integer + bit_col BIT, -- single bit (use BIT(n) to specify multiple bits) + mediumint_col INTEGER, -- no MEDIUMINT in PostgreSQL, mapped to INTEGER + middleint_col INTEGER, -- same as MEDIUMINT, mapped to INTEGER + smallint_col SMALLINT, -- small integer + tinyint_col SMALLINT, -- no TINYINT in PostgreSQL, mapped to SMALLINT + bigint_col BIGINT, -- big integer (64-bit) + boolean_col BOOLEAN, -- MySQL tinyint(1) mapped to BOOLEAN + double_col DOUBLE PRECISION, -- MySQL DOUBLE mapped to PostgreSQL DOUBLE PRECISION + float_col REAL, -- MySQL FLOAT mapped to PostgreSQL REAL (4-byte float) + decimal_col NUMERIC(10,2) -- MySQL DECIMAL mapped to PostgreSQL NUMERIC(p,s) +); + "# + } + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO number_columns ( + int_col, + bit_col, + mediumint_col, + middleint_col, + smallint_col, + tinyint_col, + bigint_col, + boolean_col, + double_col, + float_col, + decimal_col +) VALUES ( + -2147483648, + b'0', + -8388608, + -8388608, + -32768, + -128, + -9223372036854775808, + FALSE, + -1.7976931348623157e308, + -3.402823466e38, + -99999999.99 +); + "#, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO number_columns ( + int_col, + bit_col, + mediumint_col, + middleint_col, + smallint_col, + tinyint_col, + bigint_col, + boolean_col, + double_col, + float_col, + decimal_col +) VALUES ( + 2147483647, + b'1', + 8388607, + 8388607, + 32767, + 127, + 9223372036854775807, + TRUE, + 1.7976931348623157e308, + 3.402823466e38, + 99999999.99 +); + "#, + (), + Some(3), + ) + .unwrap(); + + let rows = execute_query( + &env, + &conn_str, + "SELECT * FROM number_columns;", + vec![], + Duration::from_secs(3), + Tz::UTC, + 10, + 1000, + ) + .unwrap(); + debug!("Rows Count: {}", rows.len()); + for row in &rows { + if let Value::Object(map) = row { + for (key, value) in map { + debug!("{key}: {value:?}"); + } + } + } + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + assert_eq!(*row.get("int_col").unwrap(), Value::Integer(-2147483648)); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("bit_col").unwrap(), Value::Boolean(false)), + DbType::Postgres => assert_eq!( + *row.get("bit_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0")) + ), + } + assert_eq!(*row.get("mediumint_col").unwrap(), Value::Integer(-8388608)); + assert_eq!(*row.get("middleint_col").unwrap(), Value::Integer(-8388608)); + assert_eq!(*row.get("smallint_col").unwrap(), Value::Integer(-32768)); + assert_eq!(*row.get("tinyint_col").unwrap(), Value::Integer(-128)); + assert_eq!( + *row.get("bigint_col").unwrap(), + Value::Integer(-9223372036854775808) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("boolean_col").unwrap(), Value::Boolean(false)), + DbType::Postgres => assert_eq!( + *row.get("boolean_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0")) + ), + } + assert_eq!( + *row.get("double_col").unwrap(), + Value::Float(NotNan::new(-1.7976931348623157e308).unwrap()) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(-3.40282e38).unwrap()) + ), + DbType::Postgres => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(-3.4028235e38).unwrap()) + ), + } + assert_eq!( + *row.get("decimal_col").unwrap(), + Value::Float(NotNan::new(-99999999.99).unwrap()) + ); + + let Value::Object(row) = &rows[1] else { + panic!("No second row returned") + }; + assert_eq!(*row.get("int_col").unwrap(), Value::Integer(2147483647)); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("bit_col").unwrap(), Value::Boolean(true)), + DbType::Postgres => assert_eq!( + *row.get("bit_col").unwrap(), + Value::Bytes(Bytes::from_static(b"1")) + ), + } + assert_eq!(*row.get("mediumint_col").unwrap(), Value::Integer(8388607)); + assert_eq!(*row.get("middleint_col").unwrap(), Value::Integer(8388607)); + assert_eq!(*row.get("smallint_col").unwrap(), Value::Integer(32767)); + assert_eq!(*row.get("tinyint_col").unwrap(), Value::Integer(127)); + assert_eq!( + *row.get("bigint_col").unwrap(), + Value::Integer(9223372036854775807) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("boolean_col").unwrap(), Value::Boolean(true)), + DbType::Postgres => assert_eq!( + *row.get("boolean_col").unwrap(), + Value::Bytes(Bytes::from_static(b"1")) + ), + } + assert_eq!( + *row.get("double_col").unwrap(), + Value::Float(NotNan::new(1.7976931348623157e308).unwrap()) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(3.40282e38).unwrap()) + ), + DbType::Postgres => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(3.4028235e38).unwrap()) + ), + } + assert_eq!( + *row.get("decimal_col").unwrap(), + Value::Float(NotNan::new(99999999.99).unwrap()) + ); + + debug!("{rows:#?}"); +} + +#[tokio::test] +async fn query_string_types() { + let conn_str = get_conn_str(); + let env = odbc_api::Environment::new().unwrap(); + let conn = env + .connect_with_connection_string(&conn_str, get_conn_opt()) + .unwrap(); + let _ = conn + .execute("DROP TABLE IF EXISTS string_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE string_columns ( + char10_col CHAR(10) NULL, + nchar10_col NCHAR(10) NULL, + nvarchar10_col NVARCHAR(10) NULL, + text_col TEXT NULL, + tinytext_col TINYTEXT NULL, + mediumtext_col MEDIUMTEXT NULL, + longtext_col LONGTEXT NULL +) DEFAULT CHARSET = utf8mb3 COLLATE = utf8mb3_general_ci; + "# + } + DbType::Postgres => { + r#" +CREATE TABLE string_columns ( + char10_col CHAR(10), -- fixed-length character column (10) + nchar10_col CHAR(10), -- PostgreSQL has no NCHAR; use CHAR with UTF-8 encoding + nvarchar10_col VARCHAR(10), -- PostgreSQL has no NVARCHAR; use VARCHAR with UTF-8 encoding + text_col TEXT, -- unlimited length text + tinytext_col TEXT, -- PostgreSQL has no TINYTEXT; use TEXT + mediumtext_col TEXT, -- PostgreSQL has no MEDIUMTEXT; use TEXT + longtext_col TEXT -- PostgreSQL has no LONGTEXT; use TEXT +); + "# + } + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO string_columns ( + char10_col, + nchar10_col, + nvarchar10_col, + text_col, + tinytext_col, + mediumtext_col, + longtext_col +) VALUES ( + '0123456789', + '0123456789', + '0123456789', + 'text', + 'tinytext', + 'mediumtext', + 'longtext' +); + "#, + (), + Some(3), + ) + .unwrap(); + + let rows = execute_query( + &env, + &conn_str, + "SELECT * FROM string_columns;", + vec![], + Duration::from_secs(3), + Tz::UTC, + 10, + 1000, + ) + .unwrap(); + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + + assert_eq!( + *row.get("char10_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0123456789")) + ); + assert_eq!( + *row.get("nchar10_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0123456789")) + ); + assert_eq!( + *row.get("nvarchar10_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0123456789")) + ); + assert_eq!( + *row.get("text_col").unwrap(), + Value::Bytes(Bytes::from_static(b"text")) + ); + assert_eq!( + *row.get("tinytext_col").unwrap(), + Value::Bytes(Bytes::from_static(b"tinytext")) + ); + assert_eq!( + *row.get("mediumtext_col").unwrap(), + Value::Bytes(Bytes::from_static(b"mediumtext")) + ); + assert_eq!( + *row.get("longtext_col").unwrap(), + Value::Bytes(Bytes::from_static(b"longtext")) + ); +} + +#[tokio::test] +async fn query_timestamp_columns() { + let conn_str = get_conn_str(); + let env = odbc_api::Environment::new().unwrap(); + let conn = env + .connect_with_connection_string(&conn_str, ConnectionOptions::default()) + .unwrap(); + let _ = conn + .execute("DROP TABLE IF EXISTS timestamp_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => r#" +CREATE TABLE timestamp_columns ( + date_col DATE NULL, + datetime_col DATETIME NULL, + time_col TIME NULL, + timestamp_col TIMESTAMP NULL, + year_col YEAR NULL +) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci; + "#, + DbType::Postgres => r#" +CREATE TABLE timestamp_columns ( + date_col DATE, -- MySQL DATE → PostgreSQL DATE + datetime_col TIMESTAMP, -- MySQL DATETIME → PostgreSQL TIMESTAMP + time_col TIME, -- Same in both + timestamp_col TIMESTAMP, -- Same type (use TIMESTAMPTZ if timezone is needed) + year_col SMALLINT -- MySQL YEAR → PostgreSQL SMALLINT +); + "#, + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO timestamp_columns ( + date_col, + datetime_col, + time_col, + timestamp_col, + year_col +) +VALUES ( + '2025-10-04', + '2025-10-04 12:34:56', + '15:30:00', + '2025-10-04 12:34:56', + 2025 +); + "#, + (), + Some(3), + ) + .unwrap(); + + let rows = execute_query( + &env, + &conn_str, + "SELECT * FROM timestamp_columns;", + vec![], + Duration::from_secs(3), + Tz::UTC, + 10, + 1000, + ) + .unwrap(); + + debug!("Rows Count: {}", rows.len()); + for row in &rows { + if let Value::Object(map) = row { + for (key, value) in map { + debug!("{key}: {value:?}"); + } + } + } + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + + assert_eq!( + *row.get("date_col").unwrap(), + Value::Timestamp(chrono::Utc.with_ymd_and_hms(2025, 10, 4, 0, 0, 0).unwrap()) + ); + assert_eq!( + *row.get("datetime_col").unwrap(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2025, 10, 4, 12, 34, 56) + .unwrap() + ) + ); + assert_eq!( + *row.get("time_col").unwrap(), + Value::Timestamp(chrono::Utc.with_ymd_and_hms(1970, 1, 1, 15, 30, 0).unwrap()) + ); + assert_eq!( + *row.get("timestamp_col").unwrap(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2025, 10, 4, 12, 34, 56) + .unwrap() + ) + ); + assert_eq!(*row.get("year_col").unwrap(), Value::Integer(2025)); +} diff --git a/src/sources/odbc/mod.rs b/src/sources/odbc/mod.rs new file mode 100644 index 0000000000000..850be39a0aa19 --- /dev/null +++ b/src/sources/odbc/mod.rs @@ -0,0 +1,370 @@ +//! ODBC Data Source +//! +//! This data source runs a database query through the ODBC interface on the configured schedule. +//! Query results are sent to Vector as an array of key-value maps. +//! The final row of the result set is saved to disk and used as a parameter for the next scheduled SQL query. +//! +//! The ODBC data source offers functionality similar to the [Logstash JDBC plugin](https://www.elastic.co/docs/reference/logstash/plugins/plugins-inputs-jdbc). +//! +//! # Example +//! +//! Given the following MariaDB table and sample data: +//! +//! ```sql +//! create table odbc_table +//! ( +//! id int auto_increment primary key, +//! name varchar(255) null, +//! `datetime` datetime null +//! ); +//! +//! INSERT INTO odbc_table (name, datetime) VALUES +//! ('test1', now()), +//! ('test2', now()), +//! ('test3', now()), +//! ('test4', now()), +//! ('test5', now()); +//! ``` +//! +//! The example below shows how to connect to a MariaDB database with the ODBC driver, +//! run a query periodically, and send the results to Vector. +//! Provide a database connection string. +//! +//! ```toml +//! [sources.odbc] +//! type = "odbc" +//! connection_string = "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=;" +//! statement = "SELECT * FROM odbc_table WHERE id > ? LIMIT 1;" +//! statement_init_params = { id = "0", name = "test" } +//! schedule = "*/5 * * * * *" +//! schedule_timezone = "UTC" +//! last_run_metadata_path = "/path/to/odbc_tracking.json" +//! tracking_columns = ["id"] +//! +//! [sinks.console] +//! type = "console" +//! inputs = ["odbc"] +//! encoding.codec = "json" +//! ``` +//! +//! Every five seconds, the source produces output similar to the following. +//! +//! ```json +//! {"message":[{"datetime":"2025-04-28T01:20:04Z","id":1,"name":"test1"}],"timestamp":"2025-04-28T01:50:45.075484Z"} +//! {"message":[{"datetime":"2025-04-28T01:20:04Z","id":2,"name":"test2"}],"timestamp":"2025-04-28T01:50:50.017276Z"} +//! {"message":[{"datetime":"2025-04-28T01:20:04Z","id":3,"name":"test3"}],"timestamp":"2025-04-28T01:50:55.016432Z"} +//! {"message":[{"datetime":"2025-04-28T01:20:04Z","id":4,"name":"test4"}],"timestamp":"2025-04-28T01:51:00.016328Z"} +//! {"message":[{"datetime":"2025-04-28T01:20:04Z","id":5,"name":"test5"}],"timestamp":"2025-04-28T01:51:05.010063Z"} +//! ``` + +use crate::source_sender::ClosedError; +use bytes::Bytes; +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; +use chrono_tz::Tz; +use cron::Schedule; +use futures::Stream; +use futures_util::stream; +use itertools::Itertools; +use odbc_api::IntoParameter; +use odbc_api::parameter::VarCharBox; +use ordered_float::NotNan; +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, Snafu}; +use std::cell::RefCell; +use std::fmt::{Debug, Formatter}; +use std::fs::File; +use std::io::BufReader; +use std::str::FromStr; +use std::{fmt, fs}; +use tokio::time::sleep; +use vector_config::schema::generate_string_schema; +use vector_config::{Configurable, GenerateError, Metadata, ToValue}; +use vector_config_common::schema::{SchemaGenerator, SchemaObject}; +use vrl::prelude::*; + +#[cfg(feature = "sources-odbc")] +mod client; + +#[cfg(all(test, feature = "odbc-integration-tests"))] +mod integration_tests; + +const TIMESTAMP_FORMATS: &[&str] = &[ + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S", + "%Y/%m/%d %H:%M:%S", + "%Y/%m/%dT%H:%M:%S", +]; + +struct Column { + column_name: String, + column_type: odbc_api::DataType, +} + +/// Columns of the query result. +type Columns = Vec; + +/// Rows of the query result. +type Rows = Vec; + +#[derive(Debug, Snafu)] +enum OdbcError { + #[snafu(display("ODBC database error: {source}"))] + Db { source: odbc_api::Error }, + + #[snafu(display("File IO error: {source}"))] + Io { source: std::io::Error }, + + #[snafu(display("Batch error: {source}"))] + Closed { source: ClosedError }, + + #[snafu(display("JSON error: {source}"))] + Json { source: serde_json::Error }, + + #[snafu(display("Configuration error: {cause}"))] + ConfigError { cause: &'static str }, +} + +/// Newtype around `cron::Schedule` that enables a `Configurable` implementation. +#[derive(Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct OdbcSchedule { + inner: Schedule, +} + +impl ToValue for OdbcSchedule { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(&self.inner) + .expect("Could not convert schedule(cron expression) to JSON") + } +} + +impl Configurable for OdbcSchedule { + fn referenceable_name() -> Option<&'static str> { + Some("cron::Schedule") + } + + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description("Cron expression in seconds."); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + Ok(generate_string_schema()) + } +} + +impl From<&str> for OdbcSchedule { + fn from(s: &str) -> Self { + let schedule = Schedule::from_str(s).expect("Invalid cron expression"); + Self { inner: schedule } + } +} + +impl Debug for OdbcSchedule { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(&self.inner.to_string()) + } +} + +impl OdbcSchedule { + /// Creates a stream that asynchronously waits for each scheduled cron time. + pub(crate) fn stream(self, tz: Tz) -> impl Stream> { + let schedule = self.inner.clone(); + stream::unfold(schedule, move |schedule| async move { + let now = Utc::now().with_timezone(&tz); + let mut upcoming = schedule.upcoming(tz); + let next = upcoming.next()?; + let delay = (next - now).abs(); + + sleep(delay.to_std().unwrap_or_default()).await; + Some((next, schedule)) + }) + } +} + +/// Loads the previously saved result and returns it as SQL parameters. +/// Parameters are generated in the order specified by `columns_order`. +fn load_params(path: &str, columns_order: Option<&Vec>) -> Option> { + let file = File::open(path).ok()?; + let reader = BufReader::new(file); + let map: ObjectMap = serde_json::from_reader(reader).ok()?; + + order_params(map, columns_order) +} + +/// Orders the parameters of a given `ObjectMap` based on an optional column order. +fn order_params(map: ObjectMap, columns_order: Option<&Vec>) -> Option> { + if columns_order.is_none() || columns_order.iter().len() == 0 { + let params = map + .iter() + .map(|p| p.1.to_string().into_parameter()) + .collect_vec(); + return Some(params); + } + + let binding = vec![]; + let columns_order = columns_order + .unwrap_or(&binding) + .iter() + .map(|col| col.as_str()) + .collect_vec(); + + // Ensure parameters follow the declared column order. + let params = columns_order + .into_iter() + .filter_map(|col| { + let value = map.get(col)?; + Some(value.to_string().into_parameter()) + }) + .collect_vec(); + + Some(params) +} + +/// Serializes and persists the latest tracked values for reuse as SQL parameters. +fn save_params(path: &str, obj: &ObjectMap) -> Result<(), OdbcError> { + let json = serde_json::to_string(obj).context(JsonSnafu)?; + fs::write(path, json).context(IoSnafu) +} + +/// Converts ODBC data types to Vector values. +/// +/// # Arguments +/// * `data_type`: The ODBC data type. +/// * `value`: The ODBC value to convert. +/// * `tz`: The timezone to use for date/time conversions. +/// +/// # Returns +/// A `Value` compatible with Vector events. +fn map_value(data_type: &odbc_api::DataType, value: Option<&[u8]>, tz: Tz) -> Value { + match data_type { + // Convert to bytes. + odbc_api::DataType::Unknown + | odbc_api::DataType::Char { .. } + | odbc_api::DataType::WChar { .. } + | odbc_api::DataType::Varchar { .. } + | odbc_api::DataType::WVarchar { .. } + | odbc_api::DataType::LongVarchar { .. } + | odbc_api::DataType::WLongVarchar { .. } + | odbc_api::DataType::Varbinary { .. } + | odbc_api::DataType::Binary { .. } + | odbc_api::DataType::Other { .. } + | odbc_api::DataType::LongVarbinary { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Bytes(Bytes::copy_from_slice(value)) + } + + // Convert to integer. + odbc_api::DataType::TinyInt + | odbc_api::DataType::SmallInt + | odbc_api::DataType::BigInt + | odbc_api::DataType::Integer => { + let Some(value) = value else { + return Value::Null; + }; + + // tinyint(1) -> Value::Boolean + if *data_type == odbc_api::DataType::TinyInt + && value.len() == 1 + && (value[0] == b'0' || value[0] == b'1') + { + return Value::Boolean(value[0] == b'1'); + } + + std::str::from_utf8(value) + .ok() + .and_then(|s| s.parse::().ok()) + .map_or(Value::Null, Value::Integer) + } + + // Convert to float. + odbc_api::DataType::Float { .. } + | odbc_api::DataType::Real + | odbc_api::DataType::Decimal { .. } + | odbc_api::DataType::Numeric { .. } + | odbc_api::DataType::Double => { + let Some(value) = value else { + return Value::Null; + }; + + std::str::from_utf8(value) + .ok() + .and_then(|s| NotNan::from_str(s).ok()) + .map_or(Value::Null, Value::Float) + } + + // Convert to timestamp. + odbc_api::DataType::Timestamp { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + let Ok(str) = std::str::from_utf8(value) else { + return Value::Null; + }; + + // Try RFC3339/ISO8601 first and convert to UTC + if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(str) { + return Value::Timestamp(datetime.into()); + } + + let datetime = TIMESTAMP_FORMATS + .iter() + .find_map(|fmt| NaiveDateTime::parse_from_str(str, fmt).ok()) + .map(|ndt| ndt.and_utc()); + + datetime.map(Value::Timestamp).unwrap_or(Value::Null) + } + + // Convert to timestamp. + odbc_api::DataType::Time { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + std::str::from_utf8(value) + .ok() + .and_then(|s| NaiveTime::from_str(s).ok()) + .map(|time| { + let datetime = NaiveDateTime::new(NaiveDate::default(), time); + let tz = tz.offset_from_utc_datetime(&datetime); + Value::Timestamp( + DateTime::::from_naive_utc_and_offset(datetime, tz).to_utc(), + ) + }) + .unwrap_or(Value::Null) + } + + // Convert to timestamp. + odbc_api::DataType::Date => { + let Some(value) = value else { + return Value::Null; + }; + + std::str::from_utf8(value) + .ok() + .and_then(|s| chrono::NaiveDate::from_str(s).ok()) + .map(|date| { + let datetime = NaiveDateTime::new(date, NaiveTime::default()); + let tz = tz.offset_from_utc_datetime(&datetime); + Value::Timestamp( + DateTime::::from_naive_utc_and_offset(datetime, tz).to_utc(), + ) + }) + .unwrap_or(Value::Null) + } + + // Convert to boolean. + odbc_api::DataType::Bit => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Boolean(value[0] == 1 || value[0] == b'1') + } + } +} diff --git a/tests/data/odbc/odbc-init.sh b/tests/data/odbc/odbc-init.sh new file mode 100755 index 0000000000000..403df3f8ac74a --- /dev/null +++ b/tests/data/odbc/odbc-init.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +apt update +apt install -y unixodbc odbcinst odbc-mariadb \ No newline at end of file diff --git a/website/content/en/docs/reference/configuration/sources/odbc.md b/website/content/en/docs/reference/configuration/sources/odbc.md new file mode 100644 index 0000000000000..795b401387ac0 --- /dev/null +++ b/website/content/en/docs/reference/configuration/sources/odbc.md @@ -0,0 +1,14 @@ +--- +title: ODBC +description: ODBC(Open Database Connectivity) Data Source. +component_kind: source +layout: component +tags: ["odbc", "component", "source", "database"] +--- + +{{/* +This doc is generated using: + +1. The template in layouts/docs/component.html +2. The relevant CUE data in cue/reference/components/... +*/}} diff --git a/website/cue/reference/components/sources/generated/odbc.cue b/website/cue/reference/components/sources/generated/odbc.cue new file mode 100644 index 0000000000000..0842314a7b087 --- /dev/null +++ b/website/cue/reference/components/sources/generated/odbc.cue @@ -0,0 +1,474 @@ +package metadata + +generated: components: sources: odbc: configuration: { + connection_string: { + description: """ + The connection string to use for ODBC. + If the `connection_string_filepath` is set, this value is ignored. + """ + required: true + type: string: examples: ["driver={MariaDB Unicode};server=;port=;database=;uid=;pwd="] + } + connection_string_filepath: { + description: """ + The path to the file that contains the connection string. + If this is not set or the file at that path does not exist, the `connection_string` field is used instead. + """ + required: false + type: string: examples: ["driver={MariaDB Unicode};server=;port=;database=;uid=;pwd="] + } + decoding: { + description: "Decoder to use for query results." + required: false + type: object: options: { + avro: { + description: "Apache Avro-specific encoder options." + relevant_when: "codec = \"avro\"" + required: true + type: object: options: { + schema: { + description: """ + The Avro schema definition. + **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: + * `Date` + * `Decimal` + * `Duration` + * `Fixed` + * `TimeMillis` + """ + required: true + type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] + } + strip_schema_id_prefix: { + description: """ + For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to `true` to strip the schema ID prefix. + According to [Confluent Kafka's document](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format). + """ + required: true + type: bool: {} + } + } + } + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + avro: """ + Decodes the raw bytes as as an [Apache Avro][apache_avro] message. + + [apache_avro]: https://avro.apache.org/ + """ + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + This codec is experimental for the following reason: + + The GELF specification is more strict than the actual Graylog receiver. + Vector's decoder adheres more strictly to the GELF spec, with + the exception that some characters such as `@` are allowed in field names. + + Other GELF codecs such as Loki's, use a [Go SDK][implementation] that is maintained + by Graylog, and is much more relaxed than the GELF spec. + + Going forward, Vector will use that [Go SDK][implementation] as the reference implementation, which means + the codec may continue to relax the enforcement of specification. + + [gelf]: https://docs.graylog.org/docs/gelf + [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go + """ + influxdb: """ + Decodes the raw bytes as an [Influxdb Line Protocol][influxdb] message. + + [influxdb]: https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as [native Protocol Buffers format][vector_native_protobuf]. + + This decoder can output all types of events (logs, metrics, traces). + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as [native JSON format][vector_native_json]. + + This decoder can output all types of events (logs, metrics, traces). + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + otlp: """ + Decodes the raw bytes as [OTLP (OpenTelemetry Protocol)][otlp] protobuf format. + + This decoder handles the three OTLP signal types: logs, metrics, and traces. + It automatically detects which type of OTLP message is being decoded. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + vrl: """ + Decodes the raw bytes as a string and passes them as input to a [VRL][vrl] program. + + [vrl]: https://vector.dev/docs/reference/vrl + """ + } + } + } + gelf: { + description: "GELF-specific decoding options." + relevant_when: "codec = \"gelf\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + influxdb: { + description: "Influxdb-specific decoding options." + relevant_when: "codec = \"influxdb\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + json: { + description: "JSON-specific decoding options." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + native_json: { + description: "Vector's native JSON-specific decoding options." + relevant_when: "codec = \"native_json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + protobuf: { + description: "Protobuf-specific decoding options." + relevant_when: "codec = \"protobuf\"" + required: false + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o `. + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false + type: string: default: "" + } + message_type: { + description: "The name of the message type to use for serializing." + required: false + type: string: { + default: "" + examples: ["package.Message"] + } + } + use_json_names: { + description: """ + Use JSON field names (camelCase) instead of protobuf field names (snake_case). + + When enabled, the deserializer will output fields using their JSON names as defined + in the `.proto` file (e.g., `jobDescription` instead of `job_description`). + + This is useful when working with data that needs to be converted to JSON or + when interfacing with systems that use JSON naming conventions. + """ + required: false + type: bool: default: false + } + } + } + signal_types: { + description: """ + Signal types to attempt parsing, in priority order. + + The deserializer tries to parse signal types in the order specified. This allows you to optimize + performance when you know the expected signal types. For example, if you only receive + traces, set this to `["traces"]` to avoid attempting to parse as logs or metrics first. + + If not specified, defaults to trying all types in order: logs, metrics, traces. + Duplicate signal types are automatically removed while preserving order. + """ + relevant_when: "codec = \"otlp\"" + required: false + type: array: { + default: ["logs", "metrics", "traces"] + items: type: string: enum: { + logs: "OTLP logs signal (ExportLogsServiceRequest)" + metrics: "OTLP metrics signal (ExportMetricsServiceRequest)" + traces: "OTLP traces signal (ExportTraceServiceRequest)" + } + } + } + syslog: { + description: "Syslog-specific decoding options." + relevant_when: "codec = \"syslog\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + vrl: { + description: "VRL-specific decoding options." + relevant_when: "codec = \"vrl\"" + required: true + type: object: options: { + source: { + description: """ + The [Vector Remap Language][vrl] (VRL) program to execute for each event. + Note that the final contents of the `.` target is used as the decoding result. + Compilation error or use of 'abort' in a program results in a decoding error. + + [vrl]: https://vector.dev/docs/reference/vrl + """ + required: true + type: string: {} + } + timezone: { + description: """ + The name of the timezone to apply to timestamp conversions that do not contain an explicit + time zone. The time zone name may be any name in the [TZ database][tz_database], or `local` + to indicate system local time. + + If not set, `local` is used. + + [tz_database]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: examples: ["local", "America/New_York", "EST5EDT"] + } + } + } + } + } + last_run_metadata_path: { + description: """ + The path to the file where the last row of the result set is saved. + The last row of the result set is saved in JSON format. + This file provides parameters for the SQL query in the next scheduled run. + If the file does not exist or the path is not specified, the initial value from `statement_init_params` is used. + + # Examples + + If `tracking_columns = ["id", "name"]`, it is saved as the following JSON data. + + ```json + {"id":1, "name": "vector"} + ``` + """ + required: false + type: string: examples: ["/path/to/tracking.json"] + } + odbc_batch_size: { + description: """ + Number of rows to fetch per batch from the ODBC driver. + The default is 100. + """ + required: false + type: uint: { + default: 100 + examples: [ + 100, + ] + } + } + odbc_default_timezone: { + description: """ + Timezone applied to database date/time columns that lack timezone information. + The default is UTC. + """ + required: false + type: string: { + default: "UTC" + examples: [ + "UTC", + ] + } + } + odbc_max_str_limit: { + description: """ + Maximum string length for ODBC driver operations. + The default is 4096. + """ + required: false + type: uint: { + default: 100 + examples: [ + 4096, + ] + } + } + schedule: { + description: """ + Cron expression used to schedule database queries. + When omitted, the statement runs only once by default. + """ + required: false + type: string: {} + } + schedule_timezone: { + description: """ + The timezone to use for the `schedule`. + Typically the timezone used when evaluating the cron expression. + The default is UTC. + + [Wikipedia]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: { + default: "UTC" + examples: [ + "UTC", + ] + } + } + statement: { + description: """ + The SQL statement to execute. + This SQL statement is executed periodically according to the `schedule`. + Defaults to `None`. If no SQL statement is provided, the source returns an error. + If the `statement_filepath` is set, this value is ignored. + """ + required: false + type: string: examples: ["SELECT * FROM users WHERE id = ?"] + } + statement_filepath: { + description: """ + The path to the file that contains the SQL statement. + If this is unset or the file cannot be read, the value from `statement` is used instead. + """ + required: false + type: string: {} + } + statement_init_params: { + description: """ + Initial parameters for the first execution of the statement. + Used if `last_run_metadata_path` does not exist. + Values must be strings and follow the parameter order defined in the query. + + # Examples + + When the source runs for the first time, the file at `last_run_metadata_path` does not exist. + In that case, declare the initial values in `statement_init_params`. + + ```toml + [sources.odbc] + statement = "SELECT * FROM users WHERE id = ?" + statement_init_params = { "id": "0" } + tracking_columns = ["id"] + last_run_metadata_path = "/path/to/tracking.json" + # The rest of the fields are omitted + ``` + """ + required: false + type: object: options: "*": { + description: "Initial value for the SQL statement parameters. The value is always a string." + required: true + type: "*": {} + } + } + statement_timeout: { + description: """ + Maximum time to allow the SQL statement to run. + If the query does not finish within this window, it is canceled and retried at the next scheduled run. + The default is 3 seconds. + """ + required: false + type: uint: { + default: 3 + examples: [ + 3, + ] + unit: "seconds" + } + } + tracking_columns: { + description: """ + Specifies the columns to track from the last row of the statement result set. + Their values are passed as parameters to the SQL statement in the next scheduled run. + + # Examples + + ```toml + [sources.odbc] + statement = "SELECT * FROM users WHERE id = ?" + tracking_columns = ["id"] + # The rest of the fields are omitted + ``` + """ + required: false + type: array: items: type: string: examples: ["id"] + } +} diff --git a/website/cue/reference/components/sources/odbc.cue b/website/cue/reference/components/sources/odbc.cue new file mode 100644 index 0000000000000..57d667d929cc3 --- /dev/null +++ b/website/cue/reference/components/sources/odbc.cue @@ -0,0 +1,176 @@ +package metadata + +components: sources: odbc: { + title: "ODBC" + + classes: { + commonly_used: false + delivery: "at_least_once" + deployment_roles: ["daemon", "sidecar", "aggregator"] + development: "beta" + egress_method: "batch" + stateful: true + } + + features: { + auto_generated: true + acknowledgements: true + collect: { + checkpoint: enabled: false + from: { + service: services.odbc + } + } + multiline: enabled: true + encoding: enabled: true + } + + support: { + requirements: [] + warnings: [] + notices: [] + } + + installation: { + platform_name: null + } + + configuration: generated.components.sources.odbc.configuration + + output: { + logs: record: { + description: "Records returned by the ODBC query." + fields: { + message: { + description: "The ODBC query result serialized as JSON." + required: true + type: string: { + examples: [ + """ + [{"id":1,"name":"test1"}] + """, + ] + } + } + timestamp: fields._current_timestamp + } + } + } + + how_it_works: { + requirement: { + title: "Requirement for unixODBC" + body: """ + To connect to a database and execute queries via ODBC, you must have the unixODBC package installed. + First, use your package manager to install the `unixodbc` package. + Then, install and configure the appropriate ODBC driver. + + For example, on Debian-based Linux, you can install the `unixodbc` and `odbc-mariadb` packages as follows: + ```bash + # apt-get install unixodbc odbcinst odbc-mariadb + ``` + + You can use the `odbcinst -j` command to check the installation path and configuration files for unixODBC. + ```bash + $ odbcinst -j + unixODBC 2.3.12 + DRIVERS............: /etc/odbcinst.ini + SYSTEM DATA SOURCES: /etc/odbc.ini + FILE DATA SOURCES..: /etc/ODBCDataSources + USER DATA SOURCES..: /root/.odbc.ini + SQLULEN Size.......: 8 + SQLLEN Size........: 8 + SQLSETPOSIROW Size.: 8 + ``` + + Review the `/etc/odbcinst.ini` file in the output to ensure the ODBC driver is properly configured. + If you installed the ODBC driver via a package manager, it is usually configured automatically. + When you install the `odbc-mariadb` package, the `odbcinst.ini` file will be configured as follows: + ```bash + $ cat /etc/odbcinst.ini + + [MariaDB Unicode] + Driver=libmaodbc.so + Description=MariaDB Connector/ODBC(Unicode) + Threading=0 + UsageCount=1 + ``` + """ + } + + examples: { + title: "Example ODBC Source Configuration" + body: """ + This section walks through a simple example of configuring an ODBC data source and scheduling it. + """ + sub_sections: [ + { + title: "Step 1: Configure Test Data" + body: """ + Given the following MariaDB table and sample data: + + ```sql + create table odbc_table + ( + id int auto_increment primary key, + name varchar(255) null, + `datetime` datetime null + ); + + INSERT INTO odbc_table (name, datetime) VALUES + ('test1', now()), + ('test2', now()), + ('test3', now()), + ('test4', now()), + ('test5', now()); + ``` + """ + }, + { + title: "Step 2: Configure ODBC Source" + body: """ + The example below shows how to connect to a MariaDB database with the ODBC driver, + run a query periodically, and send the results to Vector. + Start by providing a database connection string. + + ```toml + [sources.odbc] + type = "odbc" + connection_string = "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=;" + statement = "SELECT * FROM odbc_table WHERE id > ? LIMIT 1;" + statement_init_params = { id = "0" } + schedule = "*/5 * * * * *" + schedule_timezone = "UTC" + last_run_metadata_path = "/path/to/odbc_tracking.json" + tracking_columns = ["id"] + + [sinks.console] + type = "console" + inputs = ["odbc"] + encoding.codec = "json" + ``` + + Every five seconds, the source produces output similar to the following. + + ```json + {"message":[{"datetime":"2025-04-28T01:20:04Z","id":1,"name":"test1"}],"timestamp":"2025-04-28T01:50:45.075484Z"} + {"message":[{"datetime":"2025-04-28T01:20:04Z","id":2,"name":"test2"}],"timestamp":"2025-04-28T01:50:50.017276Z"} + {"message":[{"datetime":"2025-04-28T01:20:04Z","id":3,"name":"test3"}],"timestamp":"2025-04-28T01:50:55.016432Z"} + {"message":[{"datetime":"2025-04-28T01:20:04Z","id":4,"name":"test4"}],"timestamp":"2025-04-28T01:51:00.016328Z"} + {"message":[{"datetime":"2025-04-28T01:20:04Z","id":5,"name":"test5"}],"timestamp":"2025-04-28T01:51:05.010063Z"} + """ + }, + ] + } + + check_license: { + title: "Check ODBC Driver License" + body: """ + Review the license information on [the official unixODBC website](\(urls.unixodbc)). + + Because ODBC drivers are supplied by various vendors, each with different license terms, + be sure to review and comply with the terms for the driver you plan to use. + """ + } + } +} diff --git a/website/cue/reference/services/odbc.cue b/website/cue/reference/services/odbc.cue new file mode 100644 index 0000000000000..45cce15567c57 --- /dev/null +++ b/website/cue/reference/services/odbc.cue @@ -0,0 +1,8 @@ +package metadata + +services: odbc: { + name: "ODBC" + thing: "an \(name) datasource" + url: urls.odbc + versions: null +} diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 2578c0b38c60a..67acdf0b7dcbf 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -393,6 +393,7 @@ urls: { nix: "https://nixos.org/nix/" nixos: "https://nixos.org/" nixpkgs_9682: "\(github)/NixOS/nixpkgs/issues/9682" + odbc: "\(wikipedia)/wiki/Open_Database_Connectivity" openssl: "https://www.openssl.org/" openssl_conf: "https://www.openssl.org/docs/man3.1/man5/config.html" opentelemetry: "https://opentelemetry.io" @@ -539,6 +540,7 @@ urls: { uds: "\(wikipedia)/wiki/Unix_domain_socket" unicode_replacement_character: "\(wikipedia)/wiki/Specials_(Unicode_block)#Replacement_character" unicode_whitespace: "\(wikipedia)/wiki/Unicode_character_property#Whitespace" + unixodbc: "https://www.unixodbc.org/" unix_timestamp: "\(wikipedia)/wiki/Unix_time" utf8: "\(wikipedia)/wiki/UTF-8" uuidv4: "\(wikipedia)/wiki/Universally_unique_identifier#Version_4_(random)"