Add Ion Scan (#544)

Author: jpschorr

Cargo.toml

@@ -39,6 +39,10 @@ opt-level = 3
 [profile.release.build-override]
 opt-level = 3
 
+[profile.dev.package]
+insta.opt-level = 3
+similar.opt-level = 3
+
 [profile.bench]
 debug = true
 lto = true

extension/partiql-extension-ion-functions/Cargo.toml

@@ -31,14 +31,15 @@ itertools = "0.13"
 unicase = "2.7"
 rust_decimal = { version = "1.36.0", default-features = false, features = ["std"] }
 ion-rs_old = { version = "0.18", package = "ion-rs" }
-ion-rs = { version = "1.0.0-rc.10", features = ["experimental"] }
+ion-rs = { version = "1.0.0-rc.11", features = ["experimental"] }
 time = { version = "0.3", features = ["macros"] }
 once_cell = "1"
 regex = "1.10"
 thiserror = "1.0"
 delegate = "0.13"
 zstd = "0.13"
 flate2 = "1.0"
+insta = "1"
 
 [dev-dependencies]
 partiql-parser = { path = "../../partiql-parser", version = "0.11.*" }

extension/partiql-extension-ion-functions/resources/test/ion_passthrough_structs_test.ion

@@ -0,0 +1,5 @@
+{data: 1}
+{data: "foo"}
+{data: ["a", 1, null]}
+{data: {a: 1, b: {{}}, }}
+{data: (sym::var 1 2 3)}

extension/partiql-extension-ion-functions/resources/test/ion_passthrough_test.bad.ion

@@ -0,0 +1,5 @@
+{data: 1}
+{data: "foo"}
+{data: ["a", 1, null]}
+{data: {a: 1, b: {{}}, }}
+{data: (sym::var 1 2 3} // **NOTE** the unclosed SExp; should cause a parse error

extension/partiql-extension-ion-functions/resources/test/ion_passthrough_test.ion

@@ -0,0 +1,5 @@
+1
+"foo"
+["a", 1, null]
+{a: 1, b: {{}}, }
+(sym::var 1 2 3)

extension/partiql-extension-ion-functions/src/buffer.rs

@@ -0,0 +1,29 @@
+use flate2::read::GzDecoder;
+use std::io::{BufReader, Read, Seek, SeekFrom};
+use zstd::Decoder;
+
+pub(crate) enum BufferType<'a, I: 'a + Read + 'static> {
+    Gzip(BufReader<GzDecoder<I>>),
+    Zstd(BufReader<Decoder<'a, BufReader<I>>>),
+    Unknown(BufReader<I>),
+}
+
+pub(crate) fn infer_buffer_type<'a, I: 'a + Read + Seek + 'static>(
+    mut reader: I,
+) -> BufferType<'a, I> {
+    let mut header: [u8; 4] = [0; 4];
+    reader.read_exact(&mut header).expect("file header");
+    reader.seek(SeekFrom::Start(0)).expect("file seek");
+
+    if header.starts_with(&[0x1f, 0x8b]) {
+        // Cf. https://datatracker.ietf.org/doc/html/rfc1952#page-6 section 2.3.1
+        BufferType::Gzip(BufReader::new(flate2::read::GzDecoder::new(reader)))
+    } else if header.starts_with(&[0x28, 0xB5, 0x2F, 0xFD]) {
+        // Cf. https://datatracker.ietf.org/doc/rfc8478/ section 3.1.1
+        BufferType::Zstd(BufReader::new(
+            zstd::Decoder::new(reader).expect("zstd reader creation"),
+        ))
+    } else {
+        BufferType::Unknown(BufReader::new(reader))
+    }
+}