From 52862653c61a05ffff52c4dfaf0440f2ab1ee941 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 11:15:54 -0500 Subject: [PATCH 01/16] actually run test in ci --- .github/workflows/rust.yml | 3 +++ datafusion/wasmtest/README.md | 2 -- datafusion/wasmtest/src/lib.rs | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1e6cd97acea3..9f57bcd82e37 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -403,6 +403,9 @@ jobs: - name: Build with wasm-pack working-directory: ./datafusion/wasmtest run: wasm-pack build --dev + - run: wasm-pack test --headless --chrome + - run: wasm-pack test --headless --firefox + - run: wasm-pack test --headless --safari # verify that the benchmark queries return the correct results verify-benchmark-results: diff --git a/datafusion/wasmtest/README.md b/datafusion/wasmtest/README.md index 8843eed697ec..70f4daef9103 100644 --- a/datafusion/wasmtest/README.md +++ b/datafusion/wasmtest/README.md @@ -71,8 +71,6 @@ wasm-pack test --headless --chrome wasm-pack test --headless --safari ``` -**Note:** In GitHub Actions we test the compilation with `wasm-build`, but we don't currently invoke `wasm-pack test`. This is because the headless mode is not yet working. Document of adding a GitHub Action job: https://rustwasm.github.io/docs/wasm-bindgen/wasm-bindgen-test/continuous-integration.html#github-actions. - To tweak timeout setting, use `WASM_BINDGEN_TEST_TIMEOUT` environment variable. E.g., `WASM_BINDGEN_TEST_TIMEOUT=300 wasm-pack test --firefox --headless`. ## Compatibility diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rs index 6c7be9056eb4..5a2256492ec0 100644 --- a/datafusion/wasmtest/src/lib.rs +++ b/datafusion/wasmtest/src/lib.rs @@ -98,7 +98,6 @@ mod test { }; use datafusion_physical_plan::collect; use datafusion_sql::parser::DFParser; - use insta::assert_snapshot; use object_store::{memory::InMemory, path::Path, ObjectStore}; use url::Url; use wasm_bindgen_test::wasm_bindgen_test; @@ -248,14 +247,15 @@ mod test { let result = df.collect().await.unwrap(); - assert_snapshot!(batches_to_string(&result), @r" - +----+-------+ - | id | value | - +----+-------+ - | 1 | a | - | 2 | b | - | 3 | c | - +----+-------+ - "); + assert_eq!( + batches_to_string(&result), + "+----+-------+\n\ + | id | value |\n\ + +----+-------+\n\ + | 1 | a |\n\ + | 2 | b |\n\ + | 3 | c |\n\ + +----+-------+" + ); } } From 40d362cbe68adef4b0ab24ef2a1a72bc00a4d23f Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 11:37:40 -0500 Subject: [PATCH 02/16] tweak ci --- .github/workflows/rust.yml | 11 ++++++++--- datafusion/wasmtest/src/lib.rs | 8 ++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9f57bcd82e37..34d8566f64e7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -403,9 +403,14 @@ jobs: - name: Build with wasm-pack working-directory: ./datafusion/wasmtest run: wasm-pack build --dev - - run: wasm-pack test --headless --chrome - - run: wasm-pack test --headless --firefox - - run: wasm-pack test --headless --safari + - name: Run tests with headless mode + working-directory: ./datafusion/wasmtest + env: + WASM_BINDGEN_TEST_TIMEOUT: 300 + run: | + wasm-pack test --headless --chrome + wasm-pack test --headless --firefox + wasm-pack test --headless --safari # verify that the benchmark queries return the correct results verify-benchmark-results: diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rs index 5a2256492ec0..0a7e546b4b18 100644 --- a/datafusion/wasmtest/src/lib.rs +++ b/datafusion/wasmtest/src/lib.rs @@ -82,7 +82,6 @@ pub fn basic_parse() { #[cfg(test)] mod test { use super::*; - use datafusion::execution::options::ParquetReadOptions; use datafusion::{ arrow::{ array::{ArrayRef, Int32Array, RecordBatch, StringArray}, @@ -239,12 +238,13 @@ mod test { let url = Url::parse("memory://").unwrap(); session_ctx.register_object_store(&url, Arc::new(store)); - - let df = session_ctx - .read_parquet("memory:///", ParquetReadOptions::new()) + session_ctx + .register_parquet("a", "memory:///a.parquet", Default::default()) .await .unwrap(); + let df = session_ctx.sql("SELECT * FROM a").await.unwrap(); + let result = df.collect().await.unwrap(); assert_eq!( From c0cc1a30b4ace6700b363c873c6bacc0d1703446 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 11:59:54 -0500 Subject: [PATCH 03/16] try ci again --- .github/workflows/rust.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 34d8566f64e7..b5026b1e590f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -398,6 +398,7 @@ jobs: run: | apt-get update -qq apt-get install -y -qq clang + apt-get install -y -qq chromium-browser chromium-chromedriver - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: Build with wasm-pack @@ -408,9 +409,9 @@ jobs: env: WASM_BINDGEN_TEST_TIMEOUT: 300 run: | - wasm-pack test --headless --chrome wasm-pack test --headless --firefox wasm-pack test --headless --safari + wasm-pack test --headless --chrome # verify that the benchmark queries return the correct results verify-benchmark-results: From 4ea6b81b188f1f81a5bc6803925a172bc7b50d1a Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 12:02:31 -0500 Subject: [PATCH 04/16] try ci again --- .github/workflows/rust.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b5026b1e590f..c768c95614af 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -398,7 +398,6 @@ jobs: run: | apt-get update -qq apt-get install -y -qq clang - apt-get install -y -qq chromium-browser chromium-chromedriver - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: Build with wasm-pack @@ -411,7 +410,6 @@ jobs: run: | wasm-pack test --headless --firefox wasm-pack test --headless --safari - wasm-pack test --headless --chrome # verify that the benchmark queries return the correct results verify-benchmark-results: From bfdfa6606d11f9246cf7f39c8d301c11402946a4 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 12:26:00 -0500 Subject: [PATCH 05/16] try ci again --- .github/workflows/rust.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c768c95614af..b2d7ebc566f5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -400,16 +400,12 @@ jobs: apt-get install -y -qq clang - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - name: Build with wasm-pack - working-directory: ./datafusion/wasmtest - run: wasm-pack build --dev - name: Run tests with headless mode working-directory: ./datafusion/wasmtest env: WASM_BINDGEN_TEST_TIMEOUT: 300 run: | - wasm-pack test --headless --firefox - wasm-pack test --headless --safari + wasm-pack test --headless --firefox --chrome # verify that the benchmark queries return the correct results verify-benchmark-results: From 8f9b1efbab96f8c134bb9f8e09f035f6659a5fe3 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 12:39:26 -0500 Subject: [PATCH 06/16] setup debug --- .github/workflows/rust.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b2d7ebc566f5..186c99f0f8d9 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -400,6 +400,8 @@ jobs: apt-get install -y -qq clang - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 - name: Run tests with headless mode working-directory: ./datafusion/wasmtest env: From cca346d5ca6fb1fe0552ea33b260974b1d3d4c44 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 12:56:11 -0500 Subject: [PATCH 07/16] update --- .github/workflows/rust.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 186c99f0f8d9..7169602056f3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -386,8 +386,6 @@ jobs: linux-wasm-pack: name: build with wasm-pack runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain @@ -397,17 +395,15 @@ jobs: - name: Install dependencies run: | apt-get update -qq - apt-get install -y -qq clang + apt-get install -y -qq clang chromium-chromedriver chromium-browser - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - name: Run tests with headless mode working-directory: ./datafusion/wasmtest env: WASM_BINDGEN_TEST_TIMEOUT: 300 run: | - wasm-pack test --headless --firefox --chrome + wasm-pack test --headless --chrome # verify that the benchmark queries return the correct results verify-benchmark-results: From 37e6def082bd65db042e2139c773c99125989bfb Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 12:58:11 -0500 Subject: [PATCH 08/16] try again --- .github/workflows/rust.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7169602056f3..f3c3d6ccd709 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -386,6 +386,8 @@ jobs: linux-wasm-pack: name: build with wasm-pack runs-on: ubuntu-latest + container: + image: amd64/rust steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain From ae9c0ccb6f68d3a0c15e65b2a9f3bc185e6e6dd8 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 13:06:20 -0500 Subject: [PATCH 09/16] update ci --- .github/workflows/rust.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f3c3d6ccd709..36baa6b3396b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -397,14 +397,24 @@ jobs: - name: Install dependencies run: | apt-get update -qq - apt-get install -y -qq clang chromium-chromedriver chromium-browser + apt-get install -y -qq clang - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Install Chrome Environment + run: | + mkdir -p /tmp/chrome + wget $(curl https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r '.versions | sort_by(.version) | reverse | .[0] | .downloads.chrome | .[] | select(.platform == "linux64") | .url') + wget $(curl https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r '.versions | sort_by(.version) | reverse | .[0] | .downloads.chromedriver | .[] | select(.platform == "linux64") | .url') + unzip chromedriver-linux64.zip + unzip chrome-linux64.zip + cp -r chrome-linux64/ /tmp/chrome/ + cp -r chromedriver-linux64 /tmp/chrome/chromedriver - name: Run tests with headless mode working-directory: ./datafusion/wasmtest env: WASM_BINDGEN_TEST_TIMEOUT: 300 run: | + export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver-linux64/ wasm-pack test --headless --chrome # verify that the benchmark queries return the correct results From 520c966658a0398583bd8058f95c73992062d5d8 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 13:11:09 -0500 Subject: [PATCH 10/16] install jq --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 36baa6b3396b..a6b976d29656 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -397,7 +397,7 @@ jobs: - name: Install dependencies run: | apt-get update -qq - apt-get install -y -qq clang + apt-get install -y -qq clang jq - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: Install Chrome Environment From d01bfd52dcd24b4cd595f13bdd2a4cad24a0ed00 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 13:41:15 -0500 Subject: [PATCH 11/16] update ci --- .github/workflows/rust.yml | 13 +++++++++---- datafusion/wasmtest/webdriver.json | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 datafusion/wasmtest/webdriver.json diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a6b976d29656..11f01676d475 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -394,12 +394,17 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Setup for wasm32 + run: | + rustup target add wasm32-unknown-unknown - name: Install dependencies run: | apt-get update -qq apt-get install -y -qq clang jq - - name: Install wasm-pack - run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + + - name: Setup wasm-pack + run: | + cargo install wasm-pack - name: Install Chrome Environment run: | mkdir -p /tmp/chrome @@ -414,8 +419,8 @@ jobs: env: WASM_BINDGEN_TEST_TIMEOUT: 300 run: | - export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver-linux64/ - wasm-pack test --headless --chrome + export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver/ + wasm-pack test --headless --chrome --chrome-driver /tmp/chrome/chromedriver/chromedriver # verify that the benchmark queries return the correct results verify-benchmark-results: diff --git a/datafusion/wasmtest/webdriver.json b/datafusion/wasmtest/webdriver.json new file mode 100644 index 000000000000..f59a2be9955f --- /dev/null +++ b/datafusion/wasmtest/webdriver.json @@ -0,0 +1,15 @@ +{ + "moz:firefoxOptions": { + "prefs": { + "media.navigator.streams.fake": true, + "media.navigator.permission.disabled": true + }, + "args": [] + }, + "goog:chromeOptions": { + "args": [ + "--use-fake-device-for-media-stream", + "--use-fake-ui-for-media-stream" + ] + } +} \ No newline at end of file From 6ed9dd14bc1daf9679c3b1eb324b24cc6bbff42d Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 13:46:15 -0500 Subject: [PATCH 12/16] fix ci --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 11f01676d475..7741e8b0223f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -420,7 +420,7 @@ jobs: WASM_BINDGEN_TEST_TIMEOUT: 300 run: | export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver/ - wasm-pack test --headless --chrome --chrome-driver /tmp/chrome/chromedriver/chromedriver + wasm-pack test --headless --chrome --chromedriver /tmp/chrome/chromedriver/chromedriver # verify that the benchmark queries return the correct results verify-benchmark-results: From 3e206e6eafb0df10210ac5b00812be9cb7244133 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 16:01:12 -0500 Subject: [PATCH 13/16] update --- .github/workflows/rust.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7741e8b0223f..bac4ddd28a9d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -386,14 +386,8 @@ jobs: linux-wasm-pack: name: build with wasm-pack runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Setup for wasm32 run: | rustup target add wasm32-unknown-unknown From cf3db093aa49c991731f4d16241115727da24375 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 16:02:41 -0500 Subject: [PATCH 14/16] update --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bac4ddd28a9d..1704166bb0b1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -393,8 +393,8 @@ jobs: rustup target add wasm32-unknown-unknown - name: Install dependencies run: | - apt-get update -qq - apt-get install -y -qq clang jq + sudo apt-get update -qq + sudo apt-get install -y -qq clang jq - name: Setup wasm-pack run: | From e6c145cb2703d46586d594e852b8c0c02d4cde85 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sat, 5 Apr 2025 16:20:34 -0500 Subject: [PATCH 15/16] try again --- .github/workflows/rust.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1704166bb0b1..2b1af78eb334 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -385,7 +385,7 @@ jobs: linux-wasm-pack: name: build with wasm-pack - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Setup for wasm32 @@ -394,27 +394,15 @@ jobs: - name: Install dependencies run: | sudo apt-get update -qq - sudo apt-get install -y -qq clang jq - + sudo apt-get install -y -qq clang - name: Setup wasm-pack run: | cargo install wasm-pack - - name: Install Chrome Environment - run: | - mkdir -p /tmp/chrome - wget $(curl https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r '.versions | sort_by(.version) | reverse | .[0] | .downloads.chrome | .[] | select(.platform == "linux64") | .url') - wget $(curl https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r '.versions | sort_by(.version) | reverse | .[0] | .downloads.chromedriver | .[] | select(.platform == "linux64") | .url') - unzip chromedriver-linux64.zip - unzip chrome-linux64.zip - cp -r chrome-linux64/ /tmp/chrome/ - cp -r chromedriver-linux64 /tmp/chrome/chromedriver - name: Run tests with headless mode working-directory: ./datafusion/wasmtest - env: - WASM_BINDGEN_TEST_TIMEOUT: 300 run: | - export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver/ - wasm-pack test --headless --chrome --chromedriver /tmp/chrome/chromedriver/chromedriver + wasm-pack test --headless --firefox + wasm-pack test --headless --chrome --chromedriver $CHROMEWEBDRIVER/chromedriver # verify that the benchmark queries return the correct results verify-benchmark-results: From 775464ced4582bc2b86ac7a01801ab51c4f9a2f8 Mon Sep 17 00:00:00 2001 From: Xiangpeng Hao Date: Sun, 6 Apr 2025 10:43:13 -0400 Subject: [PATCH 16/16] Update .github/workflows/rust.yml Co-authored-by: Andrew Lamb --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2b1af78eb334..3fa8ce080474 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -384,7 +384,7 @@ jobs: run: ci/scripts/rust_docs.sh linux-wasm-pack: - name: build with wasm-pack + name: build and run with wasm-pack runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4