-
Notifications
You must be signed in to change notification settings - Fork 171
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #302 from Mytherin/duckdb-v1.2.0
- Loading branch information
Showing
15 changed files
with
250 additions
and
280 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,30 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Install | ||
|
|
||
| sudo apt-get update | ||
| sudo apt-get install -y python3-pip | ||
| pip install --break-system-packages duckdb==1.1.3 psutil | ||
| sudo apt-get install ninja-build cmake build-essential make ccache pip clang -y | ||
|
|
||
| export CC=clang | ||
| export CXX=clang++ | ||
| git clone https://github.com/duckdb/duckdb | ||
| cd duckdb | ||
| git checkout v1.2-histrionicus | ||
| GEN=ninja NATIVE_ARCH=1 LTO=thin make | ||
| export PATH="$PATH:`pwd`/build/release/" | ||
| cd .. | ||
|
|
||
| # Load the data | ||
| seq 0 99 | xargs -P100 -I{} bash -c 'wget --no-verbose --continue https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet' | ||
|
|
||
| ./load.py | ||
| time duckdb hits.db -f create.sql | ||
|
|
||
| # Run the queries | ||
|
|
||
| ./run.sh 2>&1 | tee log.txt | ||
|
|
||
| wc -c my-db.duckdb | ||
| wc -c hits.db | ||
|
|
||
| cat log.txt | grep -P '^\d|Killed|Segmentation' | sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/' | | ||
| awk '{ if (i % 3 == 0) { printf "[" }; printf $1; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }' | ||
| cat log.txt | | ||
| grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' | | ||
| sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' | | ||
| awk '{ if (i % 3 == 0) { printf "[" }; printf $1; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| CREATE VIEW hits AS | ||
| SELECT * | ||
| REPLACE | ||
| (epoch_ms(EventTime * 1000) AS EventTime, | ||
| DATE '1970-01-01' + INTERVAL (EventDate) DAYS AS EventDate) | ||
| FROM read_parquet('hits_*.parquet', binary_as_string=True); | ||
| REPLACE (make_date(EventDate) AS EventDate) | ||
| FROM read_parquet('hits_*.parquet', binary_as_string=True); | ||
|
|
||
| CREATE MACRO toDateTime(t) AS epoch_ms(t * 1000); |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,21 @@ | ||
| #!/bin/bash | ||
|
|
||
| TRIES=3 | ||
|
|
||
| cat queries.sql | while read -r query; do | ||
| sync | ||
| echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null | ||
| echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null | ||
|
|
||
| ./query.py <<< "${query}" | ||
| done | ||
| echo "$query"; | ||
| cli_params=() | ||
| cli_params+=("-c") | ||
| cli_params+=("SET parquet_metadata_cache=true") | ||
| cli_params+=("-c") | ||
| cli_params+=(".timer on") | ||
| for i in $(seq 1 $TRIES); do | ||
| cli_params+=("-c") | ||
| cli_params+=("${query}") | ||
| done; | ||
| echo "${cli_params[@]}" | ||
| duckdb hits.db "${cli_params[@]}" | ||
| done; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,31 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Install | ||
|
|
||
| sudo apt-get update | ||
| sudo apt-get install -y python3-pip | ||
| pip install --break-system-packages duckdb==1.1.3 psutil | ||
| sudo apt-get install ninja-build cmake build-essential make ccache pip clang -y | ||
|
|
||
| # Load the data | ||
| export CC=clang | ||
| export CXX=clang++ | ||
| git clone https://github.com/duckdb/duckdb | ||
| cd duckdb | ||
| git checkout v1.2-histrionicus | ||
| GEN=ninja NATIVE_ARCH=1 LTO=thin make | ||
| export PATH="$PATH:`pwd`/build/release/" | ||
| cd .. | ||
|
|
||
| wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz' | ||
| gzip -d hits.csv.gz | ||
| # Load the data | ||
| wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' | ||
| gzip -d hits.tsv.gz | ||
|
|
||
| ./load.py | ||
| time duckdb hits.db -f create.sql -c "COPY hits FROM 'hits.tsv' (QUOTE '')" | ||
|
|
||
| # Run the queries | ||
|
|
||
| ./run.sh 2>&1 | tee log.txt | ||
|
|
||
| wc -c my-db.duckdb | ||
| wc -c hits.db | ||
|
|
||
| cat log.txt | grep -P '^\d|Killed|Segmentation' | sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/' | | ||
| awk '{ if (i % 3 == 0) { printf "[" }; printf $1; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }' | ||
| cat log.txt | | ||
| grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' | | ||
| sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' | | ||
| awk '{ if (i % 3 == 0) { printf "[" }; printf $1; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }' |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.