From 47a8e5e6ccfc9b0f987ce1e1151935ef7e2f6610 Mon Sep 17 00:00:00 2001 From: Isanka Date: Sun, 24 Mar 2024 16:25:06 +0900 Subject: [PATCH] na --- historical_data/src/main.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/historical_data/src/main.rs b/historical_data/src/main.rs index 7a8a38a..711760b 100644 --- a/historical_data/src/main.rs +++ b/historical_data/src/main.rs @@ -238,9 +238,15 @@ async fn deduplicate_clickhouse_data( }; let create_dedup_table_query = format!( - "CREATE TABLE {table}_dedup AS + "CREATE TABLE {table}_dedup + ENGINE = MergeTree() + PARTITION BY toYYYYMM(created_at) + PRIMARY KEY id + ORDER BY (id, created_at) + SETTINGS index_granularity = 8192 + AS SELECT * FROM ( - SELECT *, row_number() OVER (PARTITION BY id ORDER BY id) AS row_num + SELECT *, row_number() OVER (PARTITION BY id ORDER BY created_at) AS row_num FROM {table} ) WHERE row_num = 1;",