From 7ae66aacbd09a6f201bfedf8bd55876841f35695 Mon Sep 17 00:00:00 2001
From: Isanka <isankadn@gmail.com>
Date: Mon, 25 Mar 2024 13:34:02 +0900
Subject: [PATCH] na

---
 historical_data/src/main.rs | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/historical_data/src/main.rs b/historical_data/src/main.rs
index e037c6b..4614d3f 100644
--- a/historical_data/src/main.rs
+++ b/historical_data/src/main.rs
@@ -307,20 +307,11 @@ async fn deduplicate_clickhouse_data(
             return Err(e.into());
         }
     };
+
     info!("processing duplicate data...");
+
     let create_dedup_table_query = format!(
-        "CREATE TABLE {table}_dedup
-        ENGINE = MergeTree()
-        PARTITION BY toYYYYMM(created_at)
-        PRIMARY KEY id
-        ORDER BY (id, created_at)
-        SETTINGS index_granularity = 8192
-        AS
-        SELECT * FROM (
-            SELECT *, row_number() OVER (PARTITION BY id ORDER BY created_at) AS row_num
-            FROM {table}
-        )
-        WHERE row_num = 1;",
+        "CREATE TABLE {table}_dedup ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at) PRIMARY KEY id ORDER BY (id, created_at) SETTINGS index_granularity = 8192 AS SELECT * FROM {table} GROUP BY id",
         table = full_table_name
     );