Merge pull request #25 from isankadn/dev

na

historical_data/src/main.rs

@@ -307,20 +307,11 @@ async fn deduplicate_clickhouse_data(
             return Err(e.into());
         }
     };
+
     info!("processing duplicate data...");
+
     let create_dedup_table_query = format!(
-        "CREATE TABLE {table}_dedup
-        ENGINE = MergeTree()
-        PARTITION BY toYYYYMM(created_at)
-        PRIMARY KEY id
-        ORDER BY (id, created_at)
-        SETTINGS index_granularity = 8192
-        AS
-        SELECT * FROM (
-            SELECT *, row_number() OVER (PARTITION BY id ORDER BY created_at) AS row_num
-            FROM {table}
-        )
-        WHERE row_num = 1;",
+        "CREATE TABLE {table}_dedup ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at) PRIMARY KEY id ORDER BY (id, created_at) SETTINGS index_granularity = 8192 AS SELECT * FROM {table} GROUP BY id",
         table = full_table_name
     );