Skip to content

Commit

Permalink
na
Browse files Browse the repository at this point in the history
  • Loading branch information
isankadn committed Mar 25, 2024
1 parent dab4932 commit 7ae66aa
Showing 1 changed file with 3 additions and 12 deletions.
15 changes: 3 additions & 12 deletions historical_data/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -307,20 +307,11 @@ async fn deduplicate_clickhouse_data(
return Err(e.into());
}
};

info!("processing duplicate data...");

let create_dedup_table_query = format!(
"CREATE TABLE {table}_dedup
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
PRIMARY KEY id
ORDER BY (id, created_at)
SETTINGS index_granularity = 8192
AS
SELECT * FROM (
SELECT *, row_number() OVER (PARTITION BY id ORDER BY created_at) AS row_num
FROM {table}
)
WHERE row_num = 1;",
"CREATE TABLE {table}_dedup ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at) PRIMARY KEY id ORDER BY (id, created_at) SETTINGS index_granularity = 8192 AS SELECT * FROM {table} GROUP BY id",
table = full_table_name
);

Expand Down

0 comments on commit 7ae66aa

Please sign in to comment.