From 0ae89647903cf46de229fc2023490b4e8419b4cd Mon Sep 17 00:00:00 2001
From: Vinicius Mesel <bioinfo.vinicius@gmail.com>
Date: Wed, 22 Nov 2023 18:22:09 -0300
Subject: [PATCH] Fix/hgi 4620 (#13)
* Fixes table truncate
* Fixes truncate
* Adds self.truncate back
* Resets truncate, assigning it to new variable and doing boolean operation
* Resets the instance_increment variable on every iteration
---
target_bigquery/processhandler.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/target_bigquery/processhandler.py b/target_bigquery/processhandler.py
index 374f407..39d8c7f 100644
--- a/target_bigquery/processhandler.py
+++ b/target_bigquery/processhandler.py
@@ -293,9 +293,8 @@ def _do_temp_table_based_load(self, rows):
instance_truncate = self.truncate or self.table_configs.get(stream, {}).get("truncate", False)
if instance_truncate:
self.logger.info(f"Truncating dataset: {stream}")
-
- self.incremental = self.incremental if not self.truncate else False
- if self.incremental:
+ instance_increment = self.incremental if not instance_truncate else False
+ if instance_increment:
self.logger.info(f"Copy {tmp_table_name} to {self.tables[stream]} by INCREMENTAL")
self.logger.warning(f"INCREMENTAL replication method (MERGE SQL statement) is not recommended. It might result in loss of production data, because historical records get updated during the sync operation. Instead, we recommend using the APPEND replication method, which will preserve historical data.")
table_id = f"{self.project_id}.{self.dataset.dataset_id}.{self.tables[stream]}"