Skip to content

Commit

Permalink
Automatically fall back to truncate if no key_properties
Browse files Browse the repository at this point in the history
  • Loading branch information
hsyyid committed Jan 9, 2025
1 parent 099f23f commit a9f0e38
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 2 deletions.
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

setup(
name="target-bigquery",
version="0.11.13",
version="0.11.14",
description="Singer.io target for writing data to Google BigQuery",
author="Adswerve",
url="https://github.com/adswerve/target-bigquery",
Expand Down
10 changes: 9 additions & 1 deletion target_bigquery/processhandler.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,9 +307,17 @@ def _do_temp_table_based_load(self, rows):
self.create_missing_columns(stream)
incremental_success = False
instance_truncate = self.truncate or self.table_configs.get(stream, {}).get("truncate", False) or self.table_configs.get(stream, {}).get("replication_method") == "truncate"
instance_increment = self.incremental if not instance_truncate else False

key_properties = [k.replace(".", "_") for k in self.key_properties[stream]]

if instance_increment and not key_properties:
# Fall back to truncate because there's no PK to upsert on
self.logger.info(f"Falling back to truncate because {stream} has no key properties")
instance_truncate = True

if instance_truncate:
self.logger.info(f"Truncating dataset: {stream}")
instance_increment = self.incremental if not instance_truncate else False
# For larger jobs we don't want to keep truncating the same table when copy temporary table to production
# using this change we will switch truncate logic off and make subsequent copies to incremental
if stream in self.truncate_counts and self.truncate_counts.get(stream, 0) > 0:
Expand Down

0 comments on commit a9f0e38

Please sign in to comment.