Merge pull request #23 from kevinsunny1996/task/add_dependency

added dependency between parquet file clearup and update page number
kevinsunny1996 · May 13, 2024 · b750045 · b750045
2 parents e87a4f7 + f39c96d
commit b750045
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/dags/rawg_api_extractor_dag.py b/dags/rawg_api_extractor_dag.py
@@ -536,8 +536,7 @@ def update_page_number(rawg_page_number: int) -> int:
         next_page_number = int(rawg_page_number) + 1
         Variable.set("api_page_number", next_page_number)
 
-    remove_extracted_api_parquet_files(rawg_landing_gcs_bucket)
-    update_page_number(rawg_page_number)
+    remove_extracted_api_parquet_files(rawg_landing_gcs_bucket) >> update_page_number(rawg_page_number)
 
   # DAG Flow
   initialize_vars_and_check_hibernation() >> extract_rawg_api_data() >> load_extracted_data_to_bq() >> post_load_cleanup()