From 1a7666c1747ca128039f4f9403d50935df46fa9d Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 6 Nov 2024 10:04:51 -0600 Subject: [PATCH 1/9] Update changelog using fresh changelog file from master update --- .github/workflows/update_source_data_schema_changelog.yml | 4 ++++ scripts/update_source_data_schema_changelog.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_source_data_schema_changelog.yml b/.github/workflows/update_source_data_schema_changelog.yml index 209f6396..0e9188aa 100644 --- a/.github/workflows/update_source_data_schema_changelog.yml +++ b/.github/workflows/update_source_data_schema_changelog.yml @@ -27,12 +27,16 @@ jobs: set -x cd $GITHUB_WORKSPACE OLD_SCHEMAS_DIR=original_schemas + OLD_CHANGELOG_FILEPATH=original_changelog.md git clone --branch master https://github.com/stellar/stellar-etl-airflow.git repo_master_copy mkdir OLD_SCHEMAS_DIR cp -r repo_master_copy/schemas/ $OLD_SCHEMAS_DIR/ + cp repo_master_copy/changelog/source_data.md $OLD_CHANGELOG_FILEPATH export OLD_SCHEMAS_DIR + export OLD_CHANGELOG_FILEPATH python3 scripts/update_source_data_schema_changelog.py rm -rf $OLD_SCHEMAS_DIR + rm $OLD_CHANGELOG_FILEPATH rm -rf repo_master_copy - name: Commit and Push Changes diff --git a/scripts/update_source_data_schema_changelog.py b/scripts/update_source_data_schema_changelog.py index 805cf77e..5368a994 100644 --- a/scripts/update_source_data_schema_changelog.py +++ b/scripts/update_source_data_schema_changelog.py @@ -4,10 +4,10 @@ from datetime import datetime OLD_SCHEMAS_DIR = os.environ.get("OLD_SCHEMAS_DIR") +OLD_CHANGELOG_FILEPATH = os.environ.get("OLD_CHANGELOG_FILEPATH") NEW_SCHEMAS_DIR = "schemas" CHANGELOG_FILEPATH = "changelog/source_data.md" - def read_json_file(filepath: str) -> {}: with open(filepath, "r") as rfp: try: @@ -173,7 +173,7 @@ def generate_changelog(schemas_added=[], schemas_deleted=[], schemas_changes={}) def main(): - existing_changelog = read_file(filepath=CHANGELOG_FILEPATH) + existing_changelog = read_file(filepath=OLD_CHANGELOG_FILEPATH) old_schema_filepaths = list_files_in_dir(directory=OLD_SCHEMAS_DIR) new_schema_filepaths = list_files_in_dir(directory=NEW_SCHEMAS_DIR) From cf5563f96050e31db60901188236b20f98d31ef1 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 6 Nov 2024 10:09:50 -0600 Subject: [PATCH 2/9] update schema --- schemas/ttl_schema.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/schemas/ttl_schema.json b/schemas/ttl_schema.json index f129bb90..a3e4937b 100644 --- a/schemas/ttl_schema.json +++ b/schemas/ttl_schema.json @@ -9,11 +9,6 @@ "name": "live_until_ledger_seq", "type": "INTEGER" }, - { - "mode": "NULLABLE", - "name": "last_modified_ledger", - "type": "INTEGER" - }, { "mode": "NULLABLE", "name": "ledger_entry_change", From 5b7875e6187944dbdc4a41eebf246159ca6a6ccb Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 6 Nov 2024 16:10:05 +0000 Subject: [PATCH 3/9] Update changelog for Source data --- changelog/source_data.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changelog/source_data.md b/changelog/source_data.md index e69de29b..206e2439 100644 --- a/changelog/source_data.md +++ b/changelog/source_data.md @@ -0,0 +1,9 @@ + +## 2024-11-06: + +## Schema Changes: +| Table Name | Operation | Columns | +|---------------------------------|---------------|----------------------------------------------------| +| - ttl | column_removed | last_modified_ledger | + + From 4ac94fadbf496e4e23de49d986214558f03f7a80 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 6 Nov 2024 10:11:06 -0600 Subject: [PATCH 4/9] update another schema --- schemas/history_assets_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/history_assets_schema.json b/schemas/history_assets_schema.json index c6576a96..7962e418 100644 --- a/schemas/history_assets_schema.json +++ b/schemas/history_assets_schema.json @@ -11,7 +11,7 @@ }, { "mode": "NULLABLE", - "name": "asset_issuer", + "name": "asset_issuer_1", "type": "STRING" }, { From 8403a17c3ee7b13009219d605a91fa5e542f385d Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 6 Nov 2024 16:11:28 +0000 Subject: [PATCH 5/9] Update changelog for Source data --- changelog/source_data.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog/source_data.md b/changelog/source_data.md index 206e2439..190d26a7 100644 --- a/changelog/source_data.md +++ b/changelog/source_data.md @@ -4,6 +4,8 @@ ## Schema Changes: | Table Name | Operation | Columns | |---------------------------------|---------------|----------------------------------------------------| +| - history_assets | column_added | asset_issuer_1 | +| - history_assets | column_removed | asset_issuer | | - ttl | column_removed | last_modified_ledger | From 179a793a1ac8a7c0294089ceaf4b583120e410dd Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 6 Nov 2024 10:13:53 -0600 Subject: [PATCH 6/9] Revert "update another schema" This reverts commit 4ac94fadbf496e4e23de49d986214558f03f7a80. --- schemas/history_assets_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/history_assets_schema.json b/schemas/history_assets_schema.json index 7962e418..c6576a96 100644 --- a/schemas/history_assets_schema.json +++ b/schemas/history_assets_schema.json @@ -11,7 +11,7 @@ }, { "mode": "NULLABLE", - "name": "asset_issuer_1", + "name": "asset_issuer", "type": "STRING" }, { From 74e5b91dbb104a3dfd7c61f4c6aa3cd2c6056c34 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 6 Nov 2024 10:14:08 -0600 Subject: [PATCH 7/9] Revert "update schema" This reverts commit cf5563f96050e31db60901188236b20f98d31ef1. --- schemas/ttl_schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schemas/ttl_schema.json b/schemas/ttl_schema.json index a3e4937b..f129bb90 100644 --- a/schemas/ttl_schema.json +++ b/schemas/ttl_schema.json @@ -9,6 +9,11 @@ "name": "live_until_ledger_seq", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "ledger_entry_change", From 13f2093cad0a476dd31f794891602667b9bb90a4 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 6 Nov 2024 10:17:32 -0600 Subject: [PATCH 8/9] Overwrite changelog --- scripts/update_source_data_schema_changelog.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/update_source_data_schema_changelog.py b/scripts/update_source_data_schema_changelog.py index 5368a994..91da596c 100644 --- a/scripts/update_source_data_schema_changelog.py +++ b/scripts/update_source_data_schema_changelog.py @@ -8,6 +8,7 @@ NEW_SCHEMAS_DIR = "schemas" CHANGELOG_FILEPATH = "changelog/source_data.md" + def read_json_file(filepath: str) -> {}: with open(filepath, "r") as rfp: try: @@ -191,6 +192,9 @@ def main(): filepath=CHANGELOG_FILEPATH, mode="w", content=new_changelog + "\n\n" ) write_file(filepath=CHANGELOG_FILEPATH, mode="a", content=existing_changelog) + else: + # Rewrite the changelog since intermediate commits may have overwritten changelog + write_file(filepath=CHANGELOG_FILEPATH, mode="w", content=existing_changelog) if __name__ == "__main__": From 02d7b143cf87962c0d3143e8c55ef7cd3ac58055 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 6 Nov 2024 16:18:02 +0000 Subject: [PATCH 9/9] Update changelog for Source data --- changelog/source_data.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/changelog/source_data.md b/changelog/source_data.md index 190d26a7..e69de29b 100644 --- a/changelog/source_data.md +++ b/changelog/source_data.md @@ -1,11 +0,0 @@ - -## 2024-11-06: - -## Schema Changes: -| Table Name | Operation | Columns | -|---------------------------------|---------------|----------------------------------------------------| -| - history_assets | column_added | asset_issuer_1 | -| - history_assets | column_removed | asset_issuer | -| - ttl | column_removed | last_modified_ledger | - -