drop table in rest catalog

Author: sfc-gh-okalaci

pg_lake_table/src/ddl/create_table.c

@@ -76,6 +76,7 @@
 #include "pg_lake/planner/dbt.h"
 #include "pg_lake/query/execute.h"
 #include "pg_lake/object_store_catalog/object_store_catalog.h"
+#include "pg_lake/transaction/track_iceberg_metadata_changes.h"
 #include "pg_lake/rest_catalog/rest_catalog.h"
 
 
@@ -965,6 +966,16 @@ ProcessCreateIcebergTableFromForeignTableStmt(ProcessUtilityParams * params)
 		 * is successful in post-commit.
 		 */
 		StartStageRestCatalogIcebergTableCreate(relationId);
+
+		/*
+		 * Record the create table operation in the rest catalog. Note that
+		 * this is not the final registration of the table in the tx, we'll
+		 * update this record in
+		 * FinalizeStagingCreateRestCatalogIcebergTableCreate. We prefer to
+		 * record it here such if table is dropped before commit, we can track
+		 * the creation of the table properly.
+		 */
+		RecordRestCatalogRequestInTx(relationId, REST_CATALOG_CREATE_TABLE, "");
 	}
 
 

pg_lake_table/src/ddl/ddl_changes.c

@@ -128,17 +128,6 @@ ApplyDDLCatalogChanges(Oid relationId, List *ddlOperations,
 		}
 		else if (ddlOperation->type == DDL_TABLE_DROP)
 		{
-			IcebergCatalogType catalogType = GetIcebergCatalogType(relationId);
-
-			if (catalogType == REST_CATALOG_READ_WRITE)
-			{
-				/*
-				 * TODO: Handle dropping of writable rest catalog iceberg
-				 * tables here.
-				 */
-				return;
-			}
-
 			/*
 			 * This is not an expected case, either user manually messed with
 			 * the catalog or we have a bug. Still, we should not fail the

pg_lake_table/src/ddl/drop_table.c

@@ -52,6 +52,7 @@
 #include "pg_lake/iceberg/operations/find_referenced_files.h"
 #include "pg_lake/util/spi_helpers.h"
 #include "pg_lake/object_store_catalog/object_store_catalog.h"
+#include "pg_lake/transaction/track_iceberg_metadata_changes.h"
 #include "pg_lake/rest_catalog/rest_catalog.h"
 #include "pg_lake/parsetree/options.h"
 #include "utils/lsyscache.h"
@@ -293,6 +294,11 @@ DropTableAccessHook(ObjectAccessType access, Oid classId, Oid objectId,
 			ApplyDDLChanges(objectId, list_make1(ddlOperation));
 
 			TriggerCatalogExportIfObjectStoreTable(objectId);
+
+			IcebergCatalogType catalogType = GetIcebergCatalogType(objectId);
+
+			if (catalogType == REST_CATALOG_READ_WRITE)
+				RecordRestCatalogRequestInTx(objectId, REST_CATALOG_DROP_TABLE, NULL);
 		}
 	}
 	else if (get_rel_type_id(objectId) != InvalidOid && subId != 0)
@@ -455,7 +461,10 @@ static bool
 MarkAllReferencedFilesForDeletion(Oid relationId)
 {
 	TimestampTz orphanedAt = GetCurrentTransactionStartTimestamp();
-	char	   *metadataLocation = GetIcebergCatalogMetadataLocation(relationId, true);
+	IcebergCatalogType catalogType = GetIcebergCatalogType(relationId);
+	char	   *metadataLocation =
+		(catalogType == REST_CATALOG_READ_WRITE) ? GetMetadataLocationForRestCatalogForIcebergTable(relationId) :
+		GetIcebergCatalogMetadataLocation(relationId, true);
 	MemoryContext savedContext = CurrentMemoryContext;
 	List	   *allFiles = NIL;
 	volatile bool success = true;

pg_lake_table/src/transaction/track_iceberg_metadata_changes.c

@@ -210,15 +210,16 @@ PostAllRestCatalogRequests(void)
 		RestCatalogRequest *createTableRequest = requestPerTable->createTableRequest;
 		RestCatalogRequest *dropTableRequest = requestPerTable->dropTableRequest;
 
-		if (createTableRequest != NULL || dropTableRequest != NULL)
+		if (createTableRequest != NULL && dropTableRequest != NULL)
 		{
 			/*
-			 * We can only have one create or one drop table request per
-			 * table. If a table is created and dropped in the same
-			 * transaction, we skip both requests.
+			 * table is created and dropped in the same transaction, skip both
+			 * requests, essentially a no-op.
 			 */
-			Assert(createTableRequest == NULL || dropTableRequest == NULL);
-
+			continue;
+		}
+		else if (createTableRequest != NULL || dropTableRequest != NULL)
+		{
 			const char *url =
 				psprintf(REST_CATALOG_TABLE,
 						 RestCatalogHost,
@@ -280,7 +281,17 @@ PostAllRestCatalogRequests(void)
 	{
 		/* TODO: can we ever have multiple catalogs? */
 		catalogName = requestPerTable->catalogName;
-		if (requestPerTable->tableModifyRequests == NIL)
+
+		if (requestPerTable->createTableRequest != NULL &&
+			requestPerTable->dropTableRequest != NULL)
+		{
+			/*
+			 * table is created and dropped in the same transaction, nothing
+			 * post to do for this table to the REST catalog.
+			 */
+			continue;
+		}
+		else if (requestPerTable->tableModifyRequests == NIL)
 		{
 			/*
 			 * no modifications to send for this table
@@ -567,16 +578,7 @@ ApplyTrackedIcebergMetadataChanges(void)
 
 		/* relation is dropped */
 		if (!RelationExistsInTheIcebergCatalog(relationId))
-		{
-			/*
-			 * if created and dropped in the same tx, treat as no-op and skip
-			 * all
-			 */
-			if (!opTracker->relationCreated)
-				RecordRestCatalogRequestInTx(relationId, REST_CATALOG_DROP_TABLE, NULL);
-
 			continue;
-		}
 
 		List	   *allTransforms = AllPartitionTransformList(relationId);