From 9fccde3e9121a105bc8a33a6fe5e1b6e1f646536 Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Thu, 6 Feb 2025 17:59:00 +0800 Subject: [PATCH 1/6] Fix hive catalog create partitions using table location. --- .../src/main/java/org/apache/paimon/hive/HiveCatalog.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 0df07014bd26..301bfaabfe07 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -350,7 +350,6 @@ public void createPartitions(Identifier identifier, List> pa Identifier tableIdentifier = Identifier.create(identifier.getDatabaseName(), identifier.getTableName()); Table hmsTable = getHmsTable(tableIdentifier); - Path location = getTableLocation(tableIdentifier, hmsTable); TableSchema schema = loadTableSchema(tableIdentifier, hmsTable); if (!metastorePartitioned(schema)) { @@ -358,13 +357,14 @@ public void createPartitions(Identifier identifier, List> pa } int currentTime = (int) (System.currentTimeMillis() / 1000); + String tableLocation = getTableLocation(tableIdentifier, hmsTable).toUri().toString(); StorageDescriptor sd = hmsTable.getSd(); String dataFilePath = hmsTable.getParameters().containsKey(DATA_FILE_PATH_DIRECTORY.key()) - ? sd.getLocation() + ? tableLocation + "/" + hmsTable.getParameters().get(DATA_FILE_PATH_DIRECTORY.key()) - : sd.getLocation(); + : tableLocation; List hivePartitions = new ArrayList<>(); for (Map partitionSpec : partitions) { Partition hivePartition = new Partition(); From 523d33f419d070a886701eca89e90cf798a1ae2c Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Thu, 6 Feb 2025 21:27:58 +0800 Subject: [PATCH 2/6] fix ut error --- .../org/apache/paimon/hive/HiveCatalog.java | 37 +++++++++++++------ .../apache/paimon/hive/LocationHelper.java | 5 +++ .../paimon/hive/StorageLocationHelper.java | 11 ++++++ .../hive/TBPropertiesLocationHelper.java | 17 +++++++++ 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 301bfaabfe07..4f767b32a5ca 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -357,29 +357,20 @@ public void createPartitions(Identifier identifier, List> pa } int currentTime = (int) (System.currentTimeMillis() / 1000); - String tableLocation = getTableLocation(tableIdentifier, hmsTable).toUri().toString(); + String dataFilePath = getDataFilePath(tableIdentifier, hmsTable); StorageDescriptor sd = hmsTable.getSd(); - String dataFilePath = - hmsTable.getParameters().containsKey(DATA_FILE_PATH_DIRECTORY.key()) - ? tableLocation - + "/" - + hmsTable.getParameters().get(DATA_FILE_PATH_DIRECTORY.key()) - : tableLocation; List hivePartitions = new ArrayList<>(); for (Map partitionSpec : partitions) { Partition hivePartition = new Partition(); StorageDescriptor newSd = new StorageDescriptor(sd); - newSd.setLocation( - dataFilePath - + "/" - + PartitionPathUtils.generatePartitionPath( - new LinkedHashMap<>(partitionSpec))); hivePartition.setDbName(identifier.getDatabaseName()); hivePartition.setTableName(identifier.getTableName()); hivePartition.setValues(new ArrayList<>(partitionSpec.values())); hivePartition.setSd(newSd); hivePartition.setCreateTime(currentTime); hivePartition.setLastAccessTime(currentTime); + String partitionLocation = getPartitionLocation(dataFilePath, partitionSpec); + locationHelper.specifyPartitionLocation(hivePartition, partitionLocation); hivePartitions.add(hivePartition); } try { @@ -403,6 +394,15 @@ public void dropPartitions(Identifier identifier, List> part for (Map part : metaPartitions) { List partitionValues = new ArrayList<>(part.values()); try { + Partition partition = + clients.run( + client -> + client.getPartition( + identifier.getDatabaseName(), + identifier.getTableName(), + partitionValues)); + String partitionLocation = locationHelper.getPartitionLocation(partition); + locationHelper.dropPathIfRequired(new Path(partitionLocation), fileIO); clients.execute( client -> client.dropPartition( @@ -422,6 +422,19 @@ public void dropPartitions(Identifier identifier, List> part } } + private String getDataFilePath(Identifier tableIdentifier, Table hmsTable) { + String tableLocation = getTableLocation(tableIdentifier, hmsTable).toUri().toString(); + return hmsTable.getParameters().containsKey(DATA_FILE_PATH_DIRECTORY.key()) + ? tableLocation + "/" + hmsTable.getParameters().get(DATA_FILE_PATH_DIRECTORY.key()) + : tableLocation; + } + + private String getPartitionLocation(String dataFilePath, Map partitionSpec) { + return dataFilePath + + "/" + + PartitionPathUtils.generatePartitionPath(new LinkedHashMap<>(partitionSpec)); + } + @Override public void alterPartitions( Identifier identifier, List partitions) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java index 71efae2cce65..a582d97f4735 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java @@ -22,6 +22,7 @@ import org.apache.paimon.fs.Path; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import java.io.IOException; @@ -40,4 +41,8 @@ public interface LocationHelper { void specifyDatabaseLocation(Path path, Database database); String getDatabaseLocation(Database database); + + void specifyPartitionLocation(Partition partition, String location); + + String getPartitionLocation(Partition partition); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java index 72f15af1b797..ea8f11ce72b7 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java @@ -22,6 +22,7 @@ import org.apache.paimon.fs.Path; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; /** Helper for Setting Location in Hive Table Storage. */ @@ -58,4 +59,14 @@ public void specifyDatabaseLocation(Path path, Database database) { public String getDatabaseLocation(Database database) { return database.getLocationUri(); } + + @Override + public void specifyPartitionLocation(Partition partition, String location) { + partition.getSd().setLocation(location); + } + + @Override + public String getPartitionLocation(Partition partition) { + return partition.getSd().getLocation(); + } } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java index c4fe2a94b1de..401e65ef6d8e 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java @@ -22,6 +22,7 @@ import org.apache.paimon.fs.Path; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import java.io.IOException; @@ -82,4 +83,20 @@ public String getDatabaseLocation(Database database) { return database.getLocationUri(); } + + @Override + public void specifyPartitionLocation(Partition partition, String location) { + partition.putToParameters(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY, location); + } + + @Override + public String getPartitionLocation(Partition partition) { + String location = + partition.getParameters().get(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY); + if (location != null) { + return location; + } + + return partition.getSd().getLocation(); + } } From 7ba1a3e0ea2889033ef85f507ef0b0465056cc8e Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Fri, 7 Feb 2025 08:53:14 +0800 Subject: [PATCH 3/6] fix ut error --- .../org/apache/paimon/hive/HiveCatalog.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 4f767b32a5ca..43e0ce6dc55c 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -357,8 +357,8 @@ public void createPartitions(Identifier identifier, List> pa } int currentTime = (int) (System.currentTimeMillis() / 1000); - String dataFilePath = getDataFilePath(tableIdentifier, hmsTable); StorageDescriptor sd = hmsTable.getSd(); + String dataFilePath = getDataFilePath(tableIdentifier, hmsTable); List hivePartitions = new ArrayList<>(); for (Map partitionSpec : partitions) { Partition hivePartition = new Partition(); @@ -391,18 +391,12 @@ public void dropPartitions(Identifier identifier, List> part tagToPart ? partitions : removePartitionsExistsInOtherBranches(identifier, partitions); + Table hmsTable = getHmsTable(identifier); + boolean externalTable = isExternalTable(hmsTable); + String dataFilePath = getDataFilePath(identifier, hmsTable); for (Map part : metaPartitions) { List partitionValues = new ArrayList<>(part.values()); try { - Partition partition = - clients.run( - client -> - client.getPartition( - identifier.getDatabaseName(), - identifier.getTableName(), - partitionValues)); - String partitionLocation = locationHelper.getPartitionLocation(partition); - locationHelper.dropPathIfRequired(new Path(partitionLocation), fileIO); clients.execute( client -> client.dropPartition( @@ -410,6 +404,11 @@ public void dropPartitions(Identifier identifier, List> part identifier.getTableName(), partitionValues, false)); + + if (!externalTable) { + String partitionLocation = getPartitionLocation(dataFilePath, part); + locationHelper.dropPathIfRequired(new Path(partitionLocation), fileIO); + } } catch (NoSuchObjectException e) { // do nothing if the partition not exists } catch (Exception e) { From 8f55b743d2eb281c57ecb036ffe8b1c5d493a193 Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Mon, 10 Feb 2025 20:45:51 +0800 Subject: [PATCH 4/6] add test --- .../org/apache/paimon/hive/HiveCatalog.java | 20 +++++++++++++++---- .../paimon/hive/HiveCatalogITCaseBase.java | 11 ++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 43e0ce6dc55c..22f5565fb1a2 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -406,8 +406,18 @@ public void dropPartitions(Identifier identifier, List> part false)); if (!externalTable) { - String partitionLocation = getPartitionLocation(dataFilePath, part); - locationHelper.dropPathIfRequired(new Path(partitionLocation), fileIO); + Path partitionLocation = new Path(getPartitionLocation(dataFilePath, part)); + try { + if (fileIO.exists(partitionLocation)) { + fileIO.deleteDirectoryQuietly(partitionLocation); + } + } catch (Exception ee) { + LOG.error( + "Delete directory[{}] fail for table {} partition.", + partitionLocation, + identifier, + ee); + } } } catch (NoSuchObjectException e) { // do nothing if the partition not exists @@ -424,13 +434,15 @@ public void dropPartitions(Identifier identifier, List> part private String getDataFilePath(Identifier tableIdentifier, Table hmsTable) { String tableLocation = getTableLocation(tableIdentifier, hmsTable).toUri().toString(); return hmsTable.getParameters().containsKey(DATA_FILE_PATH_DIRECTORY.key()) - ? tableLocation + "/" + hmsTable.getParameters().get(DATA_FILE_PATH_DIRECTORY.key()) + ? tableLocation + + Path.SEPARATOR + + hmsTable.getParameters().get(DATA_FILE_PATH_DIRECTORY.key()) : tableLocation; } private String getPartitionLocation(String dataFilePath, Map partitionSpec) { return dataFilePath - + "/" + + Path.SEPARATOR + PartitionPathUtils.generatePartitionPath(new LinkedHashMap<>(partitionSpec)); } diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index a9e6c01e39ed..eba3bfecdac9 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -679,27 +679,35 @@ private void testDropPartitionFromBranchImpl() throws Exception { assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=1", "pt=2", "pt=3", "pt=4"); + Path tablePath = new Path(path, "test_db.db/t"); + tEnv.executeSql("ALTER TABLE `t$branch_test` DROP PARTITION (pt = 1)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=1", "pt=2", "pt=3", "pt=4"); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=1"))).isTrue(); tEnv.executeSql("ALTER TABLE `t$branch_test` DROP PARTITION (pt = 3)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=1", "pt=2", "pt=4"); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=3"))).isFalse(); tEnv.executeSql("ALTER TABLE t DROP PARTITION (pt = 1)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=2", "pt=4"); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=1"))).isFalse(); tEnv.executeSql("ALTER TABLE t DROP PARTITION (pt = 4)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=2", "pt=4"); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=4"))).isTrue(); tEnv.executeSql("ALTER TABLE `t$branch_test` DROP PARTITION (pt = 4)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")).containsExactlyInAnyOrder("pt=2"); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=4"))).isFalse(); tEnv.executeSql("ALTER TABLE t DROP PARTITION (pt = 2)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")).isEmpty(); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=2"))).isFalse(); } @Test @@ -1281,6 +1289,9 @@ private void prepareTestAddPartitionsToMetastore() throws Exception { "ptb=2b/pta=2", "ptb=3a/pta=3", "ptb=3b/pta=3"); + + Path tablePath = new Path(path, "test_db.db/t"); + assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "ptb=1a/pta=1"))).isTrue(); } @Test From b5130a6d76daedfeaa74d0791ca40c57d22cfb24 Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Tue, 11 Feb 2025 10:33:25 +0800 Subject: [PATCH 5/6] fix ut --- .../paimon/hive/HiveCatalogITCaseBase.java | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index eba3bfecdac9..637e4bf2ccd6 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -18,26 +18,23 @@ package org.apache.paimon.hive; -import org.apache.paimon.catalog.Catalog; -import org.apache.paimon.catalog.DelegateCatalog; -import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.catalog.RenamingSnapshotCommit; +import org.apache.paimon.catalog.*; import org.apache.paimon.flink.FlinkCatalog; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.hive.annotation.Minio; import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner; import org.apache.paimon.operation.Lock; +import org.apache.paimon.options.Options; import org.apache.paimon.privilege.NoPrivilegeException; import org.apache.paimon.s3.MinioTestContainer; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; -import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.TimeUtils; import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.annotations.HiveSQL; -import org.apache.flink.core.fs.FSDataInputStream; -import org.apache.flink.core.fs.Path; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; @@ -94,6 +91,7 @@ public abstract class HiveCatalogITCaseBase { protected TableEnvironment tEnv; protected TableEnvironment sEnv; private boolean locationInProperties; + private FileIO fileIO; @HiveSQL(files = {}) protected static HiveShell hiveShell; @@ -134,6 +132,10 @@ private void registerHiveCatalog(String catalogName, Map catalog catalogProperties.putAll(minioTestContainer.getS3ConfigOptions()); } + Options catalogOptions = new Options(catalogProperties); + CatalogContext catalogContext = CatalogContext.create(catalogOptions); + fileIO = FileIO.get(new Path(path), catalogContext); + tEnv = TableEnvironmentImpl.create(EnvironmentSettings.newInstance().inBatchMode().build()); sEnv = TableEnvironmentImpl.create( @@ -255,7 +257,7 @@ public void testDatabaseOperations() throws Exception { .await(); tEnv.executeSql("INSERT INTO t VALUES (1, 'Hi'), (2, 'Hello')").await(); Path tablePath = new Path(path, "test_db2.db/t"); - assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue(); + assertThat(fileIO.exists(tablePath)).isTrue(); assertThatThrownBy(() -> tEnv.executeSql("DROP DATABASE test_db2").await()) .hasRootCauseInstanceOf(ValidationException.class) .hasRootCauseMessage("Cannot drop a database which is currently in use."); @@ -267,7 +269,7 @@ public void testDatabaseOperations() throws Exception { tEnv.executeSql("DROP DATABASE test_db2 CASCADE").await(); assertThat(collect("SHOW DATABASES")) .isEqualTo(Arrays.asList(Row.of("default"), Row.of("test_db"))); - assertThat(tablePath.getFileSystem().exists(tablePath)).isFalse(); + assertThat(fileIO.exists(tablePath)).isFalse(); } @Test @@ -295,11 +297,11 @@ public void testTableOperations() throws Exception { // drop table tEnv.executeSql("INSERT INTO s VALUES (1, 'Hi'), (2, 'Hello')").await(); Path tablePath = new Path(path, "test_db.db/s"); - assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue(); + assertThat(fileIO.exists(tablePath)).isTrue(); tEnv.executeSql("DROP TABLE s").await(); assertThat(collect("SHOW TABLES")) .containsExactlyInAnyOrder(Row.of("t"), Row.of("hive_table")); - assertThat(tablePath.getFileSystem().exists(tablePath)).isFalse(); + assertThat(fileIO.exists(tablePath)).isFalse(); tEnv.executeSql("DROP TABLE IF EXISTS s").await(); assertThatThrownBy(() -> tEnv.executeSql("DROP TABLE s").await()) .isInstanceOf(ValidationException.class) @@ -359,7 +361,7 @@ public void testCreateExternalTable() throws Exception { .isTrue(); tEnv.executeSql("DROP TABLE t").await(); Path tablePath = new Path(path, "test_db.db/t"); - assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue(); + assertThat(fileIO.exists(tablePath)).isTrue(); } @Test @@ -592,7 +594,7 @@ public void testCreateInsensitiveTable() throws Exception { .isTrue(); tEnv.executeSql("DROP TABLE t").await(); Path tablePath = new Path(path, "test_db.db/t"); - assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue(); + assertThat(fileIO.exists(tablePath)).isTrue(); } @Test @@ -684,30 +686,30 @@ private void testDropPartitionFromBranchImpl() throws Exception { tEnv.executeSql("ALTER TABLE `t$branch_test` DROP PARTITION (pt = 1)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=1", "pt=2", "pt=3", "pt=4"); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=1"))).isTrue(); + assertThat(fileIO.exists(new Path(tablePath, "pt=1"))).isTrue(); tEnv.executeSql("ALTER TABLE `t$branch_test` DROP PARTITION (pt = 3)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=1", "pt=2", "pt=4"); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=3"))).isFalse(); + assertThat(fileIO.exists(new Path(tablePath, "pt=3"))).isFalse(); tEnv.executeSql("ALTER TABLE t DROP PARTITION (pt = 1)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=2", "pt=4"); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=1"))).isFalse(); + assertThat(fileIO.exists(new Path(tablePath, "pt=1"))).isFalse(); tEnv.executeSql("ALTER TABLE t DROP PARTITION (pt = 4)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")) .containsExactlyInAnyOrder("pt=2", "pt=4"); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=4"))).isTrue(); + assertThat(fileIO.exists(new Path(tablePath, "pt=4"))).isTrue(); tEnv.executeSql("ALTER TABLE `t$branch_test` DROP PARTITION (pt = 4)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")).containsExactlyInAnyOrder("pt=2"); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=4"))).isFalse(); + assertThat(fileIO.exists(new Path(tablePath, "pt=4"))).isFalse(); tEnv.executeSql("ALTER TABLE t DROP PARTITION (pt = 2)"); assertThat(hiveShell.executeQuery("SHOW PARTITIONS t")).isEmpty(); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "pt=2"))).isFalse(); + assertThat(fileIO.exists(new Path(tablePath, "pt=2"))).isFalse(); } @Test @@ -1291,7 +1293,7 @@ private void prepareTestAddPartitionsToMetastore() throws Exception { "ptb=3b/pta=3"); Path tablePath = new Path(path, "test_db.db/t"); - assertThat(tablePath.getFileSystem().exists(new Path(tablePath, "ptb=1a/pta=1"))).isTrue(); + assertThat(fileIO.exists(new Path(tablePath, "ptb=1a/pta=1"))).isTrue(); } @Test @@ -1563,10 +1565,7 @@ public void testMarkDone() throws Exception { // check partition.mark-done-action=success-file Path successFile = new Path(path, "test_db.db/mark_done_t2/dt=20240501/_SUCCESS"); - String successText; - try (FSDataInputStream in = successFile.getFileSystem().open(successFile)) { - successText = IOUtils.readUTF8Fully(in); - } + String successText = fileIO.readFileUtf8(successFile); assertThat(successText).contains("creationTime").contains("modificationTime"); From 475f71a82738f3fff173f4a896f936c6deb09487 Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Tue, 11 Feb 2025 10:39:18 +0800 Subject: [PATCH 6/6] fix code style --- .../java/org/apache/paimon/hive/HiveCatalogITCaseBase.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index 637e4bf2ccd6..321f6bdeb001 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -18,7 +18,11 @@ package org.apache.paimon.hive; -import org.apache.paimon.catalog.*; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.RenamingSnapshotCommit; import org.apache.paimon.flink.FlinkCatalog; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path;