Skip to content

bugfix: The results of querying the hudi multi partition RT table are wrong #444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
felixzh2020 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

bugfix: The results of querying the hudi multi partition RT table are wrong #444

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
50131f2
bugfix: The results of querying the hudi multi partition RT table are…
felixzh2020 Nov 2, 2024
8002bed
Merge branch 'openlookeng:master' into master
felixzh2020 Nov 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions presto-hive/src/main/java/io/prestosql/plugin/hive/BackgroundHiveSplitLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,10 @@ private ListenableFuture<?> loadPartition(HivePartitionMetadata partition)
}

Path path = new Path(getPartitionLocation(table, partition.getPartition()));
InputFormat<?, ?> inputFormat = getInputFormat(configuration, schema, false, jobConf);
// For hudi mor rt table, multiple partitions and threads need to avoid modifying a JobConf at the same time,
// which can cause incorrect split partition paths.
JobConf localJobConf = new JobConf(jobConf);
InputFormat<?, ?> inputFormat = getInputFormat(configuration, schema, false, localJobConf);
FileSystem fs = hdfsEnvironment.getFileSystem(hdfsContext, path);
boolean s3SelectPushdownEnabled = shouldEnablePushdownForTable(session, table, path.toString(), partition.getPartition());

Expand All @@ -381,10 +384,10 @@ private ListenableFuture<?> loadPartition(HivePartitionMetadata partition)
// the splits must be generated using the file system for the target path
// get the configuration for the target path -- it may be a different hdfs instance
FileSystem targetFilesystem = hdfsEnvironment.getFileSystem(hdfsContext, targetPath);
jobConf.setInputFormat(TextInputFormat.class);
targetInputFormat.configure(jobConf);
FileInputFormat.setInputPaths(jobConf, targetPath);
InputSplit[] targetSplits = targetInputFormat.getSplits(jobConf, 0);
localJobConf.setInputFormat(TextInputFormat.class);
targetInputFormat.configure(localJobConf);
FileInputFormat.setInputPaths(localJobConf, targetPath);
InputSplit[] targetSplits = targetInputFormat.getSplits(localJobConf, 0);

InternalHiveSplitFactory splitFactory = new InternalHiveSplitFactory(
targetFilesystem,
Expand Down Expand Up @@ -446,8 +449,8 @@ private ListenableFuture<?> loadPartition(HivePartitionMetadata partition)
throw new PrestoException(NOT_SUPPORTED, "Hive transactional tables in an input format with UseFileSplitsFromInputFormat annotation are not supported: " + inputFormat.getClass().getSimpleName());
}

FileInputFormat.setInputPaths(jobConf, path);
InputSplit[] splits = inputFormat.getSplits(jobConf, 0);
FileInputFormat.setInputPaths(localJobConf, path);
InputSplit[] splits = inputFormat.getSplits(localJobConf, 0);

return addSplitsToSource(splits, splitFactory);
}
Expand Down