Skip to content

Commit

Permalink
Revert "[FLINK-36652][Formats/ORC] Upgrade Apache ORC Version to 1.9.4 (
Browse files Browse the repository at this point in the history 
apache#25711)"

This reverts commit 363a69d.
  • Loading branch information
@MartijnVisser
MartijnVisser committed Jan 21, 2025
1 parent e000678 commit 9ba5afc
Show file tree
Hide file tree
Showing 11 changed files with 246 additions and 734 deletions.
70 changes: 0 additions & 70 deletions flink-formats/flink-orc-nohive/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,76 +87,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protoc.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>${storage-api.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down
2 changes: 1 addition & 1 deletion ...orc-nohive/src/main/java/org/apache/flink/orc/nohive/writer/NoHivePhysicalWriterImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
import org.apache.flink.core.fs.FSDataOutputStream;
import org.apache.flink.orc.writer.PhysicalWriterImpl;

import com.google.protobuf25.CodedOutputStream;
import org.apache.orc.OrcFile;
import org.apache.orc.OrcProto;
import org.apache.orc.protobuf.CodedOutputStream;

import java.io.IOException;

Expand Down
4 changes: 2 additions & 2 deletions ...nohive/src/test/java/org/apache/flink/orc/nohive/OrcColumnarRowSplitReaderNoHiveTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Map;
import java.util.stream.IntStream;

/** Test for {@link OrcColumnarRowSplitReader}. */
class OrcColumnarRowSplitReaderNoHiveTest extends OrcColumnarRowSplitReaderTest {
Expand Down Expand Up @@ -99,13 +100,12 @@ protected void prepareReadFileWithTypes(String file, int rowSize) throws IOExcep
protected OrcColumnarRowSplitReader createReader(
int[] selectedFields,
DataType[] fullTypes,
String[] fullNames,
Map<String, Object> partitionSpec,
FileInputSplit split)
throws IOException {
return OrcNoHiveSplitReaderUtil.genPartColumnarRowReader(
new Configuration(),
fullNames,
IntStream.range(0, fullTypes.length).mapToObj(i -> "f" + i).toArray(String[]::new),
fullTypes,
partitionSpec,
selectedFields,
Expand Down
36 changes: 1 addition & 35 deletions flink-formats/flink-orc/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,7 @@ under the License.
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand All @@ -129,10 +125,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -149,32 +141,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protoc.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>${storage-api.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down
23 changes: 3 additions & 20 deletions flink-formats/flink-orc/src/main/java/org/apache/flink/orc/writer/OrcBulkWriterFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.orc.OrcFile;
import org.apache.orc.impl.WriterImpl;
import org.apache.orc.impl.writer.WriterEncryptionVariant;

import java.io.IOException;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
Expand Down Expand Up @@ -98,29 +96,14 @@ public OrcBulkWriterFactory(
@Override
public BulkWriter<T> create(FSDataOutputStream out) throws IOException {
OrcFile.WriterOptions opts = getWriterOptions();
PhysicalWriterImpl physicalWriter = new PhysicalWriterImpl(out, opts);
opts.physicalWriter(physicalWriter);
opts.physicalWriter(new PhysicalWriterImpl(out, opts));

// The path of the Writer is not used to indicate the destination file
// in this case since we have used a dedicated physical writer to write
// to the give output stream directly. However, the path would be used as
// the key of writer in the ORC memory manager, thus we need to make it unique.
Path unusedPath = new Path(UUID.randomUUID().toString());
WriterImpl writer = new WriterImpl(null, unusedPath, opts);

// Obtaining encryption variant from Writer, and setting encryption variant for
// physicalWriter.
try {
Field encryptionFiled = WriterImpl.class.getDeclaredField("encryption");
encryptionFiled.setAccessible(true);
WriterEncryptionVariant[] encryption =
(WriterEncryptionVariant[]) encryptionFiled.get(writer);
physicalWriter.setEncryptionVariant(encryption);
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new RuntimeException(
"Can not access to the encryption field in Class org.apache.orc.impl.WriterImpl",
e);
}
return new OrcBulkWriter<>(vectorizer, writer);
return new OrcBulkWriter<>(vectorizer, new WriterImpl(null, unusedPath, opts));
}

@VisibleForTesting
Expand Down
Loading

0 comments on commit 9ba5afc

Please sign in to comment.