Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import io.trino.execution.QueryManager;
import io.trino.execution.QueryState;
import io.trino.execution.buffer.PageDeserializer;
import io.trino.execution.buffer.PagesSerdeFactory;
import io.trino.memory.context.SimpleLocalMemoryContext;
import io.trino.operator.DirectExchangeClient;
import io.trino.operator.DirectExchangeClientSupplier;
Expand All @@ -43,10 +42,10 @@
import java.util.concurrent.ExecutionException;

import static io.airlift.concurrent.MoreFutures.whenAnyComplete;
import static io.trino.SystemSessionProperties.getExchangeCompressionCodec;
import static io.trino.SystemSessionProperties.getRetryPolicy;
import static io.trino.execution.QueryState.FAILED;
import static io.trino.execution.QueryState.FINISHING;
import static io.trino.execution.buffer.PagesSerdes.createExchangePagesSerdeFactory;
import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext;
import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -94,7 +93,7 @@ public DispatchQuery execute(SessionContext sessionContext, @Language("SQL") Str
}
});

PageDeserializer pageDeserializer = new PagesSerdeFactory(blockEncodingSerde, getExchangeCompressionCodec(dispatchQuery.getSession())).createDeserializer(Optional.empty());
PageDeserializer pageDeserializer = createExchangePagesSerdeFactory(blockEncodingSerde, dispatchQuery.getSession()).createDeserializer(Optional.empty());
for (QueryState state = queryManager.getQueryState(queryId);
(state != FAILED) &&
!exchangeClient.isFinished() &&
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ public class PagesSerdeFactory
private final CompressionCodec compressionCodec;
private final int blockSizeInBytes;

public PagesSerdeFactory(BlockEncodingSerde blockEncodingSerde, CompressionCodec compressionCodec)
// created via PagesSerdes.create*
PagesSerdeFactory(BlockEncodingSerde blockEncodingSerde, CompressionCodec compressionCodec)
{
this(blockEncodingSerde, compressionCodec, SERIALIZED_PAGE_DEFAULT_BLOCK_SIZE_IN_BYTES);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.execution.buffer;

import io.trino.Session;
import io.trino.spi.block.BlockEncodingSerde;

import static io.trino.SystemSessionProperties.getExchangeCompressionCodec;

public final class PagesSerdes
{
private PagesSerdes() {}

public static PagesSerdeFactory createExchangePagesSerdeFactory(BlockEncodingSerde blockEncodingSerde, Session session)
{
return new PagesSerdeFactory(blockEncodingSerde, getExchangeCompressionCodec(session));
}

public static PagesSerdeFactory createSpillingPagesSerdeFactory(BlockEncodingSerde blockEncodingSerde, CompressionCodec compressionCodec)
{
return new PagesSerdeFactory(blockEncodingSerde, compressionCodec);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import io.trino.execution.QueryState;
import io.trino.execution.StageId;
import io.trino.execution.buffer.PageDeserializer;
import io.trino.execution.buffer.PagesSerdeFactory;
import io.trino.memory.context.SimpleLocalMemoryContext;
import io.trino.operator.DirectExchangeClientSupplier;
import io.trino.server.ExternalUriInfo;
Expand Down Expand Up @@ -86,10 +85,10 @@
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static io.airlift.concurrent.MoreFutures.addTimeout;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static io.trino.SystemSessionProperties.getExchangeCompressionCodec;
import static io.trino.SystemSessionProperties.getRetryPolicy;
import static io.trino.execution.QueryState.FAILED;
import static io.trino.execution.QueryState.FINISHING;
import static io.trino.execution.buffer.PagesSerdes.createExchangePagesSerdeFactory;
import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext;
import static io.trino.server.protocol.ProtocolUtil.createColumn;
import static io.trino.server.protocol.ProtocolUtil.toStatementStats;
Expand Down Expand Up @@ -262,7 +261,7 @@ private Query(
this.resultsProcessorExecutor = resultsProcessorExecutor;
this.timeoutExecutor = timeoutExecutor;
this.supportsParametricDateTime = session.getClientCapabilities().contains(ClientCapabilities.PARAMETRIC_DATETIME.toString());
deserializer = new PagesSerdeFactory(blockEncodingSerde, getExchangeCompressionCodec(session))
deserializer = createExchangePagesSerdeFactory(blockEncodingSerde, session)
.createDeserializer(session.getExchangeEncryptionKey().map(Ciphers::deserializeAesEncryptionKey));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static io.trino.FeaturesConfig.SPILLER_SPILL_PATH;
import static io.trino.cache.SafeCaches.buildNonEvictableCacheWithWeakInvalidateAll;
import static io.trino.execution.buffer.PagesSerdes.createSpillingPagesSerdeFactory;
import static io.trino.spi.StandardErrorCode.OUT_OF_SPILL_SPACE;
import static io.trino.util.Ciphers.createRandomAesEncryptionKey;
import static java.lang.String.format;
Expand Down Expand Up @@ -107,7 +108,7 @@ public FileSingleStreamSpillerFactory(
CompressionCodec compressionCodec,
boolean spillEncryptionEnabled)
{
this.serdeFactory = new PagesSerdeFactory(blockEncodingSerde, compressionCodec);
this.serdeFactory = createSpillingPagesSerdeFactory(blockEncodingSerde, compressionCodec);
this.executor = requireNonNull(executor, "executor is null");
this.spillerStats = requireNonNull(spillerStats, "spillerStats cannot be null");
requireNonNull(spillPaths, "spillPaths is null");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import io.trino.execution.TaskId;
import io.trino.execution.TaskManagerConfig;
import io.trino.execution.buffer.OutputBuffer;
import io.trino.execution.buffer.PagesSerdeFactory;
import io.trino.metadata.MergeHandle;
import io.trino.metadata.Metadata;
import io.trino.metadata.ResolvedFunction;
Expand Down Expand Up @@ -314,7 +313,6 @@
import static io.trino.SystemSessionProperties.getAdaptivePartialAggregationUniqueRowsRatioThreshold;
import static io.trino.SystemSessionProperties.getAggregationOperatorUnspillMemoryLimit;
import static io.trino.SystemSessionProperties.getDynamicRowFilterSelectivityThreshold;
import static io.trino.SystemSessionProperties.getExchangeCompressionCodec;
import static io.trino.SystemSessionProperties.getFilterAndProjectMinOutputPageRowCount;
import static io.trino.SystemSessionProperties.getFilterAndProjectMinOutputPageSize;
import static io.trino.SystemSessionProperties.getPagePartitioningBufferPoolSize;
Expand All @@ -331,6 +329,7 @@
import static io.trino.SystemSessionProperties.isSpillEnabled;
import static io.trino.cache.CacheUtils.uncheckedCacheGet;
import static io.trino.cache.SafeCaches.buildNonEvictableCache;
import static io.trino.execution.buffer.PagesSerdes.createExchangePagesSerdeFactory;
import static io.trino.metadata.GlobalFunctionCatalog.builtinFunctionName;
import static io.trino.operator.DistinctLimitOperator.DistinctLimitOperatorFactory;
import static io.trino.operator.HashArraySizeSupplier.incrementalLoadFactorHashArraySizeSupplier;
Expand Down Expand Up @@ -675,7 +674,7 @@ public LocalExecutionPlan plan(
plan.getId(),
outputTypes,
pagePreprocessor,
new PagesSerdeFactory(plannerContext.getBlockEncodingSerde(), getExchangeCompressionCodec(session))),
createExchangePagesSerdeFactory(plannerContext.getBlockEncodingSerde(), session)),
physicalOperation),
context);

Expand Down Expand Up @@ -940,7 +939,7 @@ private PhysicalOperation createMergeSource(RemoteSourceNode node, LocalExecutio
context.getNextOperatorId(),
node.getId(),
directExchangeClientSupplier,
new PagesSerdeFactory(plannerContext.getBlockEncodingSerde(), getExchangeCompressionCodec(session)),
createExchangePagesSerdeFactory(plannerContext.getBlockEncodingSerde(), session),
orderingCompiler,
types,
outputChannels,
Expand All @@ -960,7 +959,7 @@ private PhysicalOperation createRemoteSource(RemoteSourceNode node, LocalExecuti
context.getNextOperatorId(),
node.getId(),
directExchangeClientSupplier,
new PagesSerdeFactory(plannerContext.getBlockEncodingSerde(), getExchangeCompressionCodec(session)),
createExchangePagesSerdeFactory(plannerContext.getBlockEncodingSerde(), session),
node.getRetryPolicy(),
exchangeManagerRegistry);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import io.trino.execution.buffer.BufferState;
import io.trino.execution.buffer.OutputBuffer;
import io.trino.execution.buffer.OutputBufferStateMachine;
import io.trino.execution.buffer.PagesSerdeFactory;
import io.trino.execution.buffer.PartitionedOutputBuffer;
import io.trino.execution.buffer.PipelinedOutputBuffers;
import io.trino.execution.buffer.PipelinedOutputBuffers.OutputBufferId;
Expand All @@ -50,7 +49,6 @@
import io.trino.operator.output.TaskOutputOperator.TaskOutputOperatorFactory;
import io.trino.spi.Page;
import io.trino.spi.QueryId;
import io.trino.spi.block.TestingBlockEncodingSerde;
import io.trino.spi.connector.ConnectorSplit;
import io.trino.spiller.SpillSpaceTracker;
import io.trino.sql.planner.LocalExecutionPlanner.LocalExecutionPlan;
Expand Down Expand Up @@ -79,9 +77,9 @@
import static io.trino.execution.TaskState.RUNNING;
import static io.trino.execution.TaskTestUtils.TABLE_SCAN_NODE_ID;
import static io.trino.execution.TaskTestUtils.createTestSplitMonitor;
import static io.trino.execution.buffer.CompressionCodec.NONE;
import static io.trino.execution.buffer.PagesSerdeUtil.getSerializedPagePositionCount;
import static io.trino.execution.buffer.PipelinedOutputBuffers.BufferType.PARTITIONED;
import static io.trino.execution.buffer.TestingPagesSerdes.createTestingPagesSerdeFactory;
import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext;
import static io.trino.testing.TestingHandles.TEST_CATALOG_HANDLE;
import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -127,7 +125,7 @@ public void testSimple()
TABLE_SCAN_NODE_ID,
outputBuffer,
Function.identity(),
new PagesSerdeFactory(new TestingBlockEncodingSerde(), NONE));
createTestingPagesSerdeFactory());
LocalExecutionPlan localExecutionPlan = new LocalExecutionPlan(
ImmutableList.of(new DriverFactory(
0,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import io.trino.spi.PageBuilder;
import io.trino.spi.block.BlockBuilder;
import io.trino.spi.block.RowBlockBuilder;
import io.trino.spi.block.TestingBlockEncodingSerde;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.Int128;
import io.trino.spi.type.RowType;
Expand Down Expand Up @@ -55,9 +54,9 @@
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.slice.Slices.utf8Slice;
import static io.trino.execution.buffer.BenchmarkDataGenerator.createValues;
import static io.trino.execution.buffer.CompressionCodec.NONE;
import static io.trino.execution.buffer.PagesSerdeUtil.readPages;
import static io.trino.execution.buffer.PagesSerdeUtil.writePages;
import static io.trino.execution.buffer.TestingPagesSerdes.createTestingPagesSerdeFactory;
import static io.trino.jmh.Benchmarks.benchmark;
import static io.trino.plugin.tpch.TpchTables.getTablePages;
import static io.trino.spi.type.BigintType.BIGINT;
Expand Down Expand Up @@ -207,7 +206,7 @@ public abstract static class TypeBenchmarkData

public void setup(Type type, Function<Random, ?> valueGenerator)
{
PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new TestingBlockEncodingSerde(), NONE);
PagesSerdeFactory serdeFactory = createTestingPagesSerdeFactory();
PageSerializer serializer = serdeFactory.createSerializer(Optional.empty());
PageDeserializer deserializer = serdeFactory.createDeserializer(Optional.empty());
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(type));
Expand Down Expand Up @@ -404,7 +403,7 @@ public static class LineitemBenchmarkData
@Setup
public void setup()
{
PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new TestingBlockEncodingSerde(), NONE);
PagesSerdeFactory serdeFactory = createTestingPagesSerdeFactory();
PageSerializer serializer = serdeFactory.createSerializer(Optional.empty());
PageDeserializer deserializer = serdeFactory.createDeserializer(Optional.empty());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import io.trino.spi.Page;
import io.trino.spi.PageBuilder;
import io.trino.spi.block.BlockBuilder;
import io.trino.spi.block.TestingBlockEncodingSerde;
import io.trino.spi.type.Type;
import org.junit.jupiter.api.Test;
import org.openjdk.jmh.annotations.Benchmark;
Expand Down Expand Up @@ -46,6 +45,7 @@
import static io.airlift.slice.Slices.utf8Slice;
import static io.trino.execution.buffer.CompressionCodec.LZ4;
import static io.trino.execution.buffer.CompressionCodec.NONE;
import static io.trino.execution.buffer.TestingPagesSerdes.createTestingPagesSerdeFactory;
import static io.trino.jmh.Benchmarks.benchmark;
import static io.trino.operator.PageAssertions.assertPageEquals;
import static io.trino.spi.type.VarcharType.VARCHAR;
Expand Down Expand Up @@ -115,7 +115,7 @@ public static class BenchmarkData
@Setup
public void initialize()
{
PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new TestingBlockEncodingSerde(), compressionCodec);
PagesSerdeFactory serdeFactory = createTestingPagesSerdeFactory(compressionCodec);
Optional<SecretKey> encryptionKey = encrypted ? Optional.of(createRandomAesEncryptionKey()) : Optional.empty();
serializer = serdeFactory.createSerializer(encryptionKey);
deserializer = serdeFactory.createDeserializer(encryptionKey);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import static com.google.common.base.Preconditions.checkArgument;
import static io.airlift.concurrent.MoreFutures.tryGetFutureValue;
import static io.trino.execution.buffer.BufferState.FINISHED;
import static io.trino.execution.buffer.CompressionCodec.LZ4;
import static io.trino.execution.buffer.TestingPagesSerdes.createTestingPagesSerdeFactory;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
Expand All @@ -39,7 +41,7 @@ public final class BufferTestUtils
{
private BufferTestUtils() {}

private static final PagesSerdeFactory PAGES_SERDE_FACTORY = new TestingPagesSerdeFactory();
private static final PagesSerdeFactory PAGES_SERDE_FACTORY = createTestingPagesSerdeFactory(LZ4);
static final Duration NO_WAIT = new Duration(0, MILLISECONDS);
static final Duration MAX_WAIT = new Duration(1, SECONDS);
private static final DataSize BUFFERED_PAGE_SIZE = DataSize.ofBytes(serializePage(createPage(42)).getRetainedSize());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import io.trino.spi.Page;
import io.trino.spi.PageBuilder;
import io.trino.spi.QueryId;
import io.trino.spi.block.TestingBlockEncodingSerde;
import io.trino.spi.block.VariableWidthBlockBuilder;
import io.trino.spi.exchange.ExchangeSink;
import io.trino.spi.exchange.ExchangeSinkInstanceHandle;
Expand All @@ -42,7 +41,7 @@
import static io.trino.execution.buffer.BufferState.FINISHED;
import static io.trino.execution.buffer.BufferState.FLUSHING;
import static io.trino.execution.buffer.BufferState.NO_MORE_BUFFERS;
import static io.trino.execution.buffer.CompressionCodec.NONE;
import static io.trino.execution.buffer.TestingPagesSerdes.createTestingPagesSerdeFactory;
import static io.trino.spi.type.VarcharType.VARCHAR;
import static java.util.Objects.requireNonNull;
import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -311,7 +310,7 @@ private static Slice createPage(String value)
VariableWidthBlockBuilder blockBuilder = (VariableWidthBlockBuilder) pageBuilder.getBlockBuilder(0);
blockBuilder.writeEntry(valueSlice);
Page page = pageBuilder.build();
PageSerializer serializer = new PagesSerdeFactory(new TestingBlockEncodingSerde(), NONE).createSerializer(Optional.empty());
PageSerializer serializer = createTestingPagesSerdeFactory().createSerializer(Optional.empty());
return serializer.serialize(page);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,22 @@
import io.trino.metadata.BlockEncodingManager;
import io.trino.metadata.InternalBlockEncodingSerde;

import static io.trino.execution.buffer.CompressionCodec.LZ4;
import static io.trino.execution.buffer.CompressionCodec.NONE;
import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER;

public class TestingPagesSerdeFactory
extends PagesSerdeFactory
public final class TestingPagesSerdes
{
private TestingPagesSerdes() {}

private static final InternalBlockEncodingSerde BLOCK_ENCODING_SERDE = new InternalBlockEncodingSerde(new BlockEncodingManager(), TESTING_TYPE_MANAGER);

public TestingPagesSerdeFactory()
public static PagesSerdeFactory createTestingPagesSerdeFactory()
{
return createTestingPagesSerdeFactory(NONE);
}

public static PagesSerdeFactory createTestingPagesSerdeFactory(CompressionCodec compressionCodec)
{
// compression should be enabled in as many tests as possible
super(BLOCK_ENCODING_SERDE, LZ4);
return new PagesSerdeFactory(BLOCK_ENCODING_SERDE, compressionCodec);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.airlift.units.DataSize;
import io.trino.execution.StageId;
import io.trino.execution.TaskId;
import io.trino.execution.buffer.TestingPagesSerdeFactory;
import io.trino.memory.context.LocalMemoryContext;
import io.trino.operator.Driver;
import io.trino.operator.DriverContext;
Expand Down Expand Up @@ -52,6 +51,8 @@
import static io.airlift.units.DataSize.Unit.GIGABYTE;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static io.trino.SessionTestUtils.TEST_SESSION;
import static io.trino.execution.buffer.CompressionCodec.LZ4;
import static io.trino.execution.buffer.TestingPagesSerdes.createTestingPagesSerdeFactory;
import static io.trino.testing.TestingTaskContext.createTaskContext;
import static java.lang.String.format;
import static java.util.concurrent.Executors.newCachedThreadPool;
Expand Down Expand Up @@ -100,7 +101,7 @@ private RevocableMemoryDriver createRevocableMemoryDriver(MemoryPool userPool, D
TableScanOperator.class.getSimpleName());

OutputFactory outputFactory = new PageConsumerOutputFactory(types -> (page -> {}));
Operator outputOperator = outputFactory.createOutputOperator(2, new PlanNodeId("output"), ImmutableList.of(), Function.identity(), new TestingPagesSerdeFactory()).createOperator(driverContext);
Operator outputOperator = outputFactory.createOutputOperator(2, new PlanNodeId("output"), ImmutableList.of(), Function.identity(), createTestingPagesSerdeFactory(LZ4)).createOperator(driverContext);
RevocableMemoryOperator revocableMemoryOperator = new RevocableMemoryOperator(revokableOperatorContext, reservedPerPage, numberOfPages);

Driver driver = Driver.createDriver(driverContext, revocableMemoryOperator, outputOperator);
Expand Down
Loading