-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Issue 149: Change AvroDeserializer deserialize method to reduce perfo…
…rmance overhead on DatumReader creation (#151) * [changed] AvroDeserializer deserialize method to reduce performance overhead on DatumReader creation Signed-off-by: jingerbread <[email protected]> * Issue 149: Change AvroDeserializer deserialize method to reduce performance overhead on DatumReader creation [updated] GenericAvroDeserializer to avoid creating data readers [used] computeIfAbsent Signed-off-by: jingerbread <[email protected]> * [removed] getKnownSchemaReaders() Signed-off-by: jingerbread <[email protected]> * [changed] visibility of createDatumReader and getKnownSchemaReaders methods for unit test [added] unit test for AvroDeserializer Signed-off-by: jingerbread <[email protected]> * [fixed] failing gradle check for import of io.pravega.schemaregistry.serializers.SerializerFactory Signed-off-by: jingerbread <[email protected]> * [added] licencing headers [removed] Preconditions.checkNotNull(readerSchemaInfo) for AvroGenericDeserializer Signed-off-by: jingerbread <[email protected]> Co-authored-by: shivesh ranjan <[email protected]>
- Loading branch information
1 parent
9f207de
commit 209efcc
Showing
11 changed files
with
3,093 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
123 changes: 123 additions & 0 deletions
123
...ro/src/test/java/io/pravega/schemaregistry/serializer/avro/impl/AvroDeserializerTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| /** | ||
| * Copyright (c) Dell Inc., or its subsidiaries. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| */ | ||
| package io.pravega.schemaregistry.serializer.avro.impl; | ||
|
|
||
| import com.google.common.collect.ImmutableMap; | ||
| import io.pravega.client.stream.Serializer; | ||
| import io.pravega.schemaregistry.client.SchemaRegistryClient; | ||
| import io.pravega.schemaregistry.contract.data.*; | ||
| import io.pravega.schemaregistry.serializer.avro.schemas.AvroSchema; | ||
| import io.pravega.schemaregistry.serializer.avro.testobjs.generated.avro.AddressEntry; | ||
| import io.pravega.schemaregistry.serializer.avro.testobjs.generated.avro.User; | ||
| import io.pravega.schemaregistry.serializer.shared.codec.Codecs; | ||
| import io.pravega.schemaregistry.serializer.shared.impl.SerializerConfig; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.apache.avro.generic.GenericDatumReader; | ||
| import org.apache.avro.io.DatumReader; | ||
| import org.apache.commons.lang3.tuple.Pair; | ||
| import org.junit.Assert; | ||
| import org.junit.Before; | ||
| import org.junit.Test; | ||
| import org.mockito.Mockito; | ||
| import java.nio.ByteBuffer; | ||
| import java.nio.charset.StandardCharsets; | ||
|
|
||
| import static org.mockito.ArgumentMatchers.*; | ||
| import static org.mockito.Mockito.doAnswer; | ||
| import static org.mockito.Mockito.mock; | ||
|
|
||
| @Slf4j | ||
| public class AvroDeserializerTest { | ||
|
|
||
| private Serializer<User> serializer; | ||
| private AvroDeserializer<User> avroDeserializer; | ||
| private AvroGenericDeserializer genericDeserializer; | ||
| private User user; | ||
|
|
||
| @Before | ||
| public void init() { | ||
| AvroSchema<User> userAvroSchema = AvroSchema.of(User.class); | ||
| log.info("Aliases: {}", userAvroSchema.getSchema().getAliases()); | ||
| VersionInfo versionInfo1 = new VersionInfo("avroUser1", 0, 0); | ||
| SchemaRegistryClient client = mock(SchemaRegistryClient.class); | ||
| doAnswer(x -> true).when(client).canReadUsing(anyString(), any()); | ||
| doAnswer(x -> new EncodingId(0)).when(client).getEncodingId(anyString(), any(), any()); | ||
| doAnswer(x -> new EncodingInfo(versionInfo1, userAvroSchema.getSchemaInfo(), Codecs.None.getCodec().getCodecType())).when(client).getEncodingInfo(anyString(), eq(new EncodingId(0))); | ||
| SerializerConfig serializerConfig = SerializerConfig.builder().registryClient(client).groupId("avroUser1") | ||
| .createGroup(SerializationFormat.Avro).registerSchema(true).build(); | ||
| this.serializer = AvroSerializerFactory | ||
| .serializer(serializerConfig, userAvroSchema); | ||
| this.avroDeserializer = Mockito.spy((AvroDeserializer<User>)AvroSerializerFactory.deserializer( | ||
| serializerConfig, userAvroSchema)); | ||
|
|
||
| org.apache.avro.Schema schema = userAvroSchema.getSchema(); | ||
| AvroSchema<Object> objectAvroSchema = AvroSchema.of(schema); | ||
| this.genericDeserializer = Mockito.spy((AvroGenericDeserializer)AvroSerializerFactory.genericDeserializer( | ||
| serializerConfig, objectAvroSchema)); | ||
|
|
||
| this.user = User.newBuilder() | ||
| .setUserId("111111111111") | ||
| .setBiography("Greg Egan was born 20 August 1961") | ||
| .setName("Greg Egan") | ||
| .setEventTimestamp(System.currentTimeMillis()) | ||
| .setKeyValues(null) | ||
| .setKeyValues2(null) | ||
| .setKeyValues3(null) | ||
| .setAddress(AddressEntry.newBuilder().setCity("Perth") | ||
| .setPostalCode(5018) | ||
| .setStreetAddress("4/19 Gardner Road").build()).build(); | ||
| } | ||
|
|
||
| @Test | ||
| public void testCreatingReadersOnceForSchemaGeneric() { | ||
| ImmutableMap<Pair<SchemaInfo, SchemaInfo>, GenericDatumReader<Object>> knownSchemaReaders1 = genericDeserializer.getKnownSchemaReaders(); | ||
| Assert.assertTrue(knownSchemaReaders1.isEmpty()); | ||
| Assert.assertEquals(0, knownSchemaReaders1.size()); | ||
|
|
||
| ByteBuffer serialized = serializer.serialize(user); | ||
| int payloadSize = serialized.limit(); | ||
| log.info("serialized into {}", payloadSize); | ||
| Assert.assertEquals(100, payloadSize); | ||
| byte[] bytes = serialized.array(); | ||
| log.info("bytes: {}", new String(bytes, StandardCharsets.UTF_8)); | ||
| Object user1 = genericDeserializer.deserialize(ByteBuffer.wrap(bytes)); | ||
| log.info("deserialized {}", user1); | ||
| ImmutableMap<Pair<SchemaInfo, SchemaInfo>, GenericDatumReader<Object>> knownSchemaReaders2 = genericDeserializer.getKnownSchemaReaders(); | ||
| Assert.assertEquals(1, knownSchemaReaders2.size()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testCreatingReadersOnceForSchema() { | ||
| ImmutableMap<ByteBuffer, DatumReader<User>> knownSchemaReaders1 = avroDeserializer.getKnownSchemaReaders(); | ||
| Assert.assertFalse(knownSchemaReaders1.isEmpty()); | ||
| Assert.assertEquals(1, knownSchemaReaders1.size()); | ||
| AvroSchema<User> userAvroSchema = AvroSchema.of(User.class); | ||
| DatumReader<User> datumReader = knownSchemaReaders1.get(userAvroSchema.getSchemaInfo().getSchemaData()); | ||
| Assert.assertNotNull(datumReader); | ||
|
|
||
| ByteBuffer serialized = serializer.serialize(user); | ||
| int payloadSize = serialized.limit(); | ||
| log.info("serialized into {}", payloadSize); | ||
| Assert.assertEquals(100, payloadSize); | ||
| byte[] bytes = serialized.array(); | ||
| log.info("bytes: {}", new String(bytes, StandardCharsets.UTF_8)); | ||
| User user1 = avroDeserializer.deserialize(ByteBuffer.wrap(bytes)); | ||
|
|
||
| log.info("deserialized {}", user1); | ||
| Assert.assertEquals(user, user1); | ||
| serializer.serialize(user1); | ||
| ImmutableMap<ByteBuffer, DatumReader<User>> knownSchemaReaders2 = avroDeserializer.getKnownSchemaReaders(); | ||
| Assert.assertEquals(1, knownSchemaReaders2.size()); | ||
| Assert.assertEquals(knownSchemaReaders1, knownSchemaReaders2); | ||
| // called zero times outside constructor | ||
| Mockito.verify(avroDeserializer, Mockito.times(0)).createDatumReader(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 16 additions & 0 deletions
16
...src/test/java/io/pravega/schemaregistry/serializer/avro/testobjs/EventTimestampAware.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| /** | ||
| * Copyright (c) Dell Inc., or its subsidiaries. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| */ | ||
| package io.pravega.schemaregistry.serializer.avro.testobjs; | ||
|
|
||
| public interface EventTimestampAware { | ||
| void setEventTimestamp(Long value); | ||
|
|
||
| Long getEventTimestamp(); | ||
| } |
Oops, something went wrong.