diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaGenerator.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaGenerator.java index fce721c0f..81faebbbd 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaGenerator.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaGenerator.java @@ -31,4 +31,17 @@ public AvroSchemaGenerator disableLogicalTypes() { super.disableLogicalTypes(); return this; } + + @Override + public AvroSchemaGenerator enableWriteEnumAsString() { + super.enableWriteEnumAsString(); + return this; + } + + @Override + public AvroSchemaGenerator disableWriteEnumAsString() { + super.disableWriteEnumAsString(); + return this; + } + } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/EnumVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/EnumVisitor.java new file mode 100644 index 000000000..20f61e163 --- /dev/null +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/EnumVisitor.java @@ -0,0 +1,50 @@ +package com.fasterxml.jackson.dataformat.avro.schema; + +import com.fasterxml.jackson.databind.BeanDescription; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor; + +import org.apache.avro.Schema; + +import java.util.ArrayList; +import java.util.Set; + +/** + * Specific visitor for Java Enum types that are to be exposed as + * Avro Enums. Used unless Java Enums are to be mapped to Avro Strings. + * + * @since 2.18 + */ +public class EnumVisitor extends JsonStringFormatVisitor.Base + implements SchemaBuilder +{ + protected final SerializerProvider _provider; + protected final JavaType _type; + protected final DefinedSchemas _schemas; + + protected Set _enums; + + public EnumVisitor(SerializerProvider provider, DefinedSchemas schemas, JavaType t) { + _schemas = schemas; + _type = t; + _provider = provider; + } + + @Override + public void enumTypes(Set enums) { + _enums = enums; + } + + @Override + public Schema builtAvroSchema() { + if (_enums == null) { + throw new IllegalStateException("Possible enum values cannot be null"); + } + + BeanDescription bean = _provider.getConfig().introspectClassAnnotations(_type); + Schema schema = AvroSchemaHelper.createEnumSchema(bean, new ArrayList<>(_enums)); + _schemas.addSchema(_type, schema); + return schema; + } +} diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java index 49946a04f..48116d1d4 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java @@ -1,6 +1,5 @@ package com.fasterxml.jackson.dataformat.avro.schema; -import java.util.ArrayList; import java.util.Set; import org.apache.avro.Schema; @@ -18,13 +17,9 @@ public class StringVisitor extends JsonStringFormatVisitor.Base { protected final SerializerProvider _provider; protected final JavaType _type; - protected final DefinedSchemas _schemas; - protected Set _enums; - - public StringVisitor(SerializerProvider provider, DefinedSchemas schemas, JavaType t) { - _schemas = schemas; - _type = t; + public StringVisitor(SerializerProvider provider, JavaType type) { + _type = type; _provider = provider; } @@ -35,7 +30,7 @@ public void format(JsonValueFormat format) { @Override public void enumTypes(Set enums) { - _enums = enums; + // Do nothing } @Override @@ -50,11 +45,6 @@ public Schema builtAvroSchema() { return AvroSchemaHelper.createUUIDSchema(); } BeanDescription bean = _provider.getConfig().introspectClassAnnotations(_type); - if (_enums != null) { - Schema s = AvroSchemaHelper.createEnumSchema(bean, new ArrayList<>(_enums)); - _schemas.addSchema(_type, s); - return s; - } Schema schema = Schema.create(Schema.Type.STRING); // Stringable classes need to include the type if (AvroSchemaHelper.isStringable(bean.getClassInfo()) && !_type.hasRawClass(String.class)) { diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java index 865c9cc90..a2b9d6edc 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java @@ -1,24 +1,18 @@ package com.fasterxml.jackson.dataformat.avro.schema; +import java.time.temporal.Temporal; + import com.fasterxml.jackson.core.JsonGenerator; + import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.exc.InvalidDefinitionException; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonAnyFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonArrayFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonBooleanFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonFormatVisitorWrapper; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonIntegerFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonMapFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonNullFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonNumberFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonObjectFormatVisitor; -import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor; +import com.fasterxml.jackson.databind.jsonFormatVisitors.*; + import com.fasterxml.jackson.dataformat.avro.AvroSchema; -import org.apache.avro.Schema; -import java.time.temporal.Temporal; +import org.apache.avro.Schema; public class VisitorFormatWrapperImpl implements JsonFormatVisitorWrapper @@ -32,6 +26,11 @@ public class VisitorFormatWrapperImpl */ protected boolean _logicalTypesEnabled = false; + /** + * @since 2.18 + */ + protected boolean _writeEnumAsString = false; + /** * Visitor used for resolving actual Schema, if structured type * (or one with complex configuration) @@ -105,6 +104,8 @@ public Schema getAvroSchema() { /** * Enables Avro schema with Logical Types generation. + * + * @since 2.13 */ public VisitorFormatWrapperImpl enableLogicalTypes() { _logicalTypesEnabled = true; @@ -113,6 +114,8 @@ public VisitorFormatWrapperImpl enableLogicalTypes() { /** * Disables Avro schema with Logical Types generation. + * + * @since 2.13 */ public VisitorFormatWrapperImpl disableLogicalTypes() { _logicalTypesEnabled = false; @@ -123,6 +126,31 @@ public boolean isLogicalTypesEnabled() { return _logicalTypesEnabled; } + /** + * Enable Java enum to Avro string mapping. + * + * @since 2.18 + */ + public VisitorFormatWrapperImpl enableWriteEnumAsString() { + _writeEnumAsString = true; + return this; + } + + /** + * Disable Java enum to Avro string mapping. + * + * @since 2.18 + */ + public VisitorFormatWrapperImpl disableWriteEnumAsString() { + _writeEnumAsString = false; + return this; + } + + // @since 2.18 + public boolean isWriteEnumAsStringEnabled() { + return _writeEnumAsString; + } + /* /********************************************************************** /* Callbacks @@ -177,7 +205,16 @@ public JsonStringFormatVisitor expectStringFormat(JavaType type) _valueSchema = s; return null; } - StringVisitor v = new StringVisitor(_provider, _schemas, type); + + // 06-Jun-2024: [dataformats-binary#494] Enums may be exposed either + // as native Avro Enums, or as Avro Strings: + if (type.isEnumType() && !isWriteEnumAsStringEnabled()) { + EnumVisitor v = new EnumVisitor(_provider, _schemas, type); + _builder = v; + return v; + } + + StringVisitor v = new StringVisitor(_provider, type); _builder = v; return v; } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/EnumTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/EnumTest.java index dffd032e4..26f4ba379 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/EnumTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/EnumTest.java @@ -2,7 +2,8 @@ public class EnumTest extends AvroTestBase { - protected final static String ENUM_SCHEMA_JSON = "{\n" + // gender as Avro enum + protected final static String ENUM_SCHEMA_JSON = "{\n" +"\"type\": \"record\",\n" +"\"name\": \"Employee\",\n" +"\"fields\": [\n" @@ -11,6 +12,14 @@ public class EnumTest extends AvroTestBase +"}\n" +"]}"; + // gender as Avro string + protected final static String STRING_SCHEMA_JSON = "{" + +" \"type\": \"record\", " + +" \"name\": \"Employee\", " + +" \"fields\": [" + +" {\"name\": \"gender\", \"type\": \"string\"}" + +"]}"; + protected enum Gender { M, F; } protected static class Employee { @@ -23,7 +32,7 @@ protected static class EmployeeStr { private final AvroMapper MAPPER = newMapper(); - public void testSimple() throws Exception + public void test_avroSchemaWithEnum_fromEnumValueToEnumValue() throws Exception { AvroSchema schema = MAPPER.schemaFrom(ENUM_SCHEMA_JSON); Employee input = new Employee(); @@ -31,6 +40,8 @@ public void testSimple() throws Exception byte[] bytes = MAPPER.writer(schema).writeValueAsBytes(input); assertNotNull(bytes); + // Enum Gender.M is encoded as bytes array: {0}, where DEC 0 is encoded long value 0, Gender.M ordinal value + // Enum Gender.F is encoded as bytes array: {2}, where DEC 2 is encoded long value 1, Gender.F ordinal value assertEquals(1, bytes.length); // measured to be current exp size // and then back @@ -40,7 +51,7 @@ public void testSimple() throws Exception assertEquals(Gender.F, output.gender); } - public void testEnumValueAsString() throws Exception + public void test_avroSchemaWithEnum_fromStringValueToEnumValue() throws Exception { AvroSchema schema = MAPPER.schemaFrom(ENUM_SCHEMA_JSON); EmployeeStr input = new EmployeeStr(); @@ -56,4 +67,55 @@ public void testEnumValueAsString() throws Exception assertNotNull(output); assertEquals(Gender.F, output.gender); } + + public void test_avroSchemaWithString_fromEnumValueToEnumValue() throws Exception + { + AvroSchema schema = MAPPER.schemaFrom(STRING_SCHEMA_JSON); + Employee input = new Employee(); + input.gender = Gender.F; + + byte[] bytes = MAPPER.writer(schema).writeValueAsBytes(input); + assertNotNull(bytes); + // Enum Gender.F as string is encoded as {2, 70} bytes array. + // Where + // - DEC 2, HEX 0x2, is a long value 1 written using variable-length zig-zag coding. + // It represents number of following characters in string "F" + // - DEC 70, HEX 0x46, is UTF-8 code for letter F + // + // Enum Gender.M as string is encoded as {2, 77} bytes array. + // Where + // - DEC 2, HEX 0x2, is a long value 1. It is number of following characters in string "M"), + // written using variable-length zig-zag coding. + // - DEC 77, HEX 0x4D, is UTF-8 code for letter M + // + // See https://avro.apache.org/docs/1.8.2/spec.html#Encodings + assertEquals(2, bytes.length); // measured to be current exp size + assertEquals(0x2, bytes[0]); + assertEquals(0x46, bytes[1]); + + // and then back + Employee output = MAPPER.readerFor(Employee.class).with(schema) + .readValue(bytes); + assertNotNull(output); + assertEquals(Gender.F, output.gender); + } + + // Not sure this test makes sense + public void test_avroSchemaWithString_fromStringValueToEnumValue() throws Exception + { + AvroSchema schema = MAPPER.schemaFrom(STRING_SCHEMA_JSON); + EmployeeStr input = new EmployeeStr(); + input.gender = "F"; + + byte[] bytes = MAPPER.writer(schema).writeValueAsBytes(input); + assertNotNull(bytes); + assertEquals(2, bytes.length); // measured to be current exp size + + // and then back + Employee output = MAPPER.readerFor(Employee.class).with(schema) + .readValue(bytes); + assertNotNull(output); + assertEquals(Gender.F, output.gender); + } + } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/Enum_schemaCreationTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/Enum_schemaCreationTest.java new file mode 100644 index 000000000..2a0dde6ba --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/Enum_schemaCreationTest.java @@ -0,0 +1,56 @@ +package com.fasterxml.jackson.dataformat.avro.schema; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.dataformat.avro.AvroMapper; +import com.fasterxml.jackson.dataformat.avro.AvroTestBase; + +import org.apache.avro.Schema; +import org.apache.avro.specific.SpecificData; +import org.junit.Test; + +public class Enum_schemaCreationTest extends AvroTestBase { + + static enum NumbersEnum { + ONE, TWO, THREE + } + + private final AvroMapper MAPPER = newMapper(); + + @Test + public void testJavaEnumToAvroEnum_test() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(NumbersEnum.class , gen); + Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println("schema:\n" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getType()).isEqualTo( Schema.Type.ENUM); + assertThat(actualSchema.getEnumSymbols()).containsExactlyInAnyOrder("ONE", "TWO", "THREE"); + } + + @Test + public void testJavaEnumToAvroString_test() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .enableWriteEnumAsString(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(NumbersEnum.class , gen); + Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println("schema:\n" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getType()).isEqualTo( Schema.Type.STRING); + + // When type is stringable then java-class property is addded. + assertThat(actualSchema.getProp(SpecificData.CLASS_PROP)).isNotEmpty(); + } + +} diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 9eb32cf43..9d8827c10 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -222,6 +222,9 @@ Michal Foksa (MichalFoksa@github) * Contributed #310: (avro) Avro schema generation: allow override namespace with new `@AvroNamespace` annotation (2.14.0) +* Contributed #494: Avro Schema generation: allow mapping Java Enum properties to + Avro String values + (2.18.0) Hunter Herman (hherman1@github) @@ -325,6 +328,11 @@ Yoann Vernageau (@yvrng) when source is an empty `InputStream` (2.17.1) -PJ Fanning (pjfanning@github) +PJ Fanning (@pjfanning) * Contributed #484: Rework synchronization in `ProtobufMapper` (2.18.0) + +Joachim Lous (@jlous) + * Requested #494: Avro Schema generation: allow mapping Java Enum properties to + Avro String values + (2.18.0) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 9b2c1f8f4..e6c0c73fc 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -18,6 +18,10 @@ Active maintainers: #484: (protobuf) Rework synchronization in `ProtobufMapper` (contributed by @pjfanning) +#494: Avro Schema generation: allow mapping Java Enum properties to + Avro String values + (requested by Joachim L) + (contributed by Michal F) 2.17.1 (04-May-2024)