Skip to content

Commit

Permalink
ENH: conditional required annotation for schema (opensearch-project#5109
Browse files Browse the repository at this point in the history 
)

* MAINT: conditional required schema

Signed-off-by: George Chen <[email protected]>
  • Loading branch information
@chenqi0805
chenqi0805 authored Oct 28, 2024
1 parent 1dadd9e commit 56cc569
Show file tree
Hide file tree
Showing 9 changed files with 298 additions and 0 deletions.
50 changes: 50 additions & 0 deletions ...r-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package org.opensearch.dataprepper.model.annotations;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* Annotation used in schema generation to define the if-then-else requirements.
*/
@Target({ ElementType.FIELD, ElementType.TYPE })
@Retention(RetentionPolicy.RUNTIME)
public @interface ConditionalRequired {
/**
* Array of if-then-else requirements.
*/
IfThenElse[] value();

/**
* Annotation to represent an if-then-else requirement.
*/
@interface IfThenElse {
/**
* Array of property schemas involved in if condition.
*/
SchemaProperty[] ifFulfilled();
/**
* Array of property schemas involved in then expectation.
*/
SchemaProperty[] thenExpect();
/**
* Array of property schemas involved in else expectation.
*/
SchemaProperty[] elseExpect() default {};
}

/**
* Annotation to represent a property schema.
*/
@interface SchemaProperty {
/**
* Name of the property.
*/
String field();
/**
* Value of the property. Empty string means any non-null value is allowed.
*/
String value() default "";
}
}
62 changes: 62 additions & 0 deletions ...r-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.classmate.types.ResolvedObjectType;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.victools.jsonschema.generator.FieldScope;
import com.github.victools.jsonschema.generator.Module;
Expand All @@ -13,8 +14,10 @@
import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder;
import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart;
import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart;
import com.github.victools.jsonschema.generator.SchemaKeyword;
import com.github.victools.jsonschema.generator.SchemaVersion;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.event.EventKey;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin;
Expand All @@ -25,6 +28,7 @@
import java.util.Collections;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -52,6 +56,7 @@ public ObjectNode convertIntoJsonSchema(
resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart);
resolveDependentRequiresFields(scopeSchemaGeneratorConfigPart);
overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset);
overrideTypeAttributeWithConditionalRequired(configBuilder.forTypesInGeneral());
resolveDataPrepperTypes(scopeSchemaGeneratorConfigPart);
scopeSchemaGeneratorConfigPart.withInstanceAttributeOverride(new ExampleValuesInstanceAttributeOverride());

Expand Down Expand Up @@ -107,6 +112,63 @@ private void overrideDataPrepperPluginTypeAttribute(
});
}

private void overrideTypeAttributeWithConditionalRequired(
final SchemaGeneratorGeneralConfigPart schemaGeneratorGeneralConfigPart) {
schemaGeneratorGeneralConfigPart.withTypeAttributeOverride((node, scope, context) -> {
final ConditionalRequired conditionalRequiredAnnotation = scope.getContext()
.getTypeAnnotationConsideringHierarchy(scope.getType(), ConditionalRequired.class);
if (conditionalRequiredAnnotation != null) {
final SchemaGeneratorConfig config = context.getGeneratorConfig();
final ArrayNode ifThenElseArrayNode = node.putArray(config.getKeyword(SchemaKeyword.TAG_ALLOF));
Arrays.asList(conditionalRequiredAnnotation.value()).forEach(ifThenElse -> {
ObjectNode ifThenElseNode = config.createObjectNode();
final ObjectNode ifObjectNode = constructIfObjectNode(config, ifThenElse.ifFulfilled());
ifThenElseNode.set(config.getKeyword(SchemaKeyword.TAG_IF), ifObjectNode);
final ObjectNode thenObjectNode = constructExpectObjectNode(config, ifThenElse.thenExpect());
ifThenElseNode.set(config.getKeyword(SchemaKeyword.TAG_THEN), thenObjectNode);
final ObjectNode elseObjectNode = constructExpectObjectNode(config, ifThenElse.elseExpect());
if (!elseObjectNode.isEmpty()) {
ifThenElseNode.set(config.getKeyword(SchemaKeyword.TAG_ELSE), elseObjectNode);
}
ifThenElseArrayNode.add(ifThenElseNode);
});
}
});
}

private ObjectNode constructIfObjectNode(final SchemaGeneratorConfig config,
final ConditionalRequired.SchemaProperty[] schemaProperties) {
final ObjectNode ifObjectNode = config.createObjectNode();
final ObjectNode ifPropertiesNode = ifObjectNode.putObject(config.getKeyword(SchemaKeyword.TAG_PROPERTIES));
Arrays.asList(schemaProperties).forEach(schemaProperty -> {
ifPropertiesNode.putObject(schemaProperty.field()).put(
config.getKeyword(SchemaKeyword.TAG_CONST), schemaProperty.value());
});
return ifObjectNode;
}

private ObjectNode constructExpectObjectNode(final SchemaGeneratorConfig config,
final ConditionalRequired.SchemaProperty[] schemaProperties) {
final ObjectNode expectObjectNode = config.createObjectNode();
final ObjectNode expectPropertiesNode = config.createObjectNode();
final ArrayNode expectRequiredNode = config.createArrayNode();
Arrays.asList(schemaProperties).forEach(schemaProperty -> {
if (!Objects.equals(schemaProperty.value(), "")) {
expectPropertiesNode.putObject(schemaProperty.field()).put(
config.getKeyword(SchemaKeyword.TAG_CONST), schemaProperty.value());
} else {
expectRequiredNode.add(schemaProperty.field());
}
});
if (!expectPropertiesNode.isEmpty()) {
expectObjectNode.set(config.getKeyword(SchemaKeyword.TAG_PROPERTIES), expectPropertiesNode);
}
if (!expectRequiredNode.isEmpty()) {
expectObjectNode.set(config.getKeyword(SchemaKeyword.TAG_REQUIRED), expectRequiredNode);
}
return expectObjectNode;
}

private void resolveDefaultValueFromJsonProperty(
final SchemaGeneratorConfigPart<FieldScope> scopeSchemaGeneratorConfigPart) {
scopeSchemaGeneratorConfigPart.withDefaultResolver(field -> {
Expand Down
91 changes: 91 additions & 0 deletions ...ugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.github.victools.jsonschema.generator.SchemaVersion;
import org.junit.jupiter.api.Test;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.event.EventKey;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
Expand Down Expand Up @@ -95,6 +96,94 @@ void testConvertIntoJsonSchemaWithEventKey() throws JsonProcessingException {
assertThat(propertiesNode.get("testAttributeEventKey").get("type"), is(equalTo(TextNode.valueOf("string"))));
}

@Test
void testConvertIntoJsonSchemaWithConditionalRequired() throws JsonProcessingException {
final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList(), pluginProvider);
final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema(
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class);
final JsonNode allOfNode = jsonSchemaNode.at("/allOf");
assertThat(allOfNode, instanceOf(ArrayNode.class));
assertThat(allOfNode.size(), equalTo(2));

final JsonNode ifThenElseNode1 = allOfNode.get(0);
assertThat(ifThenElseNode1.has("else"), is(false));
final JsonNode ifNode1 = ifThenElseNode1.at("/if");
assertThat(ifNode1, instanceOf(ObjectNode.class));
final JsonNode ifPropertiesNode1 = ifNode1.at("/properties");
assertThat(ifPropertiesNode1, instanceOf(ObjectNode.class));
final JsonNode attributeNode1 = ifPropertiesNode1.at("/test_mutually_exclusive_attribute_a");
assertThat(attributeNode1, instanceOf(ObjectNode.class));
final JsonNode thenNode1 = ifThenElseNode1.at("/then");
assertThat(thenNode1, instanceOf(ObjectNode.class));
assertThat(thenNode1.has("properties"), is(false));
final JsonNode thenRequiredNode1 = thenNode1.at("/required");
assertThat(thenRequiredNode1, instanceOf(ArrayNode.class));
assertThat(thenRequiredNode1.isEmpty(), is(false));

final JsonNode ifThenElseNode2 = allOfNode.get(1);
final JsonNode ifNode2 = ifThenElseNode2.at("/if");
assertThat(ifNode2, instanceOf(ObjectNode.class));
final JsonNode ifPropertiesNode2 = ifNode2.at("/properties");
assertThat(ifPropertiesNode2, instanceOf(ObjectNode.class));
final JsonNode ifAttributeNode2 = ifPropertiesNode2.at("/test_mutually_exclusive_attribute_a");
assertThat(ifAttributeNode2, instanceOf(ObjectNode.class));
final JsonNode thenNode2 = ifThenElseNode2.at("/then");
assertThat(thenNode2, instanceOf(ObjectNode.class));
assertThat(thenNode2.has("required"), is(false));
final JsonNode thenPropertiesNode2 = thenNode2.at("/properties");
assertThat(thenPropertiesNode2, instanceOf(ObjectNode.class));
assertThat(thenPropertiesNode2.isEmpty(), is(false));
final JsonNode thenAttributeNode2 = thenPropertiesNode2.at("/test_mutually_exclusive_attribute_c");
assertThat(thenAttributeNode2, instanceOf(ObjectNode.class));
final JsonNode thenAttributeValueNode2 = thenAttributeNode2.at("/const");
assertThat(thenAttributeValueNode2, instanceOf(TextNode.class));
assertThat(thenAttributeValueNode2.asText(), equalTo("\"option1\""));
final JsonNode elseNode2 = ifThenElseNode2.at("/else");
assertThat(elseNode2, instanceOf(ObjectNode.class));
assertThat(elseNode2.has("required"), is(false));
final JsonNode elsePropertiesNode2 = elseNode2.at("/properties");
assertThat(elsePropertiesNode2, instanceOf(ObjectNode.class));
assertThat(elsePropertiesNode2.isEmpty(), is(false));
final JsonNode elseAttributeNode2 = elsePropertiesNode2.at("/test_mutually_exclusive_attribute_c");
assertThat(elseAttributeNode2, instanceOf(ObjectNode.class));
final JsonNode elseAttributeValueNode2 = elseAttributeNode2.at("/const");
assertThat(elseAttributeValueNode2, instanceOf(TextNode.class));
assertThat(elseAttributeValueNode2.asText(), equalTo("\"option2\""));
}

@ConditionalRequired(value = {
@ConditionalRequired.IfThenElse(
ifFulfilled = {
@ConditionalRequired.SchemaProperty(
field = "test_mutually_exclusive_attribute_a",
value = "null")
},
thenExpect = {
@ConditionalRequired.SchemaProperty(
field = "test_mutually_exclusive_attribute_b"
)
}
),
@ConditionalRequired.IfThenElse(
ifFulfilled = {
@ConditionalRequired.SchemaProperty(
field = "test_mutually_exclusive_attribute_a",
value = "null")
},
thenExpect = {
@ConditionalRequired.SchemaProperty(
field = "test_mutually_exclusive_attribute_c",
value = "\"option1\""
)
},
elseExpect = {
@ConditionalRequired.SchemaProperty(
field = "test_mutually_exclusive_attribute_c",
value = "\"option2\""
)
}
)
})
@JsonClassDescription("test config")
static class TestConfig {
private String testAttributeWithGetter;
Expand All @@ -113,6 +202,8 @@ static class TestConfig {

private String testMutuallyExclusiveAttributeB;

private String testMutuallyExclusiveAttributeC;

@JsonProperty
@AlsoRequired(values = {
@AlsoRequired.Required(name="test_mutually_exclusive_attribute_a")
Expand Down
13 changes: 13 additions & 0 deletions .../src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import jakarta.validation.constraints.AssertTrue;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty;
import org.opensearch.dataprepper.model.annotations.ExampleValues;
import org.opensearch.dataprepper.model.annotations.ExampleValues.Example;

Expand All @@ -20,6 +23,16 @@
import java.util.Locale;
import java.time.format.DateTimeFormatter;

@ConditionalRequired(value = {
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "match", value = "null")},
thenExpect = {@SchemaProperty(field = "from_time_received", value = "true")}
),
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "from_time_received", value = "false")},
thenExpect = {@SchemaProperty(field = "match")}
)
})
@JsonPropertyOrder
@JsonClassDescription("The <code>date</code> processor adds a default timestamp to an event, parses timestamp fields, " +
"and converts timestamp information to the International Organization for Standardization (ISO) 8601 format. " +
Expand Down
34 changes: 34 additions & 0 deletions ...ava/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,44 @@
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty;

import java.util.List;
import java.util.stream.Stream;

@ConditionalRequired(value = {
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "key", value = "null")},
thenExpect = {@SchemaProperty(field = "metadata_key")}
),
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "metadata_key", value = "null")},
thenExpect = {@SchemaProperty(field = "key")}
),
@IfThenElse(
ifFulfilled = {
@SchemaProperty(field = "format", value = "null"),
@SchemaProperty(field = "value", value = "null"),
},
thenExpect = {@SchemaProperty(field = "value_expression")}
),
@IfThenElse(
ifFulfilled = {
@SchemaProperty(field = "format", value = "null"),
@SchemaProperty(field = "value_expression", value = "null"),
},
thenExpect = {@SchemaProperty(field = "value")}
),
@IfThenElse(
ifFulfilled = {
@SchemaProperty(field = "value", value = "null"),
@SchemaProperty(field = "value_expression", value = "null"),
},
thenExpect = {@SchemaProperty(field = "format")}
)
})
@JsonPropertyOrder
@JsonClassDescription("The <code>add_entries</code> processor adds entries to an event.")
public class AddEntryProcessorConfig {
Expand Down
13 changes: 13 additions & 0 deletions ...opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,24 @@
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty;
import org.opensearch.dataprepper.typeconverter.ConverterArguments;

import java.util.List;
import java.util.Optional;

@ConditionalRequired(value = {
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "key", value = "null")},
thenExpect = {@SchemaProperty(field = "keys")}
),
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "keys", value = "null")},
thenExpect = {@SchemaProperty(field = "key")}
)
})
@JsonPropertyOrder
@JsonClassDescription("The <code>convert_type</code> processor converts a value associated with the specified key in " +
"a event to the specified type. It is a casting processor that changes the types of specified fields in events.")
Expand Down
9 changes: 9 additions & 0 deletions ...va/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,21 @@
import com.fasterxml.jackson.annotation.JsonValue;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty;

import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@ConditionalRequired(value = {
@IfThenElse(
ifFulfilled = {@SchemaProperty(field = "use_source_key", value = "false")},
thenExpect = {@SchemaProperty(field = "key")}
)
})
@JsonPropertyOrder
@JsonClassDescription("The <code>list_to_map</code> processor converts a list of objects from an event, " +
"where each object contains a <code>key</code> field, into a map of target keys.")
Expand Down
Loading

0 comments on commit 56cc569

Please sign in to comment.