schema revisions, add json aliases

Signed-off-by: Katherine Shen <[email protected]>

data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java

@@ -31,7 +31,7 @@ public class AggregateProcessorConfig {
 
     @JsonPropertyDescription("An unordered list by which to group events. Events with the same values as these keys are put into the same group. " +
             "If an event does not contain one of the <code>identification_keys</code>, then the value of that key is considered to be equal to <code>null</code>. " +
-            "At least one <code>identification_key</code> is required. An example configuration is [\"sourceIp\", \"destinationIp\", \"port\"].")
+            "At least one <code>identification_key</code> is required.")
     @JsonProperty("identification_keys")
     @NotEmpty
     @ExampleValues({

data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java

@@ -34,7 +34,7 @@ public class AnomalyDetectorProcessorConfig {
     @UsesDataPrepperPlugin(pluginType = AnomalyDetectorMode.class)
     private PluginModel detectorMode;
 
-    @JsonPropertyDescription("If provided, anomalies will be detected within each unique instance of these keys. For example, if you provide the ip field, anomalies will be detected separately for each unique IP address.")
+    @JsonPropertyDescription("If provided, anomalies will be detected within each unique instance of these keys. For example, if you provide the IP field, anomalies will be detected separately for each unique IP address.")
     @JsonProperty("identification_keys")
     @ExampleValues({
             @Example(value = "ip_address", description = "Anomalies will be detected separately for each unique IP address from the existing ip_address key of the Event.")

data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/modes/RandomCutForestModeConfig.java

@@ -51,7 +51,7 @@ public class RandomCutForestModeConfig {
     @JsonProperty(value = "time_decay", defaultValue = "" + DEFAULT_TIME_DECAY)
     private double timeDecay = DEFAULT_TIME_DECAY;
 
-    @JsonPropertyDescription("Output after indicates the number of events to consume before outputting anomalies. Default is 32.")
+    @JsonPropertyDescription("The number of events to train the model before generating an anomaly event. Default is 32.")
     @JsonProperty(value = "output_after", defaultValue = "" + DEFAULT_OUTPUT_AFTER)
     private int outputAfter = DEFAULT_OUTPUT_AFTER;
 

data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java

@@ -17,7 +17,7 @@
 import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig;
 
 @JsonPropertyOrder
-@JsonClassDescription("The <code>aws_lambda</code> processor enables invocation of an AWS Lambda function within your Data Prepper pipeline in order to process events." +
+@JsonClassDescription("The <code>aws_lambda</code> processor enables invocation of an AWS Lambda function within your pipeline in order to process events. " +
         "It supports both synchronous and asynchronous invocations based on your use case.")
 public class LambdaProcessorConfig extends LambdaCommonConfig {
   static final String DEFAULT_INVOCATION_TYPE = "request-response";
@@ -40,7 +40,7 @@ public class LambdaProcessorConfig extends LambdaCommonConfig {
 
   @JsonPropertyDescription("Defines a condition for event to use this processor.")
   @ExampleValues({
-          @Example(value = "event['status'] == 'process'", description = "The processor will only run on this condition.")
+          @Example(value = "/some_key == null", description = "The processor will only run on this condition.")
   })
   @JsonProperty("lambda_when")
   private String whenCondition;

data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java

@@ -10,6 +10,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.JsonAlias;
 import jakarta.validation.constraints.AssertTrue;
 import org.opensearch.dataprepper.model.annotations.AlsoRequired;
 import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
@@ -133,8 +134,7 @@ public static boolean isValidPattern(final String pattern) {
     private Boolean fromTimeReceived = DEFAULT_FROM_TIME_RECEIVED;
 
     @JsonProperty("match")
-    @JsonPropertyDescription("The date match configuration. " +
-            "This option cannot be defined at the same time as <code>from_time_received</code>. " +
+    @JsonPropertyDescription("This option cannot be defined at the same time as <code>from_time_received</code>. " +
             "The date processor will use the first pattern that matches each event's timestamp field. " +
             "You must provide at least one pattern unless you have <code>from_time_received</code>.")
     @AlsoRequired(values = {
@@ -155,9 +155,11 @@ public static boolean isValidPattern(final String pattern) {
     })
     private String outputFormat = DEFAULT_OUTPUT_FORMAT;
 
-    @JsonProperty("to_origination_metadata")
-    @JsonPropertyDescription("When <code>true</code>, the matched time is also added to the event's metadata as an instance of " +
-            "<code>Instant</code>. Default is <code>false</code>.")
+    @JsonProperty("origination_timestamp_to_metadata")
+    @JsonAlias("to_origination_metadata")
+    @JsonPropertyDescription("Include the origination timestamp in the metadata. " +
+            "Enabling this option will use this timestamp to report the EndToEndLatency metric " +
+            "when events reach the sink. Default is <code>false</code>.")
     private Boolean toOriginationMetadata = DEFAULT_TO_ORIGINATION_METADATA;
 
     @JsonProperty("source_timezone")
@@ -187,7 +189,7 @@ public static boolean isValidPattern(final String pattern) {
             "or a string representation of the " +
             "<a href=\"https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html\">locale</a> object, such as <code>en_US</code>. " +
             "A full list of locale fields, including language, country, and variant, can be found " +
-            "<a href=\"https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry\">here</a>." +
+            "<a href=\"https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry\">here</a>. " +
             "Default is <code>Locale.ROOT</code>.")
     @ExampleValues({
             @Example("en-US"),
@@ -196,7 +198,7 @@ public static boolean isValidPattern(final String pattern) {
     private String locale;
 
     @JsonProperty("date_when")
-    @JsonPropertyDescription("Specifies under what condition the <code>date</code> processor should perform matching. " +
+    @JsonPropertyDescription("Specifies under what condition the <code>date</code> processor should run. " +
             "Default is no condition.")
     @ExampleValues({
         @Example(value = "/some_key == null", description = "Only runs the date processor on the Event if some_key is null or doesn't exist.")

data-prepper-plugins/drop-events-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/drop/DropEventProcessorConfig.java

@@ -27,7 +27,7 @@ public class DropEventProcessorConfig {
     })
     private String dropWhen;
 
-    @JsonPropertyDescription("Specifies how exceptions are handled when an exception occurs while evaluating an event. Default value is <code>skip</code>, which drops the event so that it is not sent to further processors or sinks.")
+    @JsonPropertyDescription("Specifies how exceptions are handled when an exception occurs while evaluating an event. Default value is <code>skip</code>, which sends the events to further processors or sinks.")
     @JsonProperty(value = "handle_failed_events", defaultValue = "skip")
     private HandleFailedEventsOption handleFailedEventsOption = HandleFailedEventsOption.SKIP;
 

data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessorConfig.java

@@ -7,9 +7,9 @@
 
 import com.fasterxml.jackson.annotation.JsonClassDescription;
 import com.fasterxml.jackson.annotation.JsonProperty;
-
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.JsonAlias;
 import jakarta.validation.constraints.AssertTrue;
 import jakarta.validation.constraints.NotNull;
 import org.opensearch.dataprepper.model.annotations.ExampleValues;
@@ -20,7 +20,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The <code>flatten</code> processor transforms nested objects inside of events into flattened structures.")
+@JsonClassDescription("The <code>flatten</code>  processor transforms nested objects inside of events into flattened structures.")
 public class FlattenProcessorConfig {
     static final String REMOVE_LIST_INDICES_KEY = "remove_list_indices";
 
@@ -44,12 +44,14 @@ public class FlattenProcessorConfig {
     })
     private String target;
 
-    @JsonProperty("remove_processed_fields")
+    @JsonProperty("remove_source_keys")
+    @JsonAlias("remove_processed_fields")
     @JsonPropertyDescription("When <code>true</code>, the processor removes all processed fields from the source. " +
             "The default is <code>false</code> which leaves the source fields.")
     private boolean removeProcessedFields = false;
 
-    @JsonProperty(REMOVE_LIST_INDICES_KEY)
+    @JsonProperty("remove_list_items")
+    @JsonAlias(REMOVE_LIST_INDICES_KEY)
     @JsonPropertyDescription("When <code>true</code>, the processor converts the fields from the source map into lists and " +
             "puts the lists into the target field. Default is <code>false</code>.")
     private boolean removeListIndices = false;

data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java

@@ -195,8 +195,7 @@ public class KeyValueProcessorConfig {
             "If this flag is enabled, then the content between the delimiters is considered to be one entity and " +
             "they are not parsed as key-value pairs. The following characters are used a group delimiters: " +
             "<code>{...}</code>, <code>[...]</code>, <code>&lt;...&gt;</code>, <code>(...)</code>, <code>\"...\"</code>, <code>'...'</code>, <code>http://... (space)</code>, and <code>https:// (space)</code>. " +
-            "Default is <code>false</code>. For example, if <code>value_grouping</code> is <code>true</code>, then " +
-            "<code>{\"key1=[a=b,c=d]&amp;key2=value2\"}</code> parses to <code>{\"key1\": \"[a=b,c=d]\", \"key2\": \"value2\"}</code>.")
+            "Default is <code>false</code>.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = FIELD_DELIMITER_REGEX_KEY, allowedValues = {"null"})
     })

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java

@@ -66,7 +66,7 @@ public class AddEntryProcessorConfig {
     public static class Entry {
         @JsonPropertyDescription("The key of the new entry to be added. Some examples of keys include <code>my_key</code>, " +
                 "<code>myKey</code>, and <code>object/sub_Key</code>. The key can also be a format expression, for example, <code>${/key1}</code> to " +
-                "use the value of field <code>key1</code> as the key. Either one of <code>key</code> or <code>metadata_key</code> is required.")
+                "use the value of field <code>key1</code> as the key. At least one of <code>key</code> or <code>metadata_key</code> is required.")
         @AlsoRequired(values = {
                 @AlsoRequired.Required(name=METADATA_KEY_KEY, allowedValues = {"null"})
         })
@@ -79,7 +79,8 @@ public static class Entry {
 
         @JsonProperty(METADATA_KEY_KEY)
         @JsonPropertyDescription("The key for the new metadata attribute. The argument must be a literal string key " +
-                "and not a JSON Pointer. Either one of <code>key</code> or <code>metadata_key</code> is required.")
+                "and not a JSON Pointer. Adds an attribute to the Events that will not be sent to the sinks, but can be used for condition expressions and routing with the getMetadata function. " +
+                "At least one of <code>key</code> or <code>metadata_key</code> is required.")
         @AlsoRequired(values = {
                 @AlsoRequired.Required(name="key", allowedValues = {"null"})
         })
@@ -103,7 +104,7 @@ public static class Entry {
         private Object value;
 
         @JsonPropertyDescription("A format string to use as the value of the new entry, for example, " +
-                "<code>${key1}-${key2}</code>, where <code>key1</code> and <code>key2</code> are existing keys in the event. Required if neither" +
+                "<code>${key1}-${key2}</code>, where <code>key1</code> and <code>key2</code> are existing keys in the event. Required if neither " +
                 "<code>value</code> nor <code>value_expression</code> is specified.")
         @AlsoRequired(values = {
                 @AlsoRequired.Required(name="value", allowedValues = {"null"}),
@@ -132,7 +133,7 @@ public static class Entry {
 
         @JsonProperty(OVERWRITE_IF_KEY_EXISTS_KEY)
         @JsonPropertyDescription("When set to <code>true</code>, the existing value is overwritten if <code>key</code> already exists " +
-                "in the event. The default value is <code>false</code>.")
+                "in the event. Only one of <code>overwrite_if_key_exists</code> or <code>append_if_key_exists</code> can be <code>true</code>. The default value is <code>false</code>.")
         @AlsoRequired(values = {
                 @AlsoRequired.Required(name=APPEND_IF_KEY_EXISTS_KEY, allowedValues = {"false"})
         })

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java

@@ -9,6 +9,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
+import com.fasterxml.jackson.annotation.JsonAlias;
 import org.opensearch.dataprepper.model.annotations.AlsoRequired;
 import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
 import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse;
@@ -55,7 +56,8 @@ public class ConvertEntryTypeProcessorConfig implements ConverterArguments {
     @JsonPropertyDescription("Target type for the values. Default value is <code>integer</code>.")
     private TargetType type = TargetType.INTEGER;
 
-    @JsonProperty("null_values")
+    @JsonProperty("null_conversion_values")
+    @JsonAlias("null_values")
     @JsonPropertyDescription("String representation of what constitutes a null value. If the field value equals one of these strings, then the value is considered null and is converted to null.")
     private List<String> nullValues;
 

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java

@@ -20,7 +20,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The <code>copy_values</code> processor copies values within an event to other fields within the event.")
+@JsonClassDescription("The <code>copy_values</code> processor copies values from an event to another key in an event.")
 public class CopyValueProcessorConfig {
     static final String FROM_LIST_KEY = "from_list";
     static final String TO_LIST_KEY = "to_list";
@@ -30,15 +30,13 @@ public static class Entry {
         @NotEmpty
         @NotNull
         @JsonProperty("from_key")
-        @JsonPropertyDescription("The key of the entry to be copied. Either <code>from_key</code> and " +
-                "<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
+        @JsonPropertyDescription("The key of the entry to be copied.")
         private String fromKey;
 
         @NotEmpty
         @NotNull
         @JsonProperty("to_key")
-        @JsonPropertyDescription("The key of the new entry to be added. Either <code>from_key</code> and " +
-                "<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
+        @JsonPropertyDescription("The key of the new entry to be added.")
         private String toKey;
 
         @JsonProperty("overwrite_if_to_key_exists")
@@ -88,16 +86,16 @@ public Entry() {
     private List<Entry> entries;
 
     @JsonProperty(FROM_LIST_KEY)
-    @JsonPropertyDescription("The key of the list of objects to be copied. Either <code>from_key</code> and " +
-            "<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
+    @JsonPropertyDescription("The key of the list of objects to be copied. " +
+            "Both <code>from_key</code> and <code>to_key</code> will be applied on the corresponding list.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = TO_LIST_KEY)
     })
     private String fromList;
 
     @JsonProperty(TO_LIST_KEY)
-    @JsonPropertyDescription("The key of the new list to be added. Either <code>from_key</code> and " +
-            "<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
+    @JsonPropertyDescription("The key of the new list to be added. " +
+            "Both <code>from_key</code> and <code>to_key</code> will be applied on the corresponding list.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = FROM_LIST_KEY)
     })

data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java

@@ -27,7 +27,7 @@
 public class ObfuscationProcessorConfig {
 
     @JsonProperty("source")
-    @JsonPropertyDescription("The source field to obfuscate.")
+    @JsonPropertyDescription("The source key to obfuscate. Default action is to mask ith <code>*</code>.")
     @NotEmpty
     @NotNull
     @ExampleValues({