refactor(core): move YamlFlowParser to YamlParser

since it's already able to handle all types with generic
kestra-io · Nov 4, 2024 · 6e58b3f · 6e58b3f
1 parent 93ff17a
commit 6e58b3f
Show file tree

Hide file tree

Showing 26 changed files with 102 additions and 110 deletions.
diff --git a/cli/src/main/java/io/kestra/cli/AbstractValidateCommand.java b/cli/src/main/java/io/kestra/cli/AbstractValidateCommand.java
@@ -2,7 +2,7 @@
 
 import io.kestra.core.models.validations.ModelValidator;
 import io.kestra.core.models.validations.ValidateConstraintViolation;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
 import io.micronaut.http.MediaType;
@@ -62,7 +62,7 @@ public static void handleValidateConstraintViolation(ValidateConstraintViolation
     public static String buildYamlBody(Path directory) throws IOException {
         try(var files = Files.walk(directory)) {
             return files.filter(Files::isRegularFile)
-                .filter(YamlFlowParser::isValidExtension)
+                .filter(YamlParser::isValidExtension)
                 .map(throwFunction(path -> Files.readString(path, Charset.defaultCharset())))
                 .collect(Collectors.joining("\n---\n"));
         }
@@ -71,7 +71,7 @@ public static String buildYamlBody(Path directory) throws IOException {
     // bug in micronaut, we can't inject YamlFlowParser & ModelValidator, so we inject from implementation
     public Integer call(
         Class<?> cls,
-        YamlFlowParser yamlFlowParser,
+        YamlParser yamlParser,
         ModelValidator modelValidator,
         Function<Object, String> identity,
         Function<Object, List<String>> warningsFunction,
@@ -85,10 +85,10 @@ public Integer call(
         if(this.local) {
             try(var files = Files.walk(directory)) {
                 files.filter(Files::isRegularFile)
-                    .filter(YamlFlowParser::isValidExtension)
+                    .filter(YamlParser::isValidExtension)
                     .forEach(path -> {
                         try {
-                            Object parse = yamlFlowParser.parse(path.toFile(), cls);
+                            Object parse = yamlParser.parse(path.toFile(), cls);
                             modelValidator.validate(parse);
                             stdOut("@|green \u2713|@ - " + identity.apply(parse));
                             List<String> warnings = warningsFunction.apply(parse);

diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/FlowDotCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/FlowDotCommand.java
@@ -3,7 +3,7 @@
 import io.kestra.cli.AbstractCommand;
 import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.hierarchies.GraphCluster;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.kestra.core.services.Graph2DotService;
 import io.kestra.core.utils.GraphUtils;
 import io.micronaut.context.ApplicationContext;
@@ -29,7 +29,7 @@ public class FlowDotCommand extends AbstractCommand {
     public Integer call() throws Exception {
         super.call();
 
-        YamlFlowParser parser = applicationContext.getBean(YamlFlowParser.class);
+        YamlParser parser = applicationContext.getBean(YamlParser.class);
         Flow flow = parser.parse(file.toFile(), Flow.class);
 
         GraphCluster graph = GraphUtils.of(flow, null);

diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java
@@ -3,7 +3,7 @@
 import io.kestra.cli.AbstractCommand;
 import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.validations.ModelValidator;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import jakarta.inject.Inject;
 import picocli.CommandLine;
 
@@ -21,7 +21,7 @@ public class FlowExpandCommand extends AbstractCommand {
     private Path file;
 
     @Inject
-    private YamlFlowParser yamlFlowParser;
+    private YamlParser yamlParser;
 
     @Inject
     private ModelValidator modelValidator;
@@ -31,7 +31,7 @@ public Integer call() throws Exception {
         super.call();
         stdErr("Warning, this functionality is deprecated and will be removed at some point.");
         String content = IncludeHelperExpander.expand(Files.readString(file), file.getParent());
-        Flow flow = yamlFlowParser.parse(content, Flow.class);
+        Flow flow = yamlParser.parse(content, Flow.class);
         modelValidator.validate(flow);
         stdOut(content);
         return 0;

diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/FlowUpdatesCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/FlowUpdatesCommand.java
@@ -2,7 +2,7 @@
 
 import io.kestra.cli.AbstractApiCommand;
 import io.kestra.cli.AbstractValidateCommand;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
 import io.micronaut.http.MediaType;
@@ -41,7 +41,7 @@ public Integer call() throws Exception {
         try (var files = Files.walk(directory)) {
             List<String> flows = files
                 .filter(Files::isRegularFile)
-                .filter(YamlFlowParser::isValidExtension)
+                .filter(YamlParser::isValidExtension)
                 .map(path -> {
                     try {
                         return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent());

diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/FlowValidateCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/FlowValidateCommand.java
@@ -3,7 +3,7 @@
 import io.kestra.cli.AbstractValidateCommand;
 import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.validations.ModelValidator;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.kestra.core.services.FlowService;
 import jakarta.inject.Inject;
 import picocli.CommandLine;
@@ -17,7 +17,7 @@
 )
 public class FlowValidateCommand extends AbstractValidateCommand {
     @Inject
-    private YamlFlowParser yamlFlowParser;
+    private YamlParser yamlParser;
 
     @Inject
     private ModelValidator modelValidator;
@@ -29,7 +29,7 @@ public class FlowValidateCommand extends AbstractValidateCommand {
     public Integer call() throws Exception {
         return this.call(
             Flow.class,
-            yamlFlowParser,
+            yamlParser,
             modelValidator,
             (Object object) -> {
                 Flow flow = (Flow) object;

diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java
@@ -3,7 +3,7 @@
 import io.kestra.cli.AbstractValidateCommand;
 import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
 import io.kestra.cli.commands.flows.IncludeHelperExpander;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
 import io.micronaut.http.MediaType;
@@ -28,7 +28,7 @@
 @Slf4j
 public class FlowNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand {
     @Inject
-    public YamlFlowParser yamlFlowParser;
+    public YamlParser yamlParser;
 
     @CommandLine.Option(names = {"--override-namespaces"}, negatable = true, description = "replace namespace of all flows by the one provided")
     public boolean override = false;
@@ -41,7 +41,7 @@ public Integer call() throws Exception {
         try (var files = Files.walk(directory)) {
             List<String> flows = files
                 .filter(Files::isRegularFile)
-                .filter(YamlFlowParser::isValidExtension)
+                .filter(YamlParser::isValidExtension)
                 .map(path -> {
                     try {
                         return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent());

diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java
@@ -4,7 +4,7 @@
 import io.kestra.core.models.templates.Template;
 import io.kestra.core.models.templates.TemplateEnabled;
 import io.kestra.core.models.validations.ModelValidator;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import jakarta.inject.Inject;
 import picocli.CommandLine;
 
@@ -17,7 +17,7 @@
 @TemplateEnabled
 public class TemplateValidateCommand extends AbstractValidateCommand {
     @Inject
-    private YamlFlowParser yamlFlowParser;
+    private YamlParser yamlParser;
 
     @Inject
     private ModelValidator modelValidator;
@@ -26,7 +26,7 @@ public class TemplateValidateCommand extends AbstractValidateCommand {
     public Integer call() throws Exception {
         return this.call(
             Template.class,
-            yamlFlowParser,
+            yamlParser,
             modelValidator,
             (Object object) -> {
                 Template template = (Template) object;

diff --git a/...main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java b/...main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java
@@ -2,10 +2,9 @@
 
 import io.kestra.cli.AbstractValidateCommand;
 import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
-import io.kestra.cli.commands.templates.TemplateValidateCommand;
 import io.kestra.core.models.templates.Template;
 import io.kestra.core.models.templates.TemplateEnabled;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
 import io.micronaut.http.MutableHttpRequest;
@@ -17,7 +16,7 @@
 
 import java.nio.file.Files;
 import java.util.List;
-import java.util.stream.Collectors;
+
 import jakarta.validation.ConstraintViolationException;
 
 @CommandLine.Command(
@@ -29,7 +28,7 @@
 @TemplateEnabled
 public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand {
     @Inject
-    public YamlFlowParser yamlFlowParser;
+    public YamlParser yamlParser;
 
     @Override
     public Integer call() throws Exception {
@@ -38,8 +37,8 @@ public Integer call() throws Exception {
         try (var files = Files.walk(directory)) {
             List<Template> templates = files
                 .filter(Files::isRegularFile)
-                .filter(YamlFlowParser::isValidExtension)
-                .map(path -> yamlFlowParser.parse(path.toFile(), Template.class))
+                .filter(YamlParser::isValidExtension)
+                .map(path -> yamlParser.parse(path.toFile(), Template.class))
                 .toList();
 
             if (templates.isEmpty()) {

diff --git a/cli/src/main/java/io/kestra/cli/services/FileChangedEventListener.java b/cli/src/main/java/io/kestra/cli/services/FileChangedEventListener.java
@@ -5,7 +5,7 @@
 import io.kestra.core.models.flows.FlowWithSource;
 import io.kestra.core.models.validations.ModelValidator;
 import io.kestra.core.repositories.FlowRepositoryInterface;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.kestra.core.services.FlowListenersInterface;
 import io.micronaut.context.annotation.Requires;
 import io.micronaut.context.annotation.Value;
@@ -37,7 +37,7 @@ public class FileChangedEventListener {
     private FlowRepositoryInterface flowRepositoryInterface;
 
     @Inject
-    private YamlFlowParser yamlFlowParser;
+    private YamlParser yamlParser;
 
     @Inject
     private ModelValidator modelValidator;
@@ -229,7 +229,7 @@ private void flowToFile(FlowWithSource flow, Path path) {
 
     private Optional<Flow> parseFlow(String content, Path entry) {
         try {
-            Flow flow = yamlFlowParser.parse(content, Flow.class);
+            Flow flow = yamlParser.parse(content, Flow.class);
             modelValidator.validate(flow);
             return Optional.of(flow);
         } catch (ConstraintViolationException e) {

diff --git a/core/src/main/java/io/kestra/core/repositories/LocalFlowRepositoryLoader.java b/core/src/main/java/io/kestra/core/repositories/LocalFlowRepositoryLoader.java
@@ -2,7 +2,7 @@
 
 import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.validations.ModelValidator;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.kestra.core.services.PluginDefaultService;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
@@ -19,7 +19,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -31,7 +30,7 @@
 @Slf4j
 public class LocalFlowRepositoryLoader {
     @Inject
-    private YamlFlowParser yamlFlowParser;
+    private YamlParser yamlParser;
 
     @Inject
     private FlowRepositoryInterface flowRepository;
@@ -72,13 +71,13 @@ public void load(File basePath) throws IOException {
         Map<String, Flow> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
             .collect(Collectors.toMap(Flow::uidWithoutRevision, Function.identity()));
         List<Path> list = Files.walk(basePath.toPath())
-            .filter(YamlFlowParser::isValidExtension)
+            .filter(YamlParser::isValidExtension)
             .toList();
 
         for (Path file : list) {
             try {
                 String flowSource = Files.readString(Path.of(file.toFile().getPath()), Charset.defaultCharset());
-                Flow parse = yamlFlowParser.parse(file.toFile(), Flow.class);
+                Flow parse = yamlParser.parse(file.toFile(), Flow.class);
                 modelValidator.validate(parse);
 
                 Flow inRepository = flowByUidInRepository.get(parse.uidWithoutRevision());

diff --git a/...stra/core/serializers/YamlFlowParser.java → ...o/kestra/core/serializers/YamlParser.java b/...stra/core/serializers/YamlFlowParser.java → ...o/kestra/core/serializers/YamlParser.java
@@ -21,7 +21,7 @@
 import java.util.Set;
 
 @Singleton
-public class YamlFlowParser {
+public class YamlParser {
     private static final ObjectMapper STRICT_MAPPER = JacksonMapper.ofYaml()
         .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
         .disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE);
@@ -34,7 +34,7 @@ public static boolean isValidExtension(Path path) {
     }
 
     public <T> T parse(String input, Class<T> cls) {
-        return readFlow(input, cls, type(cls));
+        return read(input, cls, type(cls));
     }
 
 
@@ -59,7 +59,7 @@ private static <T> String type(Class<T> cls) {
     public <T> T parse(File file, Class<T> cls) throws ConstraintViolationException {
         try {
             String input = IOUtils.toString(file.toURI(), StandardCharsets.UTF_8);
-            return readFlow(input, cls, type(cls));
+            return read(input, cls, type(cls));
 
         } catch (IOException e) {
             throw new ConstraintViolationException(
@@ -77,7 +77,7 @@ public <T> T parse(File file, Class<T> cls) throws ConstraintViolationException
         }
     }
 
-    private <T> T readFlow(String input, Class<T> objectClass, String resource) {
+    private <T> T read(String input, Class<T> objectClass, String resource) {
         try {
             return STRICT_MAPPER.readValue(input, objectClass);
         } catch (JsonProcessingException e) {
@@ -91,8 +91,7 @@ private <T> T readFlow(String input, Class<T> objectClass, String resource) {
     private static <T> void jsonProcessingExceptionHandler(T target, String resource, JsonProcessingException e) throws ConstraintViolationException {
         if (e.getCause() instanceof ConstraintViolationException constraintViolationException) {
             throw constraintViolationException;
-        }
-        else if (e instanceof InvalidTypeIdException invalidTypeIdException) {
+        } else if (e instanceof InvalidTypeIdException invalidTypeIdException) {
             // This error is thrown when a non-existing task is used
             throw new ConstraintViolationException(
                 "Invalid type: " + invalidTypeIdException.getTypeId(),
@@ -113,8 +112,7 @@ else if (e instanceof InvalidTypeIdException invalidTypeIdException) {
                     )
                 )
             );
-        }
-        else if (e instanceof UnrecognizedPropertyException unrecognizedPropertyException) {
+        } else if (e instanceof UnrecognizedPropertyException unrecognizedPropertyException) {
             var message = unrecognizedPropertyException.getOriginalMessage() + unrecognizedPropertyException.getMessageSuffix();
             throw new ConstraintViolationException(
                 message,
@@ -127,16 +125,15 @@ else if (e instanceof UnrecognizedPropertyException unrecognizedPropertyExceptio
                         null
                     )
                 ));
-        }
-        else {
+        } else {
             throw new ConstraintViolationException(
                 "Illegal "+ resource +" yaml: " + e.getMessage(),
                 Collections.singleton(
                     ManualConstraintViolation.of(
                         e.getCause() == null ? e.getMessage() : e.getMessage() + "\nCaused by: " + e.getCause().getMessage(),
                         target,
                         (Class<T>) target.getClass(),
-                        "flow",
+                        "yaml",
                         null
                     )
                 )

diff --git a/core/src/main/java/io/kestra/core/services/FlowService.java b/core/src/main/java/io/kestra/core/services/FlowService.java
@@ -11,7 +11,7 @@
 import io.kestra.core.plugins.PluginRegistry;
 import io.kestra.core.repositories.FlowRepositoryInterface;
 import io.kestra.core.serializers.JacksonMapper;
-import io.kestra.core.serializers.YamlFlowParser;
+import io.kestra.core.serializers.YamlParser;
 import io.kestra.core.utils.ListUtils;
 import io.micronaut.core.annotation.Nullable;
 import jakarta.inject.Inject;
@@ -54,7 +54,7 @@ public class FlowService {
     Optional<FlowRepositoryInterface> flowRepository;
 
     @Inject
-    YamlFlowParser yamlFlowParser;
+    YamlParser yamlParser;
 
     @Inject
     PluginDefaultService pluginDefaultService;
@@ -71,7 +71,7 @@ public FlowWithSource importFlow(String tenantId, String source, boolean dryRun)
             throw noRepositoryException();
         }
 
-        FlowWithSource withTenant = yamlFlowParser.parse(source, Flow.class).toBuilder()
+        FlowWithSource withTenant = yamlParser.parse(source, Flow.class).toBuilder()
             .tenantId(tenantId)
             .build()
             .withSource(source);