[Online Scoring] Automation Rules endpoints

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRule.java

@@ -0,0 +1,52 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonValue;
+import io.swagger.v3.oas.annotations.media.DiscriminatorMapping;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.UUID;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "action", visible = true)
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = AutomationRuleEvaluator.class, name = "evaluator")
+})
+@Schema(name = "AutomationRule", discriminatorProperty = "action", discriminatorMapping = {
+        @DiscriminatorMapping(value = "evaluator", schema = AutomationRuleEvaluator.class)
+})
+public sealed interface AutomationRule<T> permits AutomationRuleEvaluator {
+
+    UUID getId();
+    UUID getProjectId();
+    String getName();
+
+    AutomationRuleAction getAction();
+    Float getSamplingRate();
+
+    Instant getCreatedAt();
+    String getCreatedBy();
+    Instant getLastUpdatedAt();
+    String getLastUpdatedBy();
+
+    @Getter
+    @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+    enum AutomationRuleAction {
+
+        EVALUATOR("evaluator");
+
+        @JsonValue
+        private final String action;
+
+        public static AutomationRule.AutomationRuleAction fromString(String action) {
+            return Arrays.stream(values())
+                    .filter(v -> v.action.equals(action)).findFirst()
+                    .orElseThrow(() -> new IllegalArgumentException("Unknown rule type: " + action));
+        }
+    }
+}

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluator.java

@@ -0,0 +1,119 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonView;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.swagger.v3.oas.annotations.media.DiscriminatorMapping;
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import lombok.experimental.SuperBuilder;
+
+import java.beans.ConstructorProperties;
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+
+@Data
+@SuperBuilder(toBuilder = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type", visible = true)
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge.class, name = "llm_as_judge")
+})
+@Schema(name = "AutomationRuleEvaluator", discriminatorProperty = "type", discriminatorMapping = {
+        @DiscriminatorMapping(value = "llm_as_judge", schema = AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge.class)
+})
+@AllArgsConstructor
+public abstract sealed class AutomationRuleEvaluator<T> implements AutomationRule<T> {
+
+    @EqualsAndHashCode(callSuper = true)
+    @Data
+    @SuperBuilder(toBuilder = true)
+    @ToString(callSuper = true)
+    public static final class AutomationRuleEvaluatorLlmAsJudge extends AutomationRuleEvaluator<JsonNode> {
+
+        @NotNull @JsonView({View.Public.class, View.Write.class})
+        @Schema(accessMode = Schema.AccessMode.READ_WRITE)
+        JsonNode code;
+
+        @ConstructorProperties({"id", "projectId", "name", "samplingRate", "code", "createdAt", "createdBy", "lastUpdatedAt", "lastUpdatedBy"})
+        public AutomationRuleEvaluatorLlmAsJudge(UUID id, UUID projectId, @NotBlank String name, float samplingRate, @NotNull JsonNode code,
+                                           Instant createdAt, String createdBy, Instant lastUpdatedAt, String lastUpdatedBy) {
+            super(id, projectId, name, samplingRate, createdAt, createdBy, lastUpdatedAt, lastUpdatedBy);
+            this.code = code;
+        }
+
+        @Override
+        public AutomationRuleEvaluatorType type() {
+            return AutomationRuleEvaluatorType.LLM_AS_JUDGE;
+        }
+    }
+
+    @JsonView({View.Public.class})
+    @Schema(accessMode = Schema.AccessMode.READ_ONLY)
+    UUID id;
+
+    @JsonView({View.Public.class, View.Write.class})
+    @NotNull
+    UUID projectId;
+
+    @JsonView({View.Public.class, View.Write.class})
+    @Schema(accessMode = Schema.AccessMode.READ_WRITE)
+    @NotBlank
+    String name;
+
+    @JsonView({View.Public.class, View.Write.class})
+    @Schema(accessMode = Schema.AccessMode.READ_WRITE)
+    Float samplingRate;
+
+    @JsonView({View.Public.class})
+    @Schema(accessMode = Schema.AccessMode.READ_ONLY)
+    Instant createdAt;
+
+    @JsonView({View.Public.class})
+    @Schema(accessMode = Schema.AccessMode.READ_ONLY)
+    String createdBy;
+
+    @JsonView({View.Public.class})
+    @Schema(accessMode = Schema.AccessMode.READ_ONLY)
+    Instant lastUpdatedAt;
+
+    @JsonView({View.Public.class})
+    @Schema(accessMode = Schema.AccessMode.READ_ONLY)
+    String lastUpdatedBy;
+
+    @JsonView({View.Public.class})
+    public abstract AutomationRuleEvaluatorType type();
+
+    @JsonView({View.Public.class, View.Write.class})
+    public abstract T getCode();
+
+    @Override
+    public AutomationRuleAction getAction() {
+        return AutomationRuleAction.EVALUATOR;
+    }
+
+    public static class View {
+        public static class Write {}
+        public static class Public {}
+    }
+
+    @Builder(toBuilder = true)
+    public record AutomationRuleEvaluatorPage(
+            @JsonView({View.Public.class}) int page,
+            @JsonView({View.Public.class}) int size,
+            @JsonView({View.Public.class}) long total,
+            @JsonView({View.Public.class}) List<AutomationRuleEvaluatorLlmAsJudge> content)
+            implements Page<AutomationRuleEvaluatorLlmAsJudge>{
+
+        public static AutomationRuleEvaluator.AutomationRuleEvaluatorPage empty(int page) {
+            return new AutomationRuleEvaluator.AutomationRuleEvaluatorPage(page, 0, 0, List.of());
+        }
+    }
+}

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluatorType.java

@@ -0,0 +1,24 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+import java.util.Arrays;
+
+@Getter
+@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+public enum AutomationRuleEvaluatorType {
+
+    LLM_AS_JUDGE("llm_as_judge");
+
+    @JsonValue
+    private final String type;
+
+    public static AutomationRuleEvaluatorType fromString(String type) {
+        return Arrays.stream(values())
+                .filter(v -> v.type.equals(type)).findFirst()
+                .orElseThrow(() -> new IllegalArgumentException("Unknown evaluator type: " + type));
+    }
+}

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluatorUpdate.java

@@ -0,0 +1,17 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import jakarta.validation.constraints.NotNull;
+import lombok.Builder;
+
+@Builder(toBuilder = true)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+public record AutomationRuleEvaluatorUpdate(
+        @NotNull String name,
+        @NotNull JsonNode code,
+        @NotNull Float samplingRate) {
+}

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/AutomationRuleEvaluatorsResource.java

@@ -0,0 +1,161 @@
+package com.comet.opik.api.resources.v1.priv;
+
+import com.codahale.metrics.annotation.Timed;
+import com.comet.opik.api.AutomationRuleEvaluator;
+import com.comet.opik.api.AutomationRuleEvaluatorUpdate;
+import com.comet.opik.api.BatchDelete;
+import com.comet.opik.api.Page;
+import com.comet.opik.domain.AutomationRuleEvaluatorService;
+import com.comet.opik.infrastructure.auth.RequestContext;
+import com.comet.opik.infrastructure.ratelimit.RateLimited;
+import com.fasterxml.jackson.annotation.JsonView;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.headers.Header;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.inject.Inject;
+import jakarta.inject.Provider;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.Min;
+import jakarta.validation.constraints.NotNull;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.DefaultValue;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.PATCH;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.UriInfo;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.URI;
+import java.util.UUID;
+
+@Path("/v1/private/automations/projects/{projectId}/evaluators/")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+@Timed
+@Slf4j
+@RequiredArgsConstructor(onConstructor_ = @Inject)
+@Tag(name = "Automation rule evaluators", description = "Automation rule evaluators resource")
+public class AutomationRuleEvaluatorsResource {
+
+    private final @NonNull AutomationRuleEvaluatorService service;
+    private final @NonNull Provider<RequestContext> requestContext;
+
+    @GET
+    @Operation(operationId = "findEvaluators", summary = "Find project Evaluators", description = "Find project Evaluators", responses = {
+            @ApiResponse(responseCode = "200", description = "Evaluators resource", content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.AutomationRuleEvaluatorPage.class)))
+    })
+    @JsonView(AutomationRuleEvaluator.View.Public.class)
+    public Response find(@PathParam("projectId") UUID projectId,
+                         @QueryParam("name") String name,
+                         @QueryParam("page") @Min(1) @DefaultValue("1") int page,
+                         @QueryParam("size") @Min(1) @DefaultValue("10") int size) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        log.info("Looking for automated evaluators for project id '{}' on workspaceId '{}' (page {})", projectId,
+                workspaceId, page);
+        Page<AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge> definitionPage = service.find(projectId, workspaceId, name, page, size);
+        log.info("Found {} automated evaluators for project id '{}' on workspaceId '{}' (page {}, total {})",
+                definitionPage.size(), projectId, workspaceId, page, definitionPage.total());
+
+        return Response.ok()
+                .entity(definitionPage)
+                .build();
+    }
+
+    @GET
+    @Path("/{id}")
+    @Operation(operationId = "getEvaluatorById", summary = "Get automation rule evaluator by id", description = "Get automation rule by id", responses = {
+            @ApiResponse(responseCode = "200", description = "Automation Rule resource", content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.class)))
+    })
+    @JsonView(AutomationRuleEvaluator.View.Public.class)
+    public Response getEvaluator(@PathParam("projectId") UUID projectId, @PathParam("id") UUID evaluatorId) {
+        String workspaceId = requestContext.get().getWorkspaceId();
+
+        log.info("Looking for automated evaluator: id '{}' on project_id '{}'", projectId, workspaceId);
+        AutomationRuleEvaluator evaluator = service.findById(evaluatorId, projectId, workspaceId);
+        log.info("Found automated evaluator: id '{}' on project_id '{}'", projectId, workspaceId);
+
+        return Response.ok().entity(evaluator).build();
+    }
+
+    @POST
+    @Operation(operationId = "createAutomationRuleEvaluator", summary = "Create automation rule evaluator", description = "Create automation rule evaluator", responses = {
+            @ApiResponse(responseCode = "201", description = "Created", headers = {
+                    @Header(name = "Location", required = true, example = "${basePath}/v1/private/automations/projects/{projectId}/evaluators/{evaluatorId}", schema = @Schema(implementation = String.class))
+            })
+    })
+    @RateLimited
+    public Response createEvaluator(
+            @RequestBody(content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.class)))
+            @JsonView(AutomationRuleEvaluator.View.Write.class) @NotNull @Valid AutomationRuleEvaluator<?> evaluator,
+            @Context UriInfo uriInfo) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        String userName = requestContext.get().getUserName();
+
+        log.info("Creating {} evaluator for project_id '{}' on workspace_id '{}'", evaluator.type(),
+                evaluator.getProjectId(), workspaceId);
+        AutomationRuleEvaluator<?> savedEvaluator = service.save(evaluator, workspaceId, userName);
+        log.info("Created {} evaluator '{}' for project_id '{}' on workspace_id '{}'", evaluator.type(),
+                savedEvaluator.getId(), evaluator.getProjectId(), workspaceId);
+
+        URI uri = uriInfo.getBaseUriBuilder()
+                .path("v1/private/automations/projects/{projectId}/evaluators/{id}")
+                .resolveTemplate("projectId", savedEvaluator.getProjectId().toString())
+                .resolveTemplate("id", savedEvaluator.getId().toString())
+                .build();
+        return Response.created(uri).build();
+    }
+
+    @PATCH
+    @Path("/{id}")
+    @Operation(operationId = "updateAutomationRuleEvaluator", summary = "update Automation Rule Evaluator by id", description = "update Automation Rule Evaluator by id", responses = {
+            @ApiResponse(responseCode = "204", description = "No content"),
+    })
+    @RateLimited
+    public Response updateEvaluator(@PathParam("id") UUID id,
+            @PathParam("projectId") UUID projectId,
+            @RequestBody(content = @Content(schema = @Schema(implementation = AutomationRuleEvaluatorUpdate.class))) @NotNull @Valid AutomationRuleEvaluatorUpdate evaluatorUpdate) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        String userName = requestContext.get().getUserName();
+
+        log.info("Updating automation rule evaluator by id '{}' and project_id '{}' on workspace_id '{}'", id,
+                projectId, workspaceId);
+        service.update(id, projectId, workspaceId, userName, evaluatorUpdate);
+        log.info("Updated automation rule evaluator by id '{}' and project_id '{}' on workspace_id '{}'", id, projectId,
+                workspaceId);
+
+        return Response.noContent().build();
+    }
+
+    @POST
+    @Path("/delete")
+    @Operation(operationId = "deleteAutomationRuleEvaluatorBatch", summary = "Delete automation rule evaluators", description = "Delete automation rule evaluators batch", responses = {
+            @ApiResponse(responseCode = "204", description = "No Content"),
+    })
+    public Response deleteEvaluators(
+            @NotNull @RequestBody(content = @Content(schema = @Schema(implementation = BatchDelete.class))) @Valid BatchDelete batchDelete, @PathParam("projectId") UUID projectId) {
+        String workspaceId = requestContext.get().getWorkspaceId();
+        log.info("Deleting automation rule evaluators by ids, count '{}', on workspace_id '{}'", batchDelete.ids().size(),
+                workspaceId);
+        service.delete(batchDelete.ids(), projectId, workspaceId);
+        log.info("Deleted automation rule evaluators by ids, count '{}', on workspace_id '{}'", batchDelete.ids().size(),
+                workspaceId);
+        return Response.noContent().build();
+    }
+
+}

apps/opik-backend/src/main/java/com/comet/opik/domain/AutomationModelEvaluatorMapper.java

@@ -0,0 +1,18 @@
+package com.comet.opik.domain;
+
+import com.comet.opik.api.AutomationRuleEvaluator;
+import org.mapstruct.Mapper;
+import org.mapstruct.factory.Mappers;
+
+import java.time.Instant;
+
+@Mapper(imports = Instant.class)
+interface AutomationModelEvaluatorMapper {
+
+    AutomationModelEvaluatorMapper INSTANCE = Mappers.getMapper(AutomationModelEvaluatorMapper.class);
+
+    AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge map(LlmAsJudgeAutomationRuleEvaluatorModel model);
+
+    LlmAsJudgeAutomationRuleEvaluatorModel map(AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge dto);
+
+}