[OPIK-590] Online Scoring Automation Rules endpoints

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluator.java

@@ -0,0 +1,55 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonView;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.Pattern;
+import lombok.Builder;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+
+import static com.comet.opik.utils.ValidationUtils.NULL_OR_NOT_BLANK;
+
+@Builder(toBuilder = true)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+public record AutomationRuleEvaluator(
+        // Fields and methods
+        @JsonView({View.Public.class, View.Write.class}) UUID id,
+        @JsonView({View.Public.class, View.Write.class}) UUID projectId,
+        @JsonView({View.Public.class, View.Write.class}) AutomationRuleEvaluatorType evaluatorType,
+
+        @JsonView({View.Public.class,
+                View.Write.class}) @Pattern(regexp = NULL_OR_NOT_BLANK, message = "must not be blank") String code,
+
+        @JsonView({View.Public.class, View.Write.class}) float samplingRate,
+
+        @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) Instant createdAt,
+        @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) String createdBy,
+        @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) Instant lastUpdatedAt,
+        @JsonView({View.Public.class}) @Schema(accessMode = Schema.AccessMode.READ_ONLY) String lastUpdatedBy){
+
+    public static class View {
+        public static class Write {}
+        public static class Public {}
+    }
+
+    @Builder(toBuilder = true)
+    public record AutomationRuleEvaluatorPage(
+            @JsonView( {
+                    View.Public.class}) int page,
+            @JsonView({View.Public.class}) int size,
+            @JsonView({View.Public.class}) long total,
+            @JsonView({View.Public.class}) List<AutomationRuleEvaluator> content)
+            implements
+                Page<AutomationRuleEvaluator>{
+
+        public static AutomationRuleEvaluator.AutomationRuleEvaluatorPage empty(int page) {
+            return new AutomationRuleEvaluator.AutomationRuleEvaluatorPage(page, 0, 0, List.of());
+        }
+    }
+}

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluatorType.java

@@ -0,0 +1,17 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@Getter
+@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+public enum AutomationRuleEvaluatorType {
+
+    LLM_AS_JUDGE("llm-as-judge"),
+    PYTHON("python");
+
+    @JsonValue
+    private final String type;
+}

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluatorUpdate.java

@@ -0,0 +1,15 @@
+package com.comet.opik.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import jakarta.validation.constraints.NotBlank;
+import lombok.Builder;
+
+@Builder(toBuilder = true)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+public record AutomationRuleEvaluatorUpdate(
+        @NotBlank String code,
+        float samplingRate) {
+}

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/AutomationRuleEvaluatorsResource.java

@@ -0,0 +1,167 @@
+package com.comet.opik.api.resources.v1.priv;
+
+import com.codahale.metrics.annotation.Timed;
+import com.comet.opik.api.AutomationRuleEvaluator;
+import com.comet.opik.api.AutomationRuleEvaluatorUpdate;
+import com.comet.opik.api.Page;
+import com.comet.opik.domain.AutomationRuleService;
+import com.comet.opik.infrastructure.auth.RequestContext;
+import com.comet.opik.infrastructure.ratelimit.RateLimited;
+import com.fasterxml.jackson.annotation.JsonView;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.headers.Header;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.inject.Inject;
+import jakarta.inject.Provider;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.Min;
+import jakarta.validation.constraints.NotNull;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.DefaultValue;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.UriInfo;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.URI;
+import java.util.UUID;
+
+@Path("/v1/private/automation/evaluator")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+@Timed
+@Slf4j
+@RequiredArgsConstructor(onConstructor_ = @Inject)
+@Tag(name = "Automation rule evaluators", description = "Automation rule evaluators resource")
+public class AutomationRuleEvaluatorsResource {
+
+    private final @NonNull AutomationRuleService service;
+    private final @NonNull Provider<RequestContext> requestContext;
+
+    @GET
+    @Path("/projectId/{projectId}")
+    @Operation(operationId = "findEvaluators", summary = "Find Evaluators", description = "Find Evaluators", responses = {
+            @ApiResponse(responseCode = "200", description = "Evaluators resource", content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.AutomationRuleEvaluatorPage.class)))
+    })
+    @JsonView(AutomationRuleEvaluator.View.Public.class)
+    public Response find(@PathParam("projectId") UUID projectId,
+            @QueryParam("page") @Min(1) @DefaultValue("1") int page,
+            @QueryParam("size") @Min(1) @DefaultValue("10") int size) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        log.info("Looking for automated evaluators for project id '{}' on workspaceId '{}' (page {})", projectId,
+                workspaceId, page);
+        Page<AutomationRuleEvaluator> definitionPage = service.find(page, size, projectId);
+        log.info("Found {} automated evaluators for project id '{}' on workspaceId '{}' (page {}, total {})",
+                definitionPage.size(), projectId, workspaceId, page, definitionPage.total());
+
+        return Response.ok()
+                .entity(definitionPage)
+                .build();
+    }
+
+    @GET
+    @Path("/projectId/{projectId}/evaluatorId/{evaluatorId}")
+    @Operation(operationId = "getAutomationRulesByProjectId", summary = "Get automation rule evaluator by id", description = "Get dataset by id", responses = {
+            @ApiResponse(responseCode = "200", description = "Automation Rule resource", content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.class)))
+    })
+    @JsonView(AutomationRuleEvaluator.View.Public.class)
+    public Response getEvaluator(@PathParam("projectId") UUID projectId, @PathParam("evaluatorId") UUID evaluatorId) {
+        String workspaceId = requestContext.get().getWorkspaceId();
+
+        log.info("Finding automated evaluators by id '{}' on project_id '{}'", projectId, workspaceId);
+        AutomationRuleEvaluator evaluator = service.findById(evaluatorId, projectId, workspaceId);
+        log.info("Found automated evaluators by id '{}' on project_id '{}'", projectId, workspaceId);
+
+        return Response.ok().entity(evaluator).build();
+    }
+
+    @POST
+    @Operation(operationId = "createAutomationRuleEvaluator", summary = "Create automation rule evaluator", description = "Create automation rule evaluator", responses = {
+            @ApiResponse(responseCode = "201", description = "Created", headers = {
+                    @Header(name = "Location", required = true, example = "${basePath}/api/v1/private/automation/evaluator/projectId/{projectId}/evaluatorId/{evaluatorId}", schema = @Schema(implementation = String.class))
+            })
+    })
+    @RateLimited
+    public Response createEvaluator(
+            @RequestBody(content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.class))) @JsonView(AutomationRuleEvaluator.View.Write.class) @NotNull @Valid AutomationRuleEvaluator evaluator,
+            @Context UriInfo uriInfo) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+
+        log.info("Creating {} evaluator for project_id '{}' on workspace_id '{}'", evaluator.evaluatorType(),
+                evaluator.projectId(), workspaceId);
+        AutomationRuleEvaluator savedEvaluator = service.save(evaluator, workspaceId);
+        log.info("Created {} evaluator '{}' for project_id '{}' on workspace_id '{}'", evaluator.evaluatorType(),
+                savedEvaluator.id(), evaluator.projectId(), workspaceId);
+
+        URI uri = uriInfo.getAbsolutePathBuilder()
+                .path("/projectId/%s/evaluatorId/%s".formatted(savedEvaluator.projectId().toString(),
+                        savedEvaluator.id().toString()))
+                .build();
+        return Response.created(uri).build();
+    }
+
+    @PUT
+    @Path("/projectId/{projectId}/evaluatorId/{id}")
+    @Operation(operationId = "updateAutomationRuleEvaluator", summary = "update Automation Rule Evaluator by id", description = "update Automation Rule Evaluator by id", responses = {
+            @ApiResponse(responseCode = "204", description = "No content"),
+    })
+    @RateLimited
+    public Response updateEvaluator(@PathParam("id") UUID id,
+            @PathParam("projectId") UUID projectId,
+            @RequestBody(content = @Content(schema = @Schema(implementation = AutomationRuleEvaluatorUpdate.class))) @NotNull @Valid AutomationRuleEvaluatorUpdate evaluatorUpdate) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        log.info("Updating automation rule evaluator by id '{}' and project_id '{}' on workspace_id '{}'", id,
+                projectId, workspaceId);
+        service.update(id, projectId, workspaceId, evaluatorUpdate);
+        log.info("Updated automation rule evaluator by id '{}' and project_id '{}' on workspace_id '{}'", id, projectId,
+                workspaceId);
+
+        return Response.noContent().build();
+    }
+
+    @DELETE
+    @Path("/projectId/{projectId}/evaluatorId/{id}")
+    @Operation(operationId = "deleteAutomationRuleEvaluatorById", summary = "Delete Automation Rule Evaluator by id", description = "Delete Automation Rule Evaluator by id", responses = {
+            @ApiResponse(responseCode = "204", description = "No content"),
+    })
+    public Response deleteEvaluator(@PathParam("id") UUID id, @PathParam("projectId") UUID projectId) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        log.info("Deleting dataset by id '{}' on workspace_id '{}'", id, workspaceId);
+        service.delete(id, projectId, workspaceId);
+        log.info("Deleted dataset by id '{}' on workspace_id '{}'", id, workspaceId);
+        return Response.noContent().build();
+    }
+
+    @DELETE
+    @Path("/projectId/{projectId}")
+    @Operation(operationId = "deleteAutomationRuleEvaluatorByProject", summary = "Delete Automation Rule Evaluator by Project id", description = "Delete Automation Rule Evaluator by Project id", responses = {
+            @ApiResponse(responseCode = "204", description = "No content"),
+    })
+    public Response deleteProjectEvaluators(@PathParam("projectId") UUID projectId) {
+
+        String workspaceId = requestContext.get().getWorkspaceId();
+        log.info("Deleting evaluators from project_id '{}' on workspace_id '{}'", projectId, workspaceId);
+        service.deleteByProject(projectId, workspaceId);
+        log.info("Deleted evaluators from project_id '{}' on workspace_id '{}'", projectId, workspaceId);
+        return Response.noContent().build();
+    }
+}

apps/opik-backend/src/main/java/com/comet/opik/domain/AutomationRuleDAO.java

@@ -0,0 +1,79 @@
+package com.comet.opik.domain;
+
+import com.comet.opik.api.AutomationRuleEvaluator;
+import com.comet.opik.api.AutomationRuleEvaluatorUpdate;
+import com.comet.opik.infrastructure.db.UUIDArgumentFactory;
+import org.jdbi.v3.sqlobject.config.RegisterArgumentFactory;
+import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper;
+import org.jdbi.v3.sqlobject.customizer.AllowUnusedBindings;
+import org.jdbi.v3.sqlobject.customizer.Bind;
+import org.jdbi.v3.sqlobject.customizer.BindList;
+import org.jdbi.v3.sqlobject.customizer.BindMethods;
+import org.jdbi.v3.sqlobject.statement.SqlQuery;
+import org.jdbi.v3.sqlobject.statement.SqlUpdate;
+import org.jdbi.v3.stringtemplate4.UseStringTemplateEngine;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+
+@RegisterArgumentFactory(UUIDArgumentFactory.class)
+@RegisterConstructorMapper(AutomationRuleEvaluator.class)
+public interface AutomationRuleDAO {
+
+    @SqlUpdate("INSERT INTO automation_rule_evaluators(id, project_id, workspace_id, evaluator_type, sampling_rate, code, created_by, last_updated_by) "+
+               "VALUES (:rule.id, :rule.projectId, :workspaceId, :rule.evaluatorType, :rule.samplingRate, :rule.code, :rule.createdBy, :rule.lastUpdatedBy)")
+    void save(@BindMethods("rule") AutomationRuleEvaluator rule, @Bind("workspaceId") String workspaceId);
+
+    @SqlUpdate("""
+            UPDATE automation_rule_evaluators
+            SET
+                sampling_rate = :rule.samplingRate,
+                code = :rule.code,
+                last_updated_by = :lastUpdatedBy
+            WHERE id = :id AND project_id = :projectId AND workspace_id = :workspaceId
+            """)
+    int update(@Bind("id") UUID id,
+            @Bind("projectId") UUID projectId,
+            @Bind("workspaceId") String workspaceId,
+            @BindMethods("rule") AutomationRuleEvaluatorUpdate ruleUpdate,
+            @Bind("lastUpdatedBy") String lastUpdatedBy);
+
+    @SqlQuery("SELECT * FROM automation_rule_evaluators WHERE id = :id AND project_id = :projectId AND workspace_id = :workspaceId")
+    Optional<AutomationRuleEvaluator> findById(@Bind("id") UUID id, @Bind("projectId") UUID projectId,
+            @Bind("workspaceId") String workspaceId);
+
+    @SqlQuery("SELECT * FROM automation_rule_evaluators WHERE id IN (<ids>) AND project_id = :projectId AND workspace_id = :workspaceId")
+    List<AutomationRuleEvaluator> findByIds(@BindList("ids") Set<UUID> ids, @Bind("projectId") UUID projectId,
+            @Bind("workspaceId") String workspaceId);
+
+    @SqlQuery("SELECT * FROM automation_rule_evaluators WHERE project_id = :projectId AND workspace_id = :workspaceId")
+    List<AutomationRuleEvaluator> findByProjectId(@Bind("projectId") UUID projectId, @Bind("workspaceId") String workspaceId);
+
+    @SqlUpdate("DELETE FROM automation_rule_evaluators WHERE id = :id AND project_id = :projectId AND workspace_id = :workspaceId")
+    void delete(@Bind("id") UUID id, @Bind("projectId") UUID projectId, @Bind("workspaceId") String workspaceId);
+
+    @SqlUpdate("DELETE FROM automation_rule_evaluators WHERE project_id = :projectId AND workspace_id = :workspaceId")
+    void deleteByProject(@Bind("projectId") UUID projectId, @Bind("workspaceId") String workspaceId);
+
+    @SqlUpdate("DELETE FROM automation_rule_evaluators WHERE id IN (<ids>) AND project_id = :projectId AND workspace_id = :workspaceId")
+    void delete(@BindList("ids") Set<UUID> ids, @Bind("projectId") UUID projectId, @Bind("workspaceId") String workspaceId);
+
+    @SqlQuery("SELECT * FROM automation_rule_evaluators " +
+            " WHERE project_id = :projectId AND workspace_id = :workspaceId " +
+            " LIMIT :limit OFFSET :offset ")
+    @UseStringTemplateEngine
+    @AllowUnusedBindings
+    List<AutomationRuleEvaluator> find(@Bind("limit") int limit,
+            @Bind("offset") int offset,
+            @Bind("projectId") UUID projectId,
+            @Bind("workspaceId") String workspaceId);
+
+    @SqlQuery("SELECT COUNT(*) FROM automation_rule_evaluators WHERE project_id = :projectId AND workspace_id = :workspaceId")
+    long findCount(@Bind("projectId") UUID projectId, @Bind("workspaceId") String workspaceId);
+
+    @SqlQuery("SELECT id FROM automation_rule_evaluators WHERE id IN (<ids>) and project_id = :projectId AND workspace_id = :workspaceId")
+    Set<UUID> exists(@BindList("ids") Set<UUID> ruleIds, @Bind("projectId") UUID projectId,
+            @Bind("workspaceId") String workspaceId);
+}