feat(sdk): add synonym field mapping for experiment evaluator inputs

Author: lenatraceloop

packages/traceloop-sdk/src/lib/client/experiment/experiment.ts

@@ -6,6 +6,10 @@ import {
   validateAndNormalizeTaskOutput,
   type EvaluatorWithRequiredFields,
 } from "../evaluator/field-mapping";
+import {
+  EVALUATOR_SCHEMAS,
+  isValidEvaluatorSlug,
+} from "../../generated/evaluators/registry";
 import type {
   ExperimentTaskFunction,
   ExperimentRunOptions,
@@ -423,12 +427,23 @@ export class Experiment {
   private getEvaluatorsForFieldsValidation(
     evaluators: EvaluatorDetails[],
   ): EvaluatorWithRequiredFields[] {
-    return evaluators.filter(
-      (e): e is EvaluatorWithConfig =>
-        typeof e === "object" &&
-        Array.isArray((e as EvaluatorWithConfig).requiredInputFields) &&
-        ((e as EvaluatorWithConfig).requiredInputFields as string[]).length > 0,
-    );
+    return evaluators
+      .filter((e) => typeof e === "object")
+      .map((e) => {
+        const evaluator = e as EvaluatorWithConfig;
+        const schema = isValidEvaluatorSlug(evaluator.name)
+          ? EVALUATOR_SCHEMAS[evaluator.name]
+          : undefined;
+        return {
+          name: evaluator.name,
+          requiredInputFields: schema?.requiredInputFields,
+        };
+      })
+      .filter(
+        (e): e is { name: string; requiredInputFields: string[] } =>
+          Array.isArray(e.requiredInputFields) &&
+          e.requiredInputFields.length > 0,
+      );
   }
 
   /**

scripts/codegen/generate-evaluator-models.ts

@@ -17,6 +17,7 @@ import type { OpenAPI, OpenAPIV2, OpenAPIV3 } from "openapi-types";
 interface EvaluatorDefinition {
   slug: string;
   requestSchemaName?: string;
+  requestSchemaInSpec?: boolean;
   requestSchema?: OpenAPIV3.SchemaObject;
   description?: string;
 }
@@ -237,10 +238,16 @@ export function isValidEvaluatorSlug(slug: string): slug is EvaluatorSlug {
 function generateMbtEvaluatorsFile(
   evaluators: EvaluatorDefinition[],
   isSwagger2: boolean,
+  availableSchemas: Record<string, unknown>,
 ): string {
   // Generate type aliases for configs
   const typeAliases = evaluators
-    .filter((e) => hasConfigFields(e) && e.requestSchemaName)
+    .filter(
+      (e) =>
+        hasConfigFields(e) &&
+        e.requestSchemaName &&
+        availableSchemas[e.requestSchemaName],
+    )
     .map((e) => {
       const typeName = `${slugToClassName(e.slug)}Config`;
       const typePath = isSwagger2
@@ -255,8 +262,12 @@ function generateMbtEvaluatorsFile(
     .map((e) => {
       const methodName = slugToCamelCase(e.slug);
       const className = slugToClassName(e.slug);
-      const hasConfig = hasConfigFields(e) && e.requestSchemaName;
-      const configType = hasConfig ? `${className}Config` : null;
+      const hasConfig = hasConfigFields(e);
+      const configType = hasConfig
+        ? e.requestSchemaName && availableSchemas[e.requestSchemaName]
+          ? `${className}Config`
+          : "Record<string, unknown>"
+        : null;
 
       const { requiredInputFields } = e.requestSchema
         ? extractFieldsFromSchema(e.requestSchema)
@@ -590,9 +601,13 @@ async function main(): Promise<void> {
   );
   console.log(`  - registry.ts`);
 
+  const filteredSchemas = (
+    filteredSpec as { components: { schemas: Record<string, unknown> } }
+  ).components.schemas;
+
   fs.writeFileSync(
     path.join(outputDir, "mbt-evaluators.ts"),
-    generateMbtEvaluatorsFile(evaluators, isSwagger2),
+    generateMbtEvaluatorsFile(evaluators, isSwagger2, filteredSchemas),
   );
   console.log(`  - mbt-evaluators.ts`);