feat - tests + evals

README.md

@@ -103,6 +103,15 @@ Optionally, you can add the following to a file called `.vscode/mcp.json` in you
 }
 ```
 
+
+
+## Running evals
+
+The evals package loads an mcp client that then runs the index.ts file, so there is no need to rebuild between tests. You can load environment variables by prefixing the npx command. Full documentation can be found [here](https://www.mcpevals.io/docs).
+
+```bash
+OPENAI_API_KEY=your-key  npx mcp-eval src/evals/evals.ts src/index.ts
+```
 ## Configuration on Claude/Windsurf/Cursor/Cline
 
 ```json

package.json

@@ -27,7 +27,8 @@
     "@vectorize-io/vectorize-client": "^0.1.3",
     "dotenv": "^16.4.7",
     "p-queue": "^8.0.1",
-    "shx": "^0.3.4"
+    "shx": "^0.3.4",
+    "mcp-evals": "^1.0.18"
   },
   "devDependencies": {
     "@types/jest": "^29.5.14",
@@ -58,4 +59,4 @@
     "url": "https://github.com/vectorize-io/vectorize-mcp-server/issues"
   },
   "homepage": "https://github.com/vectorize-io/vectorize-mcp-server#readme"
-}
+}

src/evals/evals.ts

@@ -0,0 +1,42 @@
+//evals.ts
+
+import { EvalConfig } from 'mcp-evals';
+import { openai } from "@ai-sdk/openai";
+import { grade, EvalFunction } from "mcp-evals";
+
+const retrieveEval: EvalFunction = {
+    name: 'retrieveEval',
+    description: 'Evaluates retrieving documents from the pipeline',
+    run: async () => {
+        const result = await grade(openai("gpt-4"), "Retrieve 4 documents about best practices for large language models.");
+        return JSON.parse(result);
+    }
+};
+
+const deepResearchEval: EvalFunction = {
+    name: 'Deep Research Tool Evaluation',
+    description: 'Evaluates the functionality of the deep research tool',
+    run: async () => {
+        const result = await grade(openai("gpt-4"), "Perform a deep research on the impact of microplastics in oceans and provide references. Use web search if necessary.");
+        return JSON.parse(result);
+    }
+};
+
+const extractEval: EvalFunction = {
+    name: 'extract tool evaluation',
+    description: 'Evaluates the text extraction and chunking functionality of the extract tool',
+    run: async () => {
+        const prompt = "Please extract the text from the following base64-encoded document: dGVzdCBkYXRh. The content type is application/pdf. Provide the extracted text.";
+        const result = await grade(openai("gpt-4"), prompt);
+        return JSON.parse(result);
+    }
+};
+
+const config: EvalConfig = {
+    model: openai("gpt-4"),
+    evals: [retrieveEval, deepResearchEval, extractEval]
+};
+
+export default config;
+
+export const evals = [retrieveEval, deepResearchEval, extractEval];