feat(server): online copilot api test

.github/actions/copilot-test/action.yml

@@ -3,7 +3,7 @@ description: 'Run Copilot E2E Test'
 inputs:
   script:
     description: 'Script to run'
-    default: 'yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only'
+    default: 'yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only'
     required: false
   openai-key:
     description: 'OpenAI secret key'

.github/workflows/build-test.yml

@@ -504,7 +504,8 @@ jobs:
           filters: |
             changed:
               - 'packages/backend/server/src/plugins/copilot/**'
-              - 'packages/backend/server/tests/copilot.*'
+              - 'packages/backend/server/tests/copilot*'
+              - 'tests/affine-cloud/copilot/**'
 
       - name: Setup Node.js
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
@@ -526,7 +527,7 @@ jobs:
 
       - name: Run server tests
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
-        run: yarn affine @affine/server test:copilot:coverage --forbid-only
+        run: yarn affine @affine/server test:copilot:spec:coverage --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
@@ -595,7 +596,7 @@ jobs:
             changed:
               - 'packages/frontend/core/src/blocksuite/presets/ai/**'
               - 'packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/**'
-              - 'tests/affine-cloud-copilot/**'
+              - 'tests/affine-cloud/copilot/**'
 
       - name: Setup Node.js
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.e2efilter.outputs.changed == 'true' }}
@@ -616,7 +617,7 @@ jobs:
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.e2efilter.outputs.changed == 'true' }}
         uses: ./.github/actions/copilot-test
         with:
-          script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          script: yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
 

.github/workflows/copilot-test.yml

@@ -38,6 +38,16 @@ jobs:
       DISTRIBUTION: web
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       REDIS_SERVER_HOST: localhost
+    strategy:
+      fail-fast: false
+      matrix:
+        spec:
+          - {
+              name: e2e,
+              package: '@affine-test/affine-cloud-copilot',
+              type: e2e,
+            }
+          - { name: spec, package: '@affine/server', type: copilot:spec }
     services:
       postgres:
         image: postgres
@@ -78,12 +88,13 @@ jobs:
       - name: Prepare Server Test Environment
         uses: ./.github/actions/server-test-env
 
-      - name: Run server tests
-        run: yarn affine @affine/server test:copilot:coverage --forbid-only
+      - name: Run copilot api ${{ matrix.spec.name }} tests
+        run: yarn affine ${{ matrix.spec.package }} test:${{ matrix.spec.type }}:coverage --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
+          COPILOT_E2E_ENDPOINT: ${{ secrets.COPILOT_E2E_ENDPOINT }}
 
       - name: Upload server test coverage results
         uses: codecov/codecov-action@v5
@@ -144,7 +155,7 @@ jobs:
       - name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
         uses: ./.github/actions/copilot-test
         with:
-          script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          script: yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
 

package.json

@@ -12,7 +12,8 @@
     "tools/*",
     "docs/reference",
     "tools/@types/*",
-    "tests/*"
+    "tests/*",
+    "tests/affine-cloud/*"
   ],
   "engines": {
     "node": "<21.0.0"

packages/backend/server/package.json

@@ -11,9 +11,11 @@
     "build": "tsc",
     "dev": "nodemon ./src/index.ts",
     "test": "ava --concurrency 1 --serial",
-    "test:copilot": "ava \"tests/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e": "ava \"tests/**/copilot-*.e2e.ts\"",
+    "test:copilot:spec": "ava \"tests/**/copilot-*.spec.ts\"",
     "test:coverage": "c8 ava --concurrency 1 --serial",
-    "test:copilot:coverage": "c8 ava --timeout=5m \"tests/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"tests/**/copilot-*.e2e.ts\"",
+    "test:copilot:spec:coverage": "c8 ava --timeout=5m \"tests/**/copilot-*.spec.ts\"",
     "data-migration": "cross-env NODE_ENV=script r ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && NODE_ENV=script node --import ./scripts/register.js ./dist/data/index.js run",
     "postinstall": "prisma generate"

packages/backend/server/tests/copilot-provider.e2e.ts

@@ -0,0 +1,214 @@
+import { randomUUID } from 'node:crypto';
+
+import { createRandomAIUser } from '@affine-test/kit/utils/cloud';
+import type { ExecutionContext, TestFn } from 'ava';
+import ava from 'ava';
+
+import { createWorkspace } from './utils';
+import {
+  chatWithImages,
+  chatWithText,
+  chatWithWorkflow,
+  createCopilotMessage,
+  createCopilotSession,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+  sse2array,
+} from './utils/copilot';
+
+type Tester = {
+  app: any;
+  userEmail: string;
+  userToken: string;
+  workspaceId: string;
+};
+const test = ava as TestFn<Tester>;
+
+const e2eConfig = {
+  endpoint: process.env.COPILOT_E2E_ENDPOINT || 'http://localhost:3010',
+};
+
+const isCopilotConfigured =
+  !!process.env.COPILOT_OPENAI_API_KEY &&
+  !!process.env.COPILOT_FAL_API_KEY &&
+  process.env.COPILOT_OPENAI_API_KEY !== '1' &&
+  process.env.COPILOT_FAL_API_KEY !== '1';
+const runIfCopilotConfigured = test.macro(
+  async (
+    t,
+    callback: (t: ExecutionContext<Tester>) => Promise<void> | void
+  ) => {
+    if (isCopilotConfigured) {
+      await callback(t);
+    } else {
+      t.log('Skip test because copilot is not configured');
+      t.pass();
+    }
+  }
+);
+
+export const runPrisma = async <T>(
+  cb: (
+    prisma: InstanceType<
+      // eslint-disable-next-line @typescript-eslint/consistent-type-imports
+      typeof import('../../../../packages/backend/server/node_modules/@prisma/client').PrismaClient
+    >
+  ) => Promise<T>
+): Promise<T> => {
+  const {
+    PrismaClient,
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+  } = await import(
+    '../../../../packages/backend/server/node_modules/@prisma/client'
+  );
+  const client = new PrismaClient();
+  await client.$connect();
+  try {
+    return await cb(client);
+  } finally {
+    await client.$disconnect();
+  }
+};
+
+test.before(async t => {
+  if (!isCopilotConfigured) return;
+  const { endpoint } = e2eConfig;
+
+  const { email, sessionId: token } = await createRandomAIUser(
+    'affine.fail',
+    runPrisma
+  );
+  const app = { getHttpServer: () => endpoint } as any;
+  const { id } = await createWorkspace(app, token);
+
+  t.context.app = app;
+  t.context.userEmail = email;
+  t.context.userToken = token;
+  t.context.workspaceId = id;
+});
+
+test.after(async t => {
+  if (!isCopilotConfigured) return;
+  await runPrisma(async client => {
+    await client.user.delete({
+      where: {
+        email: t.context.userEmail,
+      },
+    });
+  });
+});
+
+const retry = async (
+  action: string,
+  t: ExecutionContext<Tester>,
+  callback: (t: ExecutionContext<Tester>) => void
+) => {
+  let i = 3;
+  while (i--) {
+    const ret = await t.try(callback);
+    if (ret.passed) {
+      return ret.commit();
+    } else {
+      ret.discard();
+      t.log(ret.errors.map(e => e.message).join('\n'));
+      t.log(`retrying ${action} ${3 - i}/3 ...`);
+    }
+  }
+  t.fail(`failed to run ${action}`);
+};
+
+const makeCopilotChat = async (
+  t: ExecutionContext<Tester>,
+  promptName: string,
+  { content, attachments, params }: any
+) => {
+  const { app, userToken, workspaceId } = t.context;
+  const sessionId = await createCopilotSession(
+    app,
+    userToken,
+    workspaceId,
+    randomUUID(),
+    promptName
+  );
+  const messageId = await createCopilotMessage(
+    app,
+    userToken,
+    sessionId,
+    content,
+    attachments,
+    undefined,
+    params
+  );
+  return { sessionId, messageId };
+};
+
+// ==================== action ====================
+
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
+  const prompts = Array.isArray(promptName) ? promptName : [promptName];
+  for (const promptName of prompts) {
+    test(
+      `should be able to run action: ${promptName}`,
+      runIfCopilotConfigured,
+      async t => {
+        await retry(`action: ${promptName}`, t, async t => {
+          const { app, userToken } = t.context;
+          const { sessionId, messageId } = await makeCopilotChat(
+            t,
+            promptName,
+            messages[0]
+          );
+
+          if (type === 'text') {
+            const result = await chatWithText(
+              app,
+              userToken,
+              sessionId,
+              messageId
+            );
+            t.truthy(result, 'should return result');
+            verifier?.(t, result);
+          } else if (type === 'image') {
+            const result = sse2array(
+              await chatWithImages(app, userToken, sessionId, messageId)
+            )
+              .filter(e => e.event !== 'event')
+              .map(e => e.data)
+              .filter(Boolean);
+            t.truthy(result.length, 'should return result');
+            for (const r of result) {
+              verifier?.(t, r);
+            }
+          } else {
+            t.fail('unsupported provider type');
+          }
+        });
+      }
+    );
+  }
+}
+
+// ==================== workflow ====================
+
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
+  test(
+    `should be able to run workflow: ${name}`,
+    runIfCopilotConfigured,
+    async t => {
+      await retry(`workflow: ${name}`, t, async t => {
+        const { app, userToken } = t.context;
+        const { sessionId, messageId } = await makeCopilotChat(
+          t,
+          `workflow:${name}`,
+          { content }
+        );
+        const r = await chatWithWorkflow(app, userToken, sessionId, messageId);
+        const result = sse2array(r)
+          .filter(e => e.event !== 'event' && e.data)
+          .reduce((p, c) => p + c.data, '');
+        t.truthy(result, 'should return result');
+        verifier?.(t, result);
+      });
+    }
+  );
+}