Handle node fork failures; add InternalError type

Author: richiemcilroy

apps/api/src/errors.ts

@@ -58,6 +58,10 @@ export class BillingLimitError extends Data.TaggedError('BillingLimitError')<{
   readonly message: string
 }> {}
 
+export class InternalError extends Data.TaggedError('InternalError')<{
+  readonly message: string
+}> {}
+
 export type ApiError =
   | NotFoundError
   | UnauthorizedError
@@ -73,6 +77,7 @@ export type ApiError =
   | GoneError
   | NotImplementedError
   | BillingLimitError
+  | InternalError
 
 const STATUS_MAP: Record<ApiError['_tag'], number> = {
   NotFoundError: 404,
@@ -89,6 +94,7 @@ const STATUS_MAP: Record<ApiError['_tag'], number> = {
   GoneError: 410,
   NotImplementedError: 501,
   BillingLimitError: 403,
+  InternalError: 500,
 }
 
 const CODE_MAP: Record<ApiError['_tag'], string> = {
@@ -106,6 +112,7 @@ const CODE_MAP: Record<ApiError['_tag'], string> = {
   GoneError: 'gone',
   NotImplementedError: 'not_implemented',
   BillingLimitError: 'billing_limit',
+  InternalError: 'internal_error',
 }
 
 export function errorToResponse(error: ApiError, requestId: string) {

apps/api/src/routes/sandboxes.test.ts

@@ -33,16 +33,17 @@ import { idToBytes } from '@sandchest/contract'
 import type { CreateSandboxResponse, GetSandboxResponse, ReplayBundle } from '@sandchest/contract'
 import type { BufferedEvent } from '../services/redis.js'
 import { RUN_API_INTEGRATION_TESTS } from '../test-support.js'
+import type { NodeClientApi } from '../services/node-client.js'
 
 const TEST_ORG = 'org_test_123'
 const TEST_USER = 'user_test_456'
 
-function createTestEnv() {
+function createTestEnv(overrides?: { nodeClient?: NodeClientApi }) {
   const sandboxRepo = createInMemorySandboxRepo()
   const execRepo = createInMemoryExecRepo()
   const sessionRepo = createInMemorySessionRepo()
   const objectStorage = createInMemoryObjectStorage()
-  const nodeClient = createInMemoryNodeClient()
+  const nodeClient = overrides?.nodeClient ?? createInMemoryNodeClient()
   const redis = createInMemoryRedisApi()
   const artifactRepo = createInMemoryArtifactRepo()
   const quotaApi = createInMemoryQuotaApi()
@@ -777,6 +778,48 @@ describe.skipIf(!RUN_API_INTEGRATION_TESTS)('POST /v1/sandboxes/:id/fork — quo
   })
 })
 
+describe.skipIf(!RUN_API_INTEGRATION_TESTS)('POST /v1/sandboxes/:id/fork — node failure handling', () => {
+  test('returns internal_error with node message and marks fork failed', async () => {
+    const baseNodeClient = createInMemoryNodeClient()
+    const failingNodeClient: NodeClientApi = {
+      ...baseNodeClient,
+      forkSandbox: () => Effect.die(new Error('simulated fork failure')),
+    }
+    const env = createTestEnv({ nodeClient: failingNodeClient })
+    const parentId = await createRunningSandbox(env)
+
+    const result = await env.runTest(
+      Effect.gen(function* () {
+        const client = yield* HttpClient.HttpClient
+        const response = yield* client.execute(
+          HttpClientRequest.post(`/v1/sandboxes/${parentId}/fork`).pipe(
+            HttpClientRequest.bodyUnsafeJson({}),
+          ),
+        )
+        const body = yield* response.json
+        return { status: response.status, body: body as { error: string; message: string } }
+      }),
+    )
+
+    expect(result.status).toBe(500)
+    expect(result.body.error).toBe('internal_error')
+    expect(result.body.message).toContain('simulated fork failure')
+
+    const tree = await env.runTest(
+      Effect.gen(function* () {
+        const client = yield* HttpClient.HttpClient
+        const response = yield* client.execute(
+          HttpClientRequest.get(`/v1/sandboxes/${parentId}/forks`),
+        )
+        return (yield* response.json) as { tree: Array<{ status: string; failure_reason?: string | null }> }
+      }),
+    )
+
+    const failedFork = tree.tree.find((node) => node.status === 'failed')
+    expect(failedFork).toBeDefined()
+  })
+})
+
 // ---------------------------------------------------------------------------
 // GET /v1/sandboxes/:id/stream — sandbox-level SSE
 // ---------------------------------------------------------------------------

apps/api/src/routes/sandboxes.ts

@@ -1,5 +1,5 @@
 import { HttpRouter, HttpServerRequest, HttpServerResponse } from '@effect/platform'
-import { Effect } from 'effect'
+import { Cause, Effect } from 'effect'
 import {
   generateUUIDv7,
   idToBytes,
@@ -33,6 +33,7 @@ import {
   ForkDepthExceededError,
   ForkLimitExceededError,
   GoneError,
+  InternalError,
   NotFoundError,
   QuotaExceededError,
   SandboxNotRunningError,
@@ -613,14 +614,29 @@ const forkSandbox = Effect.gen(function* () {
     ttlSeconds,
   })
 
-  // Increment parent's fork count
-  yield* repo.incrementForkCount(sourceIdBytes, auth.orgId)
-
   // Tell the node to fork the VM
   yield* nodeClient.forkSandbox({
     sourceSandboxId: sourceIdBytes,
     newSandboxId: forkId,
-  })
+  }).pipe(
+    Effect.catchAllCause((cause) =>
+      Effect.gen(function* () {
+        yield* repo.updateStatus(forkId, auth.orgId, 'failed', {
+          endedAt: new Date(),
+          failureReason: 'provision_failed',
+        })
+
+        return yield* Effect.fail(
+          new InternalError({
+            message: `Fork failed on node: ${Cause.pretty(cause)}`,
+          }),
+        )
+      }),
+    ),
+  )
+
+  // Increment parent's fork count only after the node confirms the fork succeeded.
+  yield* repo.incrementForkCount(sourceIdBytes, auth.orgId)
 
   const sandboxId = bytesToId(SANDBOX_PREFIX, forkRow.id)
   const response: ForkSandboxResponse = {