diff --git a/.claude/settings.local.json b/.claude/settings.local.json index b698e146d4..cc7cf13303 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -7,7 +7,9 @@ "Bash(cargo check:*)", "Bash(cargo fmt:*)", "Bash(pnpm format:*)", - "Bash(pnpm exec biome check:*)" + "Bash(pnpm exec biome check:*)", + "Bash(/opt/homebrew/bin/pnpm format)", + "Bash(~/.cargo/bin/cargo check -p scap-targets)" ], "deny": [], "ask": [] diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..049da63dde --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +# Enforce LF line endings for all text files +* text=auto eol=lf + +# Explicitly declare text files +*.rs text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.js text eol=lf +*.jsx text eol=lf +*.json text eol=lf +*.md text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.html text eol=lf +*.css text eol=lf +*.scss text eol=lf +*.sql text eol=lf +*.sh text eol=lf + +# Binary files +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.woff binary +*.woff2 binary +*.ttf binary +*.eot binary +*.mp4 binary +*.webm binary +*.mp3 binary +*.wav binary +*.pdf binary diff --git a/.github/actions/setup-js/action.yml b/.github/actions/setup-js/action.yml index 11c560b3c3..d26f3df4c2 100644 --- a/.github/actions/setup-js/action.yml +++ b/.github/actions/setup-js/action.yml @@ -12,7 +12,7 @@ runs: - name: Setup node uses: actions/setup-node@v3 with: - node-version: 24 + node-version: 20 cache: pnpm - name: Install frontend dependencies diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a8056edd4d..fc7ca32364 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,17 +3,12 @@ name: "publish" # change this when ready to release if you want CI/CD on: workflow_dispatch: - inputs: - interactionId: - description: "Discord Interaction ID" - required: false - type: string env: - CN_APPLICATION: cap/cap + CN_APPLICATION: inflight/inflight APP_CARGO_TOML: apps/desktop/src-tauri/Cargo.toml - SENTRY_ORG: cap-s2 - SENTRY_PROJECT: cap-desktop + SENTRY_ORG: inflight-software + SENTRY_PROJECT: inflight-desktop jobs: draft: @@ -97,36 +92,6 @@ jobs: draft: true generate_release_notes: true - - name: Update Discord interaction - if: ${{ inputs.interactionId != '' }} - uses: actions/github-script@v7 - with: - script: | - async function main() { - const token = await core.getIDToken("cap-discord-bot"); - const cnReleaseId = JSON.parse(`${{ steps.create_cn_release.outputs.stdout }}`).id; - - const resp = await fetch("https://cap-discord-bot.brendonovich.workers.dev/github-workflow", { - method: "POST", - body: JSON.stringify({ - type: "release-ready", - tag: "${{ steps.create_tag.outputs.tag_name }}", - version: "${{ steps.read_version.outputs.value }}", - releaseUrl: "${{ steps.create_gh_release.outputs.url }}", - interactionId: "${{ inputs.interactionId }}", - cnReleaseId - }), - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - } - }); - - if(resp.status !== 200) throw new Error(await resp.text()); - } - - main(); - build: needs: draft if: ${{ needs.draft.outputs.needs_release == 'true' }} @@ -138,9 +103,9 @@ jobs: matrix: settings: - target: x86_64-apple-darwin - runner: macos-latest-xlarge + runner: macos-latest - target: aarch64-apple-darwin - runner: macos-latest-xlarge + runner: macos-latest - target: x86_64-pc-windows-msvc runner: windows-latest env: @@ -155,13 +120,13 @@ jobs: run: echo "${{ secrets.APPLE_API_KEY_FILE }}" > api.p8 - uses: apple-actions/import-codesign-certs@v2 - if: ${{ matrix.settings.runner == 'macos-latest-xlarge' }} + if: ${{ startsWith(matrix.settings.runner, 'macos') }} with: p12-file-base64: ${{ secrets.APPLE_CERTIFICATE }} p12-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} - name: Verify certificate - if: ${{ matrix.settings.runner == 'macos-latest-xlarge' }} + if: ${{ startsWith(matrix.settings.runner, 'macos') }} run: security find-identity -v -p codesigning ${{ runner.temp }}/build.keychain - name: Rust setup @@ -280,39 +245,3 @@ jobs: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} run: | sentry-cli debug-files upload -o ${{ env.SENTRY_ORG }} -p ${{ env.SENTRY_PROJECT }} target/${{ matrix.settings.target }}/release/cap_desktop.pdb - - done: - needs: [draft, build] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - steps: - - name: Send Discord notification - if: ${{ inputs.interactionId != '' }} - uses: actions/github-script@v7 - with: - script: | - async function main() { - const token = await core.getIDToken("cap-discord-bot"); - const cnReleaseId = JSON.parse(`${{ needs.draft.outputs.cn_release_stdout }}`).id; - - const resp = await fetch("https://cap-discord-bot.brendonovich.workers.dev/github-workflow", { - method: "POST", - body: JSON.stringify({ - type: "release-done", - interactionId: "${{ inputs.interactionId }}", - version: "${{ needs.draft.outputs.version }}", - releaseUrl: "${{ needs.draft.outputs.gh_release_url }}", - cnReleaseId - }), - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - } - }); - - if(resp.status !== 200) throw new Error(await resp.text()); - } - - main(); diff --git a/.gitignore b/.gitignore index 40ea0b2797..b12f9dfcf7 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,10 @@ tauri.windows.conf.json # Cursor .cursor .env*.local + +# Secrets and credentials +*.pem +*.key +**/credentials*.json +**/secrets.json +**/*.secrets diff --git a/CLAUDE.md b/CLAUDE.md index ee2f0b29f0..9b6be64b8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,429 +1,203 @@ # CLAUDE.md -This file provides comprehensive guidance to Claude Code when working with code in this repository. +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Project Overview -Cap is the open source alternative to Loom. It's a Turborepo monorepo with a Tauri v2 desktop app (Rust + SolidStart) and a Next.js web app. The Next.js app at `apps/web` is the main web application for sharing and management; the desktop app at `apps/desktop` is the cross‑platform recorder/editor (macOS and Windows). - -### Product Context -- **Core Purpose**: Screen recording with instant sharing capabilities -- **Target Users**: Content creators, developers, product managers, support teams -- **Key Features**: Instant recording, studio mode, AI-generated captions, collaborative comments -- **Business Model**: Freemium SaaS with usage-based pricing - -## File Location Patterns & Key Directories - -### Core Applications -- `apps/web/` — Next.js web application (sharing, management, dashboard) -- `apps/desktop/` — Tauri desktop app (recording, editing) -- `apps/discord-bot/` — Discord integration bot -- `apps/storybook/` — UI component documentation - -### Shared Packages -- `packages/database/` — Drizzle ORM, auth, email templates -- `packages/ui/` — React components for web app -- `packages/ui-solid/` — SolidJS components for desktop -- `packages/utils/` — Shared utilities, types, constants -- `packages/env/` — Environment variable validation -- `packages/web-*` — Effect-based web API layers - -### Rust Crates -- `crates/media*/` — Video/audio processing pipeline -- `crates/recording/` — Core recording functionality -- `crates/rendering/` — Video rendering and effects -- `crates/camera*/` — Cross-platform camera handling -- `crates/scap-*/` — Screen capture implementations - -### Important File Patterns -- `**/tauri.ts` — Auto-generated IPC bindings (DO NOT EDIT) -- `**/queries.ts` — Auto-generated query bindings (DO NOT EDIT) -- `apps/web/actions/**/*.ts` — Server Actions ("use server") -- `packages/database/schema.ts` — Database schema definitions -- `*.config.*` — Configuration files (Next.js, Tailwind, etc.) +Inflight Recorder is a video messaging tool (fork of Cap, the open source Loom alternative). It's a Turborepo monorepo with a Tauri v2 desktop app (Rust + SolidStart) and a Next.js web app. + +**Core Applications:** +- `apps/web` — Next.js 15 (App Router) web application for sharing, management, dashboard +- `apps/desktop` — Tauri v2 desktop app with SolidStart (recording, editing) +- `apps/cli` — Rust CLI tool +- `apps/discord-bot` — Discord integration bot +- `apps/storybook` — UI component documentation + +**Shared Packages:** +- `packages/database` — Drizzle ORM (MySQL), auth utilities, email templates +- `packages/ui` — React components for web +- `packages/ui-solid` — SolidJS components for desktop +- `packages/utils` — Shared utilities and types +- `packages/env` — Zod-validated environment modules +- `packages/web-domain` — Shared domain types (Video, User, Organisation, etc.) +- `packages/web-backend` — Effect-based backend services (Videos, S3Buckets, Users, etc.) +- `packages/web-api-contract` — ts-rest API contracts for desktop +- `packages/web-api-contract-effect` — Effect-based HTTP API contracts + +**Rust Crates** (`crates/*`): +- `recording` — Core recording functionality +- `media`, `audio`, `video-decode` — Media processing pipeline +- `rendering`, `rendering-skia` — Video rendering and effects +- `camera*` — Cross-platform camera handling (AVFoundation, DirectShow, MediaFoundation) +- `scap-*` — Screen capture implementations (ScreenCaptureKit, Direct3D) +- `enc-*` — Encoding implementations (FFmpeg, AVFoundation, MediaFoundation, GIF) +- `export`, `editor`, `project` — Export and editing functionality ## Key Commands +### Initial Setup +```bash +pnpm install # Install dependencies +pnpm env-setup # Generate .env file (interactive) +pnpm cap-setup # Install native dependencies (FFmpeg, etc.) +``` + ### Development ```bash -pnpm dev:web # Start Next.js dev server (apps/web only) -pnpm run dev:desktop # Start Tauri desktop dev (apps/desktop) -pnpm build # Build all packages/apps via Turbo -pnpm lint # Lint with Biome across the repo -pnpm format # Format with Biome -pnpm typecheck # TypeScript project references build +pnpm dev # Start web + desktop + Docker services +pnpm dev:web # Web only (starts Docker for MySQL/MinIO) +pnpm dev:desktop # Desktop only +cd apps/web && pnpm dev # Web without Docker ``` -### Database Operations +### Build & Quality ```bash -pnpm db:generate # Generate Drizzle migrations -pnpm db:push # Push schema changes to MySQL -pnpm db:studio # Open Drizzle Studio -pnpm --dir packages/database db:check # Verify database schema +pnpm build # Build all via Turbo +pnpm tauri:build # Build desktop release +pnpm lint # Lint with Biome +pnpm format # Format with Biome +pnpm typecheck # TypeScript check +cargo fmt # Format Rust code +cargo build -p # Build specific Rust crate +cargo test -p # Test specific Rust crate ``` -### App-Specific Commands +### Database ```bash -# Web app (apps/web) -cd apps/web && pnpm dev # Start Next.js dev server +pnpm db:generate # Generate Drizzle migrations +pnpm db:push # Push schema to MySQL +pnpm db:studio # Open Drizzle Studio +``` -# Desktop (apps/desktop) -cd apps/desktop && pnpm dev # Start SolidStart + Tauri dev -pnpm tauri:build # Build desktop app (release) +### Docker +```bash +pnpm docker:up # Start MySQL/MinIO containers +pnpm docker:stop # Stop containers +pnpm docker:clean # Remove containers and volumes ``` -## Development Environment Guidelines +### Analytics (Tinybird) +```bash +pnpm analytics:setup # Provision Tinybird data sources (destructive) +pnpm analytics:check # Validate Tinybird schema +``` -### Server Management -- Do not start additional development servers or localhost services unless explicitly asked. Assume the developer already has the environment running and focus on code changes. -- Prefer `pnpm dev:web` or `pnpm run dev:desktop` when you only need one app. Avoid starting multiple overlapping servers. -- Avoid running Docker or external services yourself unless requested; root workflows handle them as needed. -- **Database**: MySQL via Docker Compose; schema managed through Drizzle migrations -- **Storage**: S3-compatible (AWS, Cloudflare R2, etc.) for video/audio files - -### Auto-generated Bindings (Desktop) -- **NEVER EDIT**: `tauri.ts`, `queries.ts` (auto-generated on app load) -- **NEVER EDIT**: Files under `apps/desktop/src-tauri/gen/` -- **Icons**: Auto-imported in desktop app; do not import manually -- **Regeneration**: These files update automatically when Rust types change - -### Common Development Pain Points -- **Node Version**: Must use Node 20 (specified in package.json engines) -- **PNPM Version**: Locked to 10.5.2 for consistency -- **Turbo Cache**: May need clearing if builds behave unexpectedly (`rm -rf .turbo`) -- **Database Migrations**: Always run `pnpm db:generate` before `pnpm db:push` -- **Desktop Icons**: Use `unplugin-icons` auto-import instead of manual imports - -## Architecture Overview - -### Monorepo Structure -- `apps/web` — Next.js 14 (App Router) web application -- `apps/desktop` — Tauri v2 desktop app with SolidStart (SolidJS) -- `packages/database` — Drizzle ORM (MySQL) + auth utilities -- `packages/ui` — React UI components for the web -- `packages/ui-solid` — SolidJS UI components for desktop -- `packages/utils` — Shared utilities and types -- `packages/env` — Zod-validated build/server env modules -- `crates/*` — Rust crates for media, rendering, recording, camera, etc. +## Critical Rules -### Technology Stack -- **Package Manager**: pnpm (`pnpm@10.5.2`) -- **Build System**: Turborepo -- **Frontend (Web)**: React 19 + Next.js 14.2.x (App Router) -- **Desktop**: Tauri v2, Rust 2024, SolidStart -- **Styling**: Tailwind CSS (web consumes `@cap/ui/tailwind`) -- **Server State**: TanStack Query v5 on web; `@tanstack/solid-query` on desktop -- **Database**: MySQL (PlanetScale) with Drizzle ORM -- **AI Integration**: Groq preferred, OpenAI fallback; invoked in Next.js Server Actions -- **Analytics**: PostHog -- **Payments**: Stripe - -### Critical Architectural Decisions -1. **AI on the Server**: All Groq/OpenAI calls execute in Server Actions under `apps/web/actions`. Never call AI from client components. -2. **Authentication**: NextAuth with a custom Drizzle adapter. Session handling via NextAuth cookies; API keys are supported for certain endpoints. -3. **API Surface**: Prefer Server Actions. When routes are necessary, implement under `app/api/*` (Hono-based utilities present), set proper CORS, and revalidate precisely. -4. **Desktop IPC**: Use `tauri_specta` for strongly typed commands/events; do not modify generated bindings. - -#### Desktop event pattern -Rust (emit): -```rust -use specta::Type; -use tauri_specta::Event; +### Auto-generated Files (NEVER EDIT) +- `**/tauri.ts` — IPC bindings (regenerated on app load) +- `**/queries.ts` — Query bindings +- `apps/desktop/src-tauri/gen/**` — Tauri generated files -#[derive(Serialize, Type, tauri_specta::Event, Debug, Clone)] -pub struct UploadProgress { - progress: f64, - message: String, -} +### NO CODE COMMENTS +Never add comments (`//`, `/* */`, `///`, `//!`, `#`, etc.) to any code. Code must be self-explanatory through naming, types, and structure. -UploadProgress { progress: 0.0, message: "Starting upload...".to_string() } - .emit(&app) - .ok(); -``` +### Server Management +Do not start additional dev servers unless asked. Assume the developer already has the environment running. -Frontend (listen; generated bindings): -```ts -import { events } from "./tauri"; // auto-generated -await events.uploadProgress.listen((event) => { - // update UI with event.payload -}); -``` +### Database Changes +Always run: `pnpm db:generate` → `pnpm db:push` → test -## Development Workflow & Best Practices +### Desktop Permissions (macOS) +When running from terminal, grant screen/mic permissions to the terminal app, not the Inflight app. -### Code Organization Principles -1. **Follow Local Patterns**: Study neighboring files and shared packages first -2. **Database Changes**: Always `pnpm db:generate` → `pnpm db:push` → test -3. **Strict Typing**: Use existing types; validate config via `@cap/env` -4. **Component Consistency**: Use `@cap/ui` (React) or `@cap/ui-solid` (Solid) -5. **No Manual Edits**: Never touch auto-generated bindings or schemas +## Architecture Patterns -### Key Implementation Patterns +### Technology Stack +- **Package Manager**: pnpm 10.5.2 +- **Node**: 20+ +- **Rust**: 1.88+ +- **Build**: Turborepo +- **Frontend (Web)**: React 19 + Next.js 15 (App Router) +- **Desktop**: Tauri v2, SolidStart, Solid.js +- **Database**: MySQL (PlanetScale) with Drizzle ORM +- **Storage**: S3-compatible (AWS, Cloudflare R2, MinIO for local) +- **AI**: Groq (primary) + OpenAI (fallback) — Server Actions only -#### Server Actions (Web App) +### Server Actions (Web) ```typescript "use server"; -import { db } from "@cap/database"; -import { getCurrentUser } from "@cap/database/auth/session"; +import { db } from "@inflight/database"; +import { getCurrentUser } from "@inflight/database/auth/session"; +import { videos } from "@inflight/database/schema"; -export async function updateVideo(data: FormData) { +export async function updateVideo(videoId: string, title: string) { const user = await getCurrentUser(); if (!user?.id) throw new Error("Unauthorized"); - - // Database operations with Drizzle - return await db().update(videos).set({ ... }).where(eq(videos.id, id)); + return await db().update(videos).set({ name: title }).where(eq(videos.id, videoId)); } ``` -#### Desktop IPC Commands +### Desktop IPC (Tauri + specta) +Rust emit: ```rust -// Rust side - emit events +#[derive(Serialize, Type, tauri_specta::Event, Debug, Clone)] +pub struct UploadProgress { progress: f64, message: String } + UploadProgress { progress: 0.5, message: "Uploading...".to_string() } - .emit(&app) - .ok(); + .emit(&app).ok(); ``` +Frontend listen (auto-generated): ```typescript -// Frontend side - listen to events (auto-generated) import { events, commands } from "./tauri"; - -// Call commands await commands.startRecording({ ... }); - -// Listen to events await events.uploadProgress.listen((event) => { setProgress(event.payload.progress); }); ``` -#### React Query Patterns +### Effect System (API Routes) +API routes use `@effect/platform`'s `HttpApi` pattern. The main handler in `apps/web/app/api/[[...route]]/route.ts`: +```typescript +import { HttpApiScalar } from "@effect/platform"; +import { HttpLive } from "@inflight/web-backend"; +import { Layer } from "effect"; +import { apiToHandler } from "@/lib/server"; + +const handler = apiToHandler( + HttpApiScalar.layer({ path: "/api" }).pipe(Layer.provideMerge(HttpLive)), +); +export const GET = handler; +export const POST = handler; +``` + +Backend services are in `packages/web-backend/src/` organized by domain (Videos, Users, S3Buckets, etc.). +Run server effects through `runPromise` from `apps/web/lib/server.ts`. + +### React Query Pattern ```typescript -// Queries with Server Actions const { data, isLoading } = useQuery({ queryKey: ["videos", userId], queryFn: () => getUserVideos(), staleTime: 5 * 60 * 1000, }); - -// Mutations with cache updates -const updateMutation = useMutation({ - mutationFn: updateVideo, - onSuccess: (updated) => { - queryClient.setQueryData(["video", updated.id], updated); - }, -}); ``` -## Environment Variables - -### Build/Client (selected) -- `NEXT_PUBLIC_WEB_URL` -- `NEXT_PUBLIC_POSTHOG_KEY`, `NEXT_PUBLIC_POSTHOG_HOST` -- `NEXT_PUBLIC_DOCKER_BUILD` (enables Next.js standalone output) - -### Server (selected) -- Core: `DATABASE_URL`, `WEB_URL`, `NEXTAUTH_SECRET`, `NEXTAUTH_URL` -- S3: `CAP_AWS_BUCKET`, `CAP_AWS_REGION`, `CAP_AWS_ACCESS_KEY`, `CAP_AWS_SECRET_KEY`, optional `CAP_AWS_ENDPOINT`, `CAP_AWS_BUCKET_URL` -- AI: `GROQ_API_KEY`, `OPENAI_API_KEY` -- Email/Analytics: `RESEND_API_KEY`, `RESEND_FROM_DOMAIN`, `POSTHOG_PERSONAL_API_KEY`, `DUB_API_KEY`, `DEEPGRAM_API_KEY` -- OAuth: `GOOGLE_CLIENT_ID/SECRET`, `WORKOS_CLIENT_ID`, `WORKOS_API_KEY` -- Stripe: `STRIPE_SECRET_KEY_TEST`, `STRIPE_SECRET_KEY_LIVE`, `STRIPE_WEBHOOK_SECRET` -- CDN signing: `CLOUDFRONT_KEYPAIR_ID`, `CLOUDFRONT_KEYPAIR_PRIVATE_KEY` -- Optional S3 endpoints: `S3_PUBLIC_ENDPOINT`, `S3_INTERNAL_ENDPOINT` - -## Testing & Build Optimization - -### Testing Strategy -- **Package-Specific**: Check each `package.json` for test commands -- **Web App**: Uses Vitest for utilities, no comprehensive frontend tests yet -- **Desktop**: Vitest for SolidJS components in some packages -- **Tasks Service**: Jest for API endpoint testing -- **Rust**: Standard Cargo test framework for crates - -### Build Performance -- **Turborepo Caching**: Aggressive caching across all packages -- **Cache Invalidation**: Prefer targeted `--filter` over global rebuilds -- **Docker Builds**: `NEXT_PUBLIC_DOCKER_BUILD=true` enables standalone output -- **Development**: Incremental builds via TypeScript project references - -### Performance Monitoring -- **Bundle Analysis**: Check Next.js bundle size regularly -- **Database Queries**: Monitor with Drizzle Studio -- **S3 Operations**: Watch for excessive uploads/downloads -- **Desktop Memory**: Rust crates handle heavy media processing - -## Troubleshooting Common Issues - -### Build Failures -- **"Cannot find module"**: Check workspace dependencies in package.json -- **TypeScript errors**: Run `pnpm typecheck` to see project-wide issues -- **Turbo cache issues**: Clear with `rm -rf .turbo` -- **Node version mismatch**: Ensure Node 20 is active - -### Database Issues -- **Migration failures**: Check `packages/database/migrations/meta/` -- **Connection errors**: Verify Docker containers are running -- **Schema drift**: Run `pnpm --dir packages/database db:check` - -### Desktop App Issues -- **IPC binding errors**: Restart dev server to regenerate `tauri.ts` -- **Rust compile errors**: Check Cargo.toml dependencies -- **Permission issues**: macOS/Windows may require app permissions -- **Recording failures**: Verify screen capture permissions - -### Web App Issues -- **Auth failures**: Check NextAuth configuration and database -- **S3 upload errors**: Verify AWS credentials and bucket policies -- **Server Action errors**: Check network tab for detailed error messages -- **Hot reload issues**: Restart Next.js dev server - -## React/Next.js Coding Standards - -### Data Fetching & Server State -- Use TanStack Query v5 for all client-side server state and fetching. -- Use Server Components for initial data when possible; pass `initialData` to client components and let React Query take over. -- Mutations should call Server Actions directly and perform precise cache updates (`setQueryData`/`setQueriesData`) rather than broad invalidations. - -Basic query pattern: -```tsx -import { useQuery } from "@tanstack/react-query"; - -function Example() { - const { data, isLoading, error } = useQuery({ - queryKey: ["items"], - queryFn: fetchItems, - staleTime: 5 * 60 * 1000, - gcTime: 10 * 60 * 1000, - }); - if (isLoading) return ; - if (error) return { /* refetch */ }} />; - return ; -} -``` - -Server Action mutation with targeted cache updates: -```tsx -"use client"; -import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { updateItem } from "@/actions/items"; // 'use server' - -function useUpdateItem() { - const qc = useQueryClient(); - return useMutation({ - mutationFn: updateItem, - onSuccess: (updated) => { - qc.setQueriesData({ queryKey: ["items"] }, (old: any[] | undefined) => - old?.map((it) => (it.id === updated.id ? { ...it, ...updated } : it)) - ); - qc.setQueryData(["item", updated.id], updated); - }, - }); -} -``` - -Minimize `useEffect` usage: compute during render, handle logic in event handlers, and ensure cleanups for any subscriptions/timers. - -### Next.js App Router -- Prefer Server Components for SEO/initial rendering; hydrate interactivity in client components. -- Co-locate feature components, keep components focused, and use Suspense boundaries for long fetches. +## Important File Patterns -### UI/UX Guidelines -- Styling: Tailwind CSS only; stay consistent with spacing and tokens. -- Loading: Use static skeletons that mirror content; no bouncing animations. -- Performance: Memoize expensive work; code-split naturally; use Next/Image for remote assets. - -## Effect Patterns - -### Managed Runtimes -- `apps/web/lib/server.ts` builds a `ManagedRuntime` from `Layer.mergeAll` so database, S3, policy, and tracing services are available to every request. Always run server-side effects through `EffectRuntime.runPromise`/`runPromiseExit` from this module so cookie-derived context and `VideoPasswordAttachment` are attached automatically. -- `apps/web/lib/EffectRuntime.ts` exposes a browser runtime that merges the RPC client and tracing layers. Client code should lean on `useEffectQuery`, `useEffectMutation`, and `useRpcClient`; never call `ManagedRuntime.make` yourself inside components. - -### API Route Construction -- Next.js API folders under `apps/web/app/api/*` wrap Effect handlers with `@effect/platform`'s `HttpApi`/`HttpApiBuilder`. Follow the existing pattern: declare a contract class via `HttpApi.make`, configure groups/endpoints with `Schema`, and only export the `handler` returned by `apiToHandler(ApiLive)`. -- Inside `HttpApiBuilder.group` blocks, acquire services (e.g., `Videos`, `S3Buckets`) with `yield*` inside `Effect.gen`. Provide layers using `Layer.provide` rather than manual `provideService` calls so dependencies stay declarative. -- Map domain-level errors to transport errors with `HttpApiError.*`. Keep error translation exhaustive (`Effect.catchTags`, `Effect.tapErrorCause(Effect.logError)`) to preserve observability. -- Use `HttpAuthMiddleware` for required auth and `provideOptionalAuth` when guests are allowed. The middleware/utility already hydrate `CurrentUser`, so avoid duplicating session lookups in route handlers. -- Shared HTTP contracts that power the desktop app live in `packages/web-api-contract-effect`; update them alongside route changes to keep schemas in sync. - -### Server Components & Effects -- Server components that need Effect services should call `EffectRuntime.runPromise(effect.pipe(provideOptionalAuth))`. This keeps request cookies, tracing spans, and optional auth consistent with the API layer. -- Prefer lifting Drizzle queries or other async work into `Effect.gen` blocks and reusing domain services (`Videos`, `VideosPolicy`, etc.) rather than writing ad-hoc logic. - -### Client Integration -- React Query hooks should wrap Effect workflows with `useEffectQuery`/`useEffectMutation` from `apps/web/lib/EffectRuntime.ts`; these helpers surface Fail/Die causes consistently and plug into tracing/span metadata. -- When a mutation or query needs the RPC transport, resolve it through `useRpcClient()` and invoke the strongly-typed procedures exposed by `packages/web-domain` instead of reaching into fetch directly. - -## Desktop (Solid + Tauri) Patterns -- Data fetching: `@tanstack/solid-query` for server state. -- IPC: Call generated `commands` and `events` from `tauri_specta`. Listen directly to generated events and prefer the typed interfaces. -- Windowing/permissions are handled in Rust; keep UI logic in Solid and avoid mixing IPC with rendering logic. +- `apps/web/actions/**/*.ts` — Server Actions ("use server") +- `packages/database/schema.ts` — Database schema +- `apps/web/app/api/*` — API routes (Effect-based) +- `packages/web-backend/src/` — Backend services (Videos, Users, S3Buckets, Folders, etc.) +- `packages/web-domain/` — Shared domain types +- `apps/web/lib/server.ts` — Effect runtime and `apiToHandler` utility ## Conventions -- **CRITICAL: NO CODE COMMENTS**: Never add any form of comments to code. This includes: - - Single-line comments: `//` (JavaScript/TypeScript/Rust), `#` (Python/Shell) - - Multi-line comments: `/* */` (JavaScript/TypeScript), `/* */` (Rust) - - Documentation comments: `///`, `//!` (Rust), `/** */` (JSDoc) - - Any other comment syntax in any language - - Code must be self-explanatory through naming, types, and structure. Use docs/READMEs for explanations when necessary. -- Directory naming: lower-case-dashed -- Components: PascalCase; hooks: camelCase starting with `use` -- Strict TypeScript; avoid `any`; leverage shared types -- Use Biome for linting/formatting; match existing formatting - -## Security & Privacy Considerations - -### Data Handling -- **Video Storage**: S3-compatible storage with signed URLs -- **Database**: MySQL with connection pooling via PlanetScale -- **Authentication**: NextAuth with custom Drizzle adapter -- **API Security**: CORS policies, rate limiting via Hono middleware - -### Privacy Controls -- **Recording Permissions**: Platform-specific (macOS Screen Recording, Windows) -- **Data Retention**: User-controlled deletion of recordings -- **Sharing Controls**: Password protection, expiry dates on shared links -- **Analytics**: PostHog with privacy-focused configuration - -## AI & Processing Pipeline - -### AI Integration Points -- **Transcription**: Deepgram API for captions generation -- **Metadata Generation**: Groq (primary) + OpenAI (fallback) for titles/descriptions -- **Processing Location**: All AI calls in Next.js Server Actions only -- **Privacy**: Transcripts stored in database, audio sent to external APIs - -### Media Processing Flow -``` -Desktop Recording → Local Files → Upload to S3 → -Background Processing (tasks service) → -Transcription/AI Enhancement → Database Storage -``` - -## References & Documentation - -### Core Technologies -- **TanStack Query**: https://tanstack.com/query/latest -- **React Patterns**: https://react.dev/learn/you-might-not-need-an-effect -- **Tauri v2**: https://github.com/tauri-apps/tauri -- **tauri_specta**: https://github.com/oscartbeaumont/tauri-specta -- **Drizzle ORM**: https://orm.drizzle.team/ -- **SolidJS**: https://solidjs.com/ -### Cap-Specific -- **Self-hosting**: https://cap.so/docs/self-hosting -- **API Documentation**: Generated from TypeScript contracts -- **Architecture Decisions**: See individual package READMEs +- **Directory naming**: lower-case-dashed +- **Components**: PascalCase +- **Hooks**: camelCase starting with `use` +- **Rust modules**: snake_case +- **Rust crates**: kebab-case +- **Files**: kebab-case (`user-menu.tsx`) +- Strict TypeScript; avoid `any` +- Use Biome for TS/JS; rustfmt for Rust -### Development Resources -- **Monorepo Guide**: Turborepo documentation -- **Effect System**: Used in web-backend packages -- **Media Processing**: FFmpeg documentation for Rust bindings +## Troubleshooting -## Code Formatting - -Always format code before completing work: -- **TypeScript/JavaScript**: Run `pnpm format` to format all code with Biome -- **Rust**: Run `cargo fmt` to format all Rust code with rustfmt - -These commands should be run regularly during development and always at the end of a coding session to ensure consistent formatting across the codebase. +- **Turbo cache issues**: `rm -rf .turbo` +- **IPC binding errors**: Restart dev server to regenerate `tauri.ts` +- **Node version**: Must be 20+ +- **Clean rebuild**: `pnpm clean` diff --git a/Cargo.lock b/Cargo.lock index 4dde011982..6ab5fceef4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,109 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "Inflight" +version = "0.4.21" +dependencies = [ + "anyhow", + "async-stream", + "axum", + "base64 0.22.1", + "bytemuck", + "bytes", + "cap-audio", + "cap-camera", + "cap-editor", + "cap-export", + "cap-fail", + "cap-flags", + "cap-media", + "cap-project", + "cap-recording", + "cap-rendering", + "cap-utils", + "chrono", + "cidre", + "clipboard-rs", + "cocoa", + "core-foundation 0.10.1", + "core-graphics 0.24.0", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", + "device_query", + "dirs", + "dotenvy_macro", + "ffmpeg-next", + "flume", + "futures", + "futures-intrusive", + "global-hotkey", + "image 0.25.8", + "kameo", + "keyed_priority_queue", + "lazy_static", + "log", + "md5", + "nix 0.29.0", + "objc", + "objc2-app-kit", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "png 0.17.16", + "posthog-rs", + "rand 0.8.5", + "relative-path", + "reqwest 0.12.24", + "rodio", + "scap-direct3d", + "scap-screencapturekit", + "scap-targets", + "sentry", + "serde", + "serde_json", + "specta", + "specta-typescript", + "swift-rs", + "tauri", + "tauri-build", + "tauri-nspanel", + "tauri-plugin-clipboard-manager", + "tauri-plugin-deep-link", + "tauri-plugin-dialog", + "tauri-plugin-fs", + "tauri-plugin-global-shortcut", + "tauri-plugin-http", + "tauri-plugin-notification", + "tauri-plugin-oauth", + "tauri-plugin-opener", + "tauri-plugin-os", + "tauri-plugin-positioner", + "tauri-plugin-process", + "tauri-plugin-sentry", + "tauri-plugin-shell", + "tauri-plugin-single-instance", + "tauri-plugin-store", + "tauri-plugin-updater", + "tauri-plugin-window-state", + "tauri-specta", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "tracing-appender", + "tracing-futures", + "tracing-opentelemetry", + "tracing-subscriber", + "uuid", + "wgpu", + "whisper-rs", + "windows 0.60.0", + "windows-sys 0.59.0", + "workspace-hack", +] + [[package]] name = "actix-codec" version = "0.5.2" @@ -1170,109 +1273,6 @@ dependencies = [ "workspace-hack", ] -[[package]] -name = "cap-desktop" -version = "0.4.0" -dependencies = [ - "anyhow", - "async-stream", - "axum", - "base64 0.22.1", - "bytemuck", - "bytes", - "cap-audio", - "cap-camera", - "cap-editor", - "cap-export", - "cap-fail", - "cap-flags", - "cap-media", - "cap-project", - "cap-recording", - "cap-rendering", - "cap-utils", - "chrono", - "cidre", - "clipboard-rs", - "cocoa", - "core-foundation 0.10.1", - "core-graphics 0.24.0", - "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=3cc779a7b4ca)", - "device_query", - "dirs", - "dotenvy_macro", - "ffmpeg-next", - "flume", - "futures", - "futures-intrusive", - "global-hotkey", - "image 0.25.8", - "kameo", - "keyed_priority_queue", - "lazy_static", - "log", - "md5", - "nix 0.29.0", - "objc", - "objc2-app-kit", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", - "png 0.17.16", - "posthog-rs", - "rand 0.8.5", - "relative-path", - "reqwest 0.12.24", - "rodio", - "scap-direct3d", - "scap-screencapturekit", - "scap-targets", - "sentry", - "serde", - "serde_json", - "specta", - "specta-typescript", - "swift-rs", - "tauri", - "tauri-build", - "tauri-nspanel", - "tauri-plugin-clipboard-manager", - "tauri-plugin-deep-link", - "tauri-plugin-dialog", - "tauri-plugin-fs", - "tauri-plugin-global-shortcut", - "tauri-plugin-http", - "tauri-plugin-notification", - "tauri-plugin-oauth", - "tauri-plugin-opener", - "tauri-plugin-os", - "tauri-plugin-positioner", - "tauri-plugin-process", - "tauri-plugin-sentry", - "tauri-plugin-shell", - "tauri-plugin-single-instance", - "tauri-plugin-store", - "tauri-plugin-updater", - "tauri-plugin-window-state", - "tauri-specta", - "tempfile", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "tracing-appender", - "tracing-futures", - "tracing-opentelemetry", - "tracing-subscriber", - "uuid", - "wgpu", - "whisper-rs", - "windows 0.60.0", - "windows-sys 0.59.0", - "workspace-hack", -] - [[package]] name = "cap-editor" version = "0.1.0" diff --git a/FLYOVER_FEATURE_PLAN.md b/FLYOVER_FEATURE_PLAN.md new file mode 100644 index 0000000000..6c7b472d7a --- /dev/null +++ b/FLYOVER_FEATURE_PLAN.md @@ -0,0 +1,1808 @@ +# Flyover Camera Feature - Architecture Analysis & Implementation Plan + +## Table of Contents +1. [Project Overview](#project-overview) +2. [Architecture Analysis](#architecture-analysis) +3. [Current System Deep Dive](#current-system-deep-dive) +4. [Feature Requirements](#feature-requirements) +5. [Implementation Plan](#implementation-plan) +6. [Technical Decisions](#technical-decisions) +7. [Risks & Mitigations](#risks--mitigations) + +--- + +## Project Overview + +### Feature Goal +Add a "flyover" camera mode to Cap's instant recordings where the user's camera video dynamically follows cursor movements during both recording preview and playback. Users should be able to toggle between flyover mode (camera follows cursor) and fixed position mode (camera in bottom-right corner). + +### Key Requirements +- **Mode**: Instant mode (not studio mode) +- **Real-time Preview**: Camera follows cursor during recording in desktop app +- **Playback Format**: Separate video tracks (display.mp4, camera.mp4) + cursor.json uploaded to web +- **Editable**: Users can adjust flyover settings after recording +- **Toggle**: Switch between flyover and fixed position modes + +--- + +## Architecture Analysis + +### Current Recording Modes + +#### Instant Mode (`crates/recording/src/instant_recording.rs`) +**Current Behavior**: +- Single-file output: `content/output.mp4` +- Real-time compositing during recording +- Screen + system audio + microphone → single MP4 +- **Camera is NOT recorded** (captured but not encoded) +- Designed for fast sharing with immediate upload + +**Key Code References**: +```rust +// Line 302: Camera explicitly disabled +camera_feed: None + +// Lines 220-229: Single pipeline creation +let output = ScreenCaptureMethod::make_instant_mode_pipeline( + screen_capture, + system_audio, + mic_feed, + output_path.clone(), + output_resolution, + encoder_preferences, +).await? +``` + +**Output Structure**: +``` +project/ + content/ + output.mp4 ← Single composited video + recording-meta.json +``` + +#### Studio Mode (`crates/recording/src/studio_recording.rs`) +**Current Behavior**: +- **Separate streams** for each source +- Screen, camera, microphone, system audio recorded independently +- Multiple segments support (pause/resume) +- **Comprehensive cursor tracking** (position + clicks) +- Post-processing rendering pipeline for final output + +**Key Code References**: +```rust +// Lines 816-892: Separate pipelines for each source +Pipeline { + screen: OutputPipeline, // display.mp4 + camera: Option, // camera.mp4 (separate!) + microphone: Option, + system_audio: Option, + cursor: Option, +} +``` + +**Output Structure**: +``` +project/ + content/ + segments/ + segment-0/ + display.mp4 ← Screen recording + camera.mp4 ← Camera feed (SEPARATE!) + audio-input.ogg ← Microphone + system_audio.ogg ← System audio + cursor.json ← Cursor events + cursors/ + cursor_0.png ← Cursor image assets + cursor_1.png + recording-meta.json + cap-project.json +``` + +### Key Finding: Streams Are Already Separate in Studio Mode ✅ + +Studio mode demonstrates the pattern we need: +1. Screen and camera recorded to separate MP4 files +2. Cursor positions tracked at 100Hz (10ms intervals) +3. Post-processing compositor positions camera over screen +4. All data available for flexible editing + +--- + +## Current System Deep Dive + +### Cursor Tracking System (`crates/recording/src/cursor.rs`) + +**Already Implemented in Studio Mode** ✅ + +**Capture Details**: +- Polling frequency: 10ms (100 Hz) - Line 80 +- Position normalization: 0.0-1.0 coordinates (relative to screen) +- Click tracking: Down/Up events with timestamps +- Cursor image capture: PNG files for different cursor shapes +- Modifier keys: Ctrl, Shift, Alt, etc. + +**Data Structures**: +```rust +pub struct CursorMoveEvent { + cursor_id: String, + time_ms: f64, // Milliseconds since recording start + x: f64, // Normalized 0.0-1.0 + y: f64, // Normalized 0.0-1.0 + active_modifiers: Vec, +} + +pub struct CursorClickEvent { + down: bool, + cursor_num: u8, + cursor_id: String, + time_ms: f64, +} + +pub struct Cursor { + pub file_name: String, // cursor_N.png + pub id: u32, + pub hotspot: XY, // Click point offset + pub shape: Option, +} +``` + +**Output Format** (`cursor.json`): +```json +{ + "moves": [ + { + "active_modifiers": [], + "cursor_id": "0", + "time_ms": 123.45, + "x": 0.5234, + "y": 0.6789 + } + ], + "clicks": [ + { + "down": true, + "cursor_num": 0, + "cursor_id": "0", + "time_ms": 456.78 + } + ] +} +``` + +**Reusability**: Can be directly reused in instant mode with minimal changes. + +### Camera Recording System (`crates/recording/src/feeds/camera.rs`) + +**Actor-Based Architecture**: +```rust +pub struct CameraFeed { + state: State, // Open or Locked + senders: Vec>, + on_ready: Vec>, +} +``` + +**Capture Flow**: +1. Platform-specific camera APIs: + - macOS: AVFoundation (`camera-avfoundation`) + - Windows: Media Foundation (`camera-mediafoundation`) + - Fallback: FFmpeg (`camera-ffmpeg`) +2. Format selection prioritizes: + - Frame rate ≥ 30 FPS + - Resolution < 2000x2000 + - 16:9 aspect ratio +3. Native capture → FFmpeg frames → H.264 encoding +4. Output to separate `camera.mp4` file + +**Studio Mode Integration**: +```rust +// Lines 827-836 in studio_recording.rs +let camera = OptionFuture::from(base_inputs.camera_feed.map(|camera_feed| { + OutputPipeline::builder(dir.join("camera.mp4")) + .with_video::(camera_feed) + .with_timestamps(start_time) + .build::(()) +})) +``` + +### Rendering Pipeline (`crates/rendering/`) + +**GPU-Accelerated Compositor** built on WGPU. + +**Layer Architecture**: +1. **Background Layer** (`background.rs`) - Solid color or gradient +2. **Display Layer** (`display.rs`) - Screen recording video +3. **Camera Layer** (`camera.rs`) - Separate video texture with position/size/opacity control +4. **Cursor Layer** (`cursor.rs`) - Cursor images with interpolated motion +5. **Captions Layer** (`captions.rs`) - AI-generated subtitles + +**Camera Layer Key Features**: +```rust +pub struct CameraLayer { + frame_texture: wgpu::Texture, + uniforms_buffer: wgpu::Buffer, + hidden: bool, +} + +pub fn prepare( + &mut self, + data: Option<(CompositeVideoFrameUniforms, XY, &DecodedFrame)> +) { + // CompositeVideoFrameUniforms includes: + // - position (x, y in screen space) + // - size (width, height) + // - opacity + // - corner_radius +} +``` + +**Cursor Rendering** (`cursor.rs`, `cursor_interpolation.rs`): +- Spring-mass-damper physics for smooth motion +- Motion blur based on velocity +- Click animations (shrink effect) +- Three spring profiles: Default, Snappy (near clicks), Drag (button held) + +**Coordinate Systems** (`coord.rs`): +```rust +pub struct RawDisplayUVSpace; // Normalized 0.0-1.0 +pub struct FrameSpace; // Output video pixels +pub struct ZoomedFrameSpace; // After zoom transform +``` + +### Preview System (`apps/desktop/src/routes/editor/Player.tsx`) + +**Current Implementation**: +- 2D Canvas element (line 426-437) +- Rust GPU renderer generates frames +- Frames sent via IPC as `ImageData` +- Canvas displays with `ctx.putImageData(frame.data, 0, 0)` (line 373) +- `renderFrameEvent` triggers frame generation +- 30-60 FPS rendering + +**Frame Pipeline**: +``` +SolidJS Component → renderFrameEvent (frame number) + ↓ +Rust editor_instance.rs → Handle event + ↓ +cap_rendering crate → GPU composition + ↓ +Return RenderedFrame with ImageData + ↓ +Canvas display +``` + +--- + +## Feature Requirements + +Based on user answers to clarifying questions: + +1. **Recording Mode**: Instant mode (current default) +2. **Live Preview**: Yes - camera follows cursor in real-time during recording +3. **Playback Format**: Separate tracks (display.mp4, camera.mp4, cursor.json) +4. **Post-Recording Editing**: Yes - users can adjust flyover settings after recording + +### Functional Requirements + +**Desktop App (Recording)**: +- Record screen, camera, and cursor position separately +- Real-time preview shows camera following cursor during recording +- Toggle between flyover mode and fixed position mode +- Configurable offset (camera position relative to cursor) +- Smooth motion with spring physics +- Output multiple files: display.mp4, camera.mp4, cursor.json + +**Web App (Playback)**: +- Upload multiple video tracks + cursor data +- Custom player with synchronized video elements +- Camera position calculated from cursor.json in real-time +- Smooth interpolation between cursor events +- Editable flyover settings (offset, smoothing, enable/disable) +- Preview changes without re-rendering video + +**Web App (Editor)**: +- Toggle flyover on/off +- Adjust camera offset from cursor +- Adjust smoothing strength +- Real-time preview of changes +- Save settings to database +- Optional: Re-render video with baked camera positions + +--- + +## Implementation Plan + +### Phase 1: Desktop Recording Infrastructure (Rust) + +#### 1.1 Add Camera Recording to Instant Mode + +**Files to Modify**: +- `crates/recording/src/instant_recording.rs` +- `crates/recording/src/capture_pipeline.rs` + +**Changes**: +1. Update `ActorBuilder` to accept camera feed +```rust +pub fn with_camera_feed(mut self, camera_feed: Arc) -> Self { + self.camera_feed = Some(camera_feed); + self +} +``` + +2. Modify line 302 to use camera feed instead of `None`: +```rust +RecordingBaseInputs { + capture_target: self.capture_target, + capture_system_audio: self.system_audio, + mic_feed: self.mic_feed, + camera_feed: self.camera_feed, // Change from None + ... +} +``` + +3. Update `create_pipeline` to create separate camera output: +```rust +let camera = if let Some(camera_feed) = base_inputs.camera_feed { + Some(OutputPipeline::builder(content_dir.join("camera.mp4")) + .with_video::(camera_feed) + .with_timestamps(start_time) + .build::(()) + .await?) +} else { + None +} +``` + +4. Update `make_instant_mode_pipeline` trait to accept optional camera parameter + +**Output**: Instant recordings produce `display.mp4` and `camera.mp4` separately + +#### 1.2 Add Cursor Tracking to Instant Mode + +**Files to Modify**: +- `crates/recording/src/instant_recording.rs` +- Reuse: `crates/recording/src/cursor.rs` (no changes needed) + +**Changes**: +1. Add cursor recorder to pipeline creation: +```rust +let cursor = if enable_cursor { + let cursor_crop_bounds = target.cursor_crop() + .ok_or_else(|| anyhow!("No cursor bounds"))?; + + let cursor = spawn_cursor_recorder( + cursor_crop_bounds, + display, + content_dir.join("cursors"), + HashMap::new(), // prev_cursors + 0, // next_cursor_id + start_time, + ); + + Some(CursorPipeline { + output_path: content_dir.join("cursor.json"), + actor: cursor, + }) +} else { + None +} +``` + +2. Add cursor directory creation +3. Update Pipeline struct to include cursor + +**Output**: `cursor.json` with 100Hz position data, cursor images in `cursors/` directory + +#### 1.3 Update Metadata Format + +**Files to Modify**: +- `crates/project/src/meta.rs` + +**Changes**: +1. Add new variant to `InstantRecordingMeta`: +```rust +pub enum InstantRecordingMeta { + InProgress { recording: bool }, + Failed { error: String }, + Complete { + fps: u32, + sample_rate: Option + }, + // NEW: Multi-track format + MultiTrack { + display: VideoTrackMeta, + camera: Option, + mic: Option, + system_audio: Option, + cursor: Option, + fps: u32, + } +} + +pub struct VideoTrackMeta { + pub path: RelativePathBuf, + pub fps: u32, + pub resolution: (u32, u32), + pub start_time: f64, +} +``` + +2. Update serialization/deserialization +3. Maintain backward compatibility with legacy `output.mp4` format + +**Output Structure**: +``` +project/ + content/ + display.mp4 ← Screen only + camera.mp4 ← Camera only + audio-input.ogg ← Mic audio + system_audio.ogg ← System audio (optional) + cursor.json ← Cursor data + cursors/ ← Cursor images + cursor_0.png + recording-meta.json ← Updated format +``` + +--- + +### Phase 2: Real-time Preview Compositor (Rust + SolidJS) + +#### 2.1 Camera Follow Position Calculator + +**New File**: `crates/rendering/src/camera_follow.rs` + +**Implementation**: +```rust +use crate::coord::{Coord, RawDisplayUVSpace}; +use crate::spring_mass_damper::{SpringMassDamperSimulationConfig, SpringMassDamper}; + +pub struct CameraFollowConfig { + pub enabled: bool, + pub offset: XY, // Offset from cursor (e.g., 150px right, 150px down) + pub camera_size: XY, // Camera dimensions + pub smoothing: SpringMassDamperSimulationConfig, + pub boundary_padding: f64, // Padding from screen edges +} + +pub struct CameraFollowState { + position: SpringMassDamper>, + config: CameraFollowConfig, +} + +impl CameraFollowState { + pub fn update( + &mut self, + cursor_pos: Coord, + delta_time: f64, + ) -> Coord { + let target_pos = self.calculate_target_position(cursor_pos); + self.position.update(target_pos, delta_time); + + // Apply boundary constraints + self.constrain_to_bounds(self.position.position()) + } + + fn calculate_target_position(&self, cursor_pos: Coord) -> XY { + // Camera position = cursor position + offset + let target = XY { + x: cursor_pos.x() + self.config.offset.x, + y: cursor_pos.y() + self.config.offset.y, + }; + target + } + + fn constrain_to_bounds(&self, pos: XY) -> Coord { + // Ensure camera stays within screen bounds (0.0 - 1.0) + // with padding for camera size + let constrained = XY { + x: pos.x.clamp( + self.config.boundary_padding, + 1.0 - self.config.camera_size.x - self.config.boundary_padding + ), + y: pos.y.clamp( + self.config.boundary_padding, + 1.0 - self.config.camera_size.y - self.config.boundary_padding + ), + }; + Coord::from_xy(constrained) + } +} +``` + +**Features**: +- Spring-mass-damper smoothing (reuse existing implementation) +- Configurable offset from cursor +- Boundary checking to keep camera on-screen +- Smooth transitions + +#### 2.2 Extend GPU Renderer for Live Camera Overlay + +**Files to Modify**: +- `crates/rendering/src/lib.rs` +- `crates/rendering/src/layers/camera.rs` + +**Changes to `lib.rs`**: +1. Add preview mode flag to renderer +2. Accept live camera feed and cursor position +3. Calculate camera position using `CameraFollowState` +4. Update `ProjectUniforms` to include camera follow config + +**Changes to `camera.rs`**: +1. Support real-time camera feed (not just pre-recorded video) +2. Accept dynamic position parameter +3. Update uniforms buffer with new position each frame + +**Implementation**: +```rust +// In rendering loop +let cursor_position = get_current_cursor_position(); +let camera_position = camera_follow_state.update(cursor_position, delta_time); + +// Prepare camera layer with dynamic position +camera_layer.prepare(Some(( + CompositeVideoFrameUniforms { + position: camera_position, + size: camera_config.size, + opacity: 1.0, + corner_radius: camera_config.corner_radius, + }, + camera_frame.size, + &camera_frame, +))); +``` + +**Output**: Composited frames with camera overlay at cursor-following position + +#### 2.3 Update Preview Frame Generation + +**Files to Modify**: +- `apps/desktop/src-tauri/src/editor_instance.rs` +- `apps/desktop/src-tauri/src/recording.rs` + +**Changes**: +1. Pass camera feed to renderer during recording +2. Pass current cursor position to renderer +3. Enable camera overlay for instant mode preview +4. Stream composited frames to frontend at 30-60 FPS + +**IPC Event**: +```rust +#[derive(Serialize, Type, tauri_specta::Event, Debug, Clone)] +pub struct PreviewFrameReady { + frame: ImageData, + timestamp: f64, +} +``` + +#### 2.4 Desktop UI Controls + +**Files to Modify**: +- `apps/desktop/src/routes/editor/Player.tsx` +- New: `apps/desktop/src/components/CameraFollowControls.tsx` + +**UI Components**: +1. Toggle switch: "Flyover Mode" vs "Fixed Position" +2. Offset controls: + - X offset slider (-500 to 500 pixels) + - Y offset slider (-500 to 500 pixels) +3. Smoothing strength slider (0.1 to 1.0) +4. Camera size dropdown (Small, Medium, Large) +5. Preview indicator showing camera will follow cursor + +**State Management**: +```typescript +const [cameraMode, setCameraMode] = createSignal<'flyover' | 'fixed'>('fixed'); +const [offset, setOffset] = createSignal({ x: 150, y: 150 }); +const [smoothing, setSmoothing] = createSignal(0.5); + +// Send config to Rust via IPC +createEffect(() => { + commands.updateCameraFollowConfig({ + enabled: cameraMode() === 'flyover', + offset: offset(), + smoothing: smoothing(), + }); +}); +``` + +**Real-time Preview**: +- Canvas receives composited frames from Rust +- Settings updates trigger immediate re-render +- Visual feedback of camera position + +--- + +### Phase 3: Web Upload & Storage + +#### 3.1 Multi-File Upload Pipeline + +**Files to Modify**: +- `apps/web/actions/video/upload.ts` (or create new Server Action) +- `apps/web/lib/s3.ts` + +**Changes**: +1. Create upload function for multiple files: +```typescript +"use server"; + +export async function uploadMultiTrackVideo(data: { + videoId: string; + displayFile: File; + cameraFile: File; + cursorData: string; // JSON string + metadata: VideoMetadata; +}) { + const s3 = getS3Client(); + + // Upload files concurrently + const [displayUrl, cameraUrl, cursorUrl] = await Promise.all([ + s3.upload({ + key: `videos/${data.videoId}/display.mp4`, + file: data.displayFile, + }), + s3.upload({ + key: `videos/${data.videoId}/camera.mp4`, + file: data.cameraFile, + }), + s3.upload({ + key: `videos/${data.videoId}/cursor.json`, + file: new Blob([data.cursorData], { type: 'application/json' }), + }), + ]); + + // Update database with all URLs + await db.update(videos).set({ + videoPath: displayUrl, + cameraVideoPath: cameraUrl, + cursorDataPath: cursorUrl, + status: 'ready', + }).where(eq(videos.id, data.videoId)); + + return { success: true, videoId: data.videoId }; +} +``` + +2. Progress tracking for all uploads +3. Atomic commit (only mark ready when all files uploaded) +4. Error handling and retry logic + +#### 3.2 Database Schema Updates + +**Files to Modify**: +- `packages/database/schema.ts` + +**Schema Changes**: +```typescript +export const videos = mysqlTable("videos", { + id: varchar("id", { length: 26 }).primaryKey(), + + // Existing fields + videoPath: text("videoPath").notNull(), + + // NEW: Multi-track support + cameraVideoPath: text("cameraVideoPath"), // camera.mp4 URL + cursorDataPath: text("cursorDataPath"), // cursor.json URL + recordingMode: varchar("recordingMode", { + length: 20, + enum: ["instant", "studio"] + }).default("instant"), + + // Camera follow settings + cameraFollowEnabled: boolean("cameraFollowEnabled").default(false), + cameraFollowOffset: json("cameraFollowOffset").$type<{ x: number; y: number }>(), + cameraFollowSmoothing: float("cameraFollowSmoothing").default(0.5), + + // ... other existing fields +}); +``` + +**Migration Script**: +```sql +ALTER TABLE videos + ADD COLUMN cameraVideoPath TEXT, + ADD COLUMN cursorDataPath TEXT, + ADD COLUMN recordingMode VARCHAR(20) DEFAULT 'instant', + ADD COLUMN cameraFollowEnabled BOOLEAN DEFAULT FALSE, + ADD COLUMN cameraFollowOffset JSON, + ADD COLUMN cameraFollowSmoothing FLOAT DEFAULT 0.5; +``` + +**Run Migration**: +```bash +pnpm db:generate +pnpm db:push +``` + +#### 3.3 S3 Storage Structure + +**New Structure**: +``` +s3://cap-bucket/ + videos/ + {videoId}/ + display.mp4 ← Screen recording + camera.mp4 ← Camera recording + cursor.json ← Cursor position data + thumbnail.jpg ← Thumbnail (existing) +``` + +**Backward Compatibility**: +- Old recordings: `videos/{videoId}.mp4` (single file) +- New recordings: `videos/{videoId}/display.mp4` (multi-track) +- Check for `cameraVideoPath` to determine format + +--- + +### Phase 4: Web Video Player + +#### 4.1 Multi-Track Player Component + +**New File**: `apps/web/components/VideoPlayer/MultiTrackPlayer.tsx` + +**Implementation**: +```typescript +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { useCursorPositioning } from "./useCursorPositioning"; + +interface MultiTrackPlayerProps { + displayVideoUrl: string; + cameraVideoUrl: string; + cursorDataUrl: string; + cameraFollowConfig: { + enabled: boolean; + offset: { x: number; y: number }; + smoothing: number; + }; +} + +export function MultiTrackPlayer({ + displayVideoUrl, + cameraVideoUrl, + cursorDataUrl, + cameraFollowConfig, +}: MultiTrackPlayerProps) { + const displayRef = useRef(null); + const cameraRef = useRef(null); + const containerRef = useRef(null); + + const [currentTime, setCurrentTime] = useState(0); + const [isPlaying, setIsPlaying] = useState(false); + + // Load and process cursor data + const { getCameraPosition } = useCursorPositioning( + cursorDataUrl, + cameraFollowConfig + ); + + // Sync both videos + useEffect(() => { + const display = displayRef.current; + const camera = cameraRef.current; + if (!display || !camera) return; + + const syncVideos = () => { + const timeDiff = Math.abs(display.currentTime - camera.currentTime); + + // If drift > 50ms, correct it + if (timeDiff > 0.05) { + camera.currentTime = display.currentTime; + } + + setCurrentTime(display.currentTime); + }; + + display.addEventListener('timeupdate', syncVideos); + display.addEventListener('play', () => { + camera.play(); + setIsPlaying(true); + }); + display.addEventListener('pause', () => { + camera.pause(); + setIsPlaying(false); + }); + display.addEventListener('seeked', () => { + camera.currentTime = display.currentTime; + }); + + return () => { + display.removeEventListener('timeupdate', syncVideos); + }; + }, []); + + // Update camera position based on cursor data + useEffect(() => { + if (!cameraRef.current || !containerRef.current) return; + + let animationFrame: number; + + const updatePosition = () => { + const position = getCameraPosition(currentTime); + + if (position && cameraRef.current) { + const container = containerRef.current!; + const containerRect = container.getBoundingClientRect(); + + // Convert normalized position (0-1) to pixels + const x = position.x * containerRect.width; + const y = position.y * containerRect.height; + + cameraRef.current.style.transform = `translate(${x}px, ${y}px)`; + } + + animationFrame = requestAnimationFrame(updatePosition); + }; + + if (isPlaying) { + updatePosition(); + } + + return () => cancelAnimationFrame(animationFrame); + }, [currentTime, isPlaying, getCameraPosition]); + + return ( +
+ {/* Screen video (base layer) */} +
+ ); +} +``` + +**Key Features**: +- Two synchronized `