From 9d8058cfcbaf788d50044f80684e81c9324b2457 Mon Sep 17 00:00:00 2001 From: csupenn Date: Sat, 26 Sep 2026 16:26:34 -0700 Subject: [PATCH] chore(ci): enforce honest coverage thresholds; refresh stale contributor docs Coverage (H6): the old 75/70/75/75 global threshold was never met (actual ~17%) and CI hid it with continue-on-error. - jest: per-file thresholds hold the security core (ssrf, rate-limit, workflow-graph, urw) and the execute-workflow route near their current 90-100%; global is a ratchet floor at today's measured level - ci: single blocking `pnpm test:ci` step replaces test + non-blocking coverage - verified: dropping ssrf tests fails the ssrf threshold Docs (H5): CONTRIBUTING project structure / node-type steps / Node 22 now match the code (builder in app/builder, validation + codegen in @charliesu/workflow-core, security controls in lib/security); remove link to a private docs path; TESTING.md, testing-guide.md and ci.md show real coverage numbers and the enforced policy. eslint: ignore generated coverage/ and Playwright report dirs. --- .github/workflows/ci.yml | 9 ++-- CONTRIBUTING.md | 73 ++++++++++++++++++++----------- TESTING.md | 56 ++++++++++++------------ docs/development/ci.md | 3 +- docs/development/testing-guide.md | 25 ++++------- eslint.config.mjs | 3 ++ jest.config.js | 18 +++++--- 7 files changed, 105 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00d2ede..9617bce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,12 +83,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Run tests - run: pnpm test - - - name: Generate coverage report - run: pnpm test:coverage - continue-on-error: true + # Single blocking run: tests + coverage thresholds (see jest.config.js). + - name: Run tests with coverage thresholds + run: pnpm test:ci - name: Upload coverage to artifacts uses: actions/upload-artifact@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a51cd89..17a340b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -135,7 +135,7 @@ We currently do not have a bug bounty program, but we will publicly acknowledge ### Prerequisites -- **Node.js**: 18+ (we recommend using [nvm](https://github.com/nvm-sh/nvm)) +- **Node.js**: 22 LTS (matches CI; we recommend [nvm](https://github.com/nvm-sh/nvm)) - **pnpm**: 9+ (`npm install -g pnpm`) - **Git**: Latest version - **AI Provider API Key**: OpenAI, Anthropic, Google, or Groq (for testing) @@ -170,24 +170,38 @@ Open [http://localhost:3000](http://localhost:3000) to see the app. ``` topflow/ -├── app/ # Next.js App Router -│ ├── page.tsx # Main application orchestrator -│ ├── layout.tsx # Root layout -│ └── api/ # API routes -│ └── execute-workflow/ # Workflow execution engine -├── components/ # React components -│ ├── nodes/ # Workflow node types -│ ├── advanced-editors/ # Complex node editors -│ └── ui/ # shadcn/ui components -├── lib/ # Business logic -│ ├── validation.ts # Workflow validation + SSRF prevention -│ ├── code-generator.ts # TypeScript code export -│ └── storage.ts # localStorage abstraction -├── hooks/ # React hooks -├── docs/ # Documentation -└── public/ # Static assets +├── app/ # Next.js App Router +│ ├── page.tsx # Redirects to /builder +│ ├── builder/page.tsx # Workflow builder — canvas, panels, dialogs, main state +│ ├── showcase/security-scanner # GitHub Security Scanner demo +│ ├── blog/, docs/, about/ # Content pages +│ └── api/ +│ ├── execute-workflow/ # Streaming execution route (rate limit, cycle check, validation) +│ ├── scan/github/ # Real OSV-backed repository scan +│ └── badge/ # Security badge API +├── components/ +│ ├── nodes/ # Workflow node components +│ ├── advanced-editors/ # Prompt / HTTP / conditional / schema editors +│ ├── node-config-panel.tsx # Per-node configuration UI +│ ├── node-palette.tsx # Draggable node library +│ └── ui/ # shadcn/ui primitives +├── lib/ +│ ├── security/ # SSRF guard, cycle detection, rate limiting, +│ │ # key encryption, URW trust boundary, validation engine +│ ├── osv/ # OSV.dev vulnerability scanning +│ ├── topflow-execution-engine.ts # Extends the workflow-core engine (scanner, URW) +│ ├── templates/, security-templates.ts # Workflow templates +│ ├── demo-mode.ts, demo-data/ # Demo (no-API-key) execution +│ └── storage.ts # localStorage abstraction +├── hooks/ # React hooks +├── e2e/ # Playwright tests +└── docs/ # Architecture, development, and tutorial docs ``` +Core workflow logic — base execution engine, validation, code generation, and node +utilities — lives in the [`@charliesu/workflow-core`](https://www.npmjs.com/package/@charliesu/workflow-core) +package (see below). + ### Development Commands ```bash @@ -387,7 +401,7 @@ main → Protected branch for stable releases (maintainers only) - **Components**: kebab-case files, PascalCase exports - `text-model-node.tsx` exports `TextModelNode` - **Libraries**: kebab-case - - `workflow-store.ts`, `code-generator.ts` + - `workflow-store.ts`, `topflow-execution-engine.ts` - **Types**: Export from same file as implementation ### Security Guidelines @@ -424,6 +438,13 @@ For security-related changes: - [ ] **Rate Limiting**: Verify rate limits can't be bypassed - [ ] **Input Validation**: Test with edge cases (empty, null, special chars) - [ ] **Authentication**: Verify auth/authz logic is correct +- [ ] **Logging**: No user-supplied content (inputs, prompts, URLs, keys) in server logs + (`app/api/execute-workflow/__tests__/log-privacy.test.ts` guards the execution path) + +### Automated Checks + +CI runs lint, type-check, `pnpm test:ci` (tests + **blocking** coverage thresholds), and build. +Run them locally before pushing: `pnpm lint && pnpm type-check && pnpm test:ci && pnpm build`. ### Workflow Testing @@ -512,12 +533,14 @@ CVE-2026-XXXXX 1. **Create node component** in `components/nodes/new-node.tsx` 2. **Define data interface**: `NewNodeData` with typed fields -3. **Register in page.tsx**: Add to `nodeTypes` object -4. **Add to NodePalette**: Include icon and description -5. **Implement configuration UI**: Add case to `NodeConfigPanel` -6. **Implement execution logic**: Add case to `app/api/execute-workflow/route.ts` -7. **Implement code generation**: Add case to `lib/code-generator.ts` -8. **Add validation rules**: Update `lib/validation.ts` if needed +3. **Register in `app/builder/page.tsx`**: Add to the `nodeTypes` object +4. **Add to NodePalette** (`components/node-palette.tsx`): Include icon and description +5. **Implement configuration UI**: Add case to `components/node-config-panel.tsx` +6. **Implement execution logic**: Add a case to `executeNode()` in + `lib/topflow-execution-engine.ts` (or in `@charliesu/workflow-core` if it is a general node) +7. **Implement code generation / validation rules**: in `@charliesu/workflow-core` + (`lib/code-generator`, `lib/validation`); TopFlow-specific security checks go in `lib/security/` +8. **Write tests**: unit tests beside the code in `__tests__/`; security logic tests first 9. **Write documentation**: Add usage guide to docs 10. **Test thoroughly**: Manual testing + edge cases @@ -526,7 +549,7 @@ CVE-2026-XXXXX 1. **Design workflow**: Map out node graph and data flow 2. **Build in UI**: Create workflow in TopFlow builder 3. **Test execution**: Verify it works end-to-end -4. **Document specification**: Create detailed spec document (see `docs/repositioning-proposal/00-gdpr-ultimate-plan/` for examples) +4. **Document specification**: Describe the node graph, data flow, and security controls in the PR 5. **Add to templates**: Include in template gallery with description 6. **Cache demo data**: Add cached execution for demo mode 7. **Write user guide**: Document use case, setup, and customization diff --git a/TESTING.md b/TESTING.md index eddf553..49b4352 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1,8 +1,8 @@ # Testing Documentation -![Test Status](https://img.shields.io/badge/tests-437%20passing-brightgreen) +![Test Status](https://img.shields.io/badge/tests-538%20passing-brightgreen) ![E2E Tests](https://img.shields.io/badge/e2e-5%20smoke%20tests-blue) -![Coverage](https://img.shields.io/badge/coverage-95%25%20(tested%20modules)-brightgreen) +![Coverage](https://img.shields.io/badge/coverage-17%25%20overall%20%C2%B7%2090%25%2B%20security%20core-yellow) ![Build Status](https://img.shields.io/badge/build-passing-brightgreen) ## Overview @@ -322,16 +322,33 @@ See [docs/development/testing-guide.md](docs/development/testing-guide.md) for d | Business Logic | 90% | ✅ 95.23% | | State Management | 90% | ✅ 95% | -### Overall Project Coverage +### Overall Project Coverage (measured 2026-09-26, 538 tests) -| Metric | Target | Current | Status | -|--------|--------|---------|--------| -| Statements | 75% | 2.03% | 🟡 In Progress | -| Branches | 70% | 0.83% | 🟡 In Progress | -| Functions | 75% | 2.85% | 🟡 In Progress | -| Lines | 75% | 1.86% | 🟡 In Progress | +| Metric | Current | +|--------|--------:| +| Statements | 17.1% | +| Branches | 16.1% | +| Functions | 13.8% | +| Lines | 17.0% | -**Note**: Low overall coverage is expected at this stage. We're following TDD principles by testing critical business logic first. UI components and integration tests are planned for future iterations. +The security core and execution route are at 90–100% line coverage; the low overall number is +mostly untested UI (`components/ui`, builder page, docs pages). Known gap: +`lib/security/validation-engine.ts` is at 0% and is next in line for tests. + +### Enforced thresholds (`jest.config.js`, blocking in CI via `pnpm test:ci`) + +Thresholds are set to what is **actually true**, then ratcheted up — never lowered to pass a PR. + +| Scope | Statements | Branches | Functions | Lines | +|-------|-----------:|---------:|----------:|------:| +| `lib/security/ssrf.ts` | 90% | 90% | 100% | 95% | +| `lib/security/rate-limit.ts` | 95% | 85% | 100% | 95% | +| `lib/security/workflow-graph.ts` | 95% | 85% | 100% | 95% | +| `lib/security/urw.ts` | 95% | 60% | 100% | 95% | +| `app/api/execute-workflow/route.ts` | 90% | 85% | 90% | 95% | +| global floor (all other files) | 13% | 12% | 11% | 13% | + +Files with a path-specific threshold are excluded from the global calculation. ### Coverage Roadmap @@ -353,23 +370,8 @@ Tests run automatically on every: - Pull request creation/update - Pre-deployment hooks -**Workflow file**: `.github/workflows/test.yml` (coming soon) - -```yaml -name: Tests - -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: pnpm/action-setup@v2 - - run: pnpm install - - run: pnpm test:ci - - run: pnpm build -``` +**Workflow file**: `.github/workflows/ci.yml` — lint + type-check → `pnpm test:ci` (tests + +coverage thresholds, blocking) → build. See [docs/development/ci.md](docs/development/ci.md). --- diff --git a/docs/development/ci.md b/docs/development/ci.md index 335c6bd..4c92b71 100644 --- a/docs/development/ci.md +++ b/docs/development/ci.md @@ -13,7 +13,8 @@ TopFlow runs CI via GitHub Actions, defined in `.github/workflows/ci.yml`. ## Jobs 1. **Lint and Type Check** — `pnpm lint`, then `pnpm type-check` (blocking). -2. **Run Tests** — `pnpm test` (Jest); a coverage report is uploaded as an artifact. +2. **Run Tests** — `pnpm test:ci` (Jest with coverage). Coverage thresholds in `jest.config.js` + are **blocking**; the report is uploaded as an artifact. 3. **Build Application** — `pnpm build`, gated on the first two jobs passing. All jobs run on Node 22 + pnpm 9 with a cached pnpm store. diff --git a/docs/development/testing-guide.md b/docs/development/testing-guide.md index f1da272..cb04062 100644 --- a/docs/development/testing-guide.md +++ b/docs/development/testing-guide.md @@ -412,12 +412,10 @@ describe('divide', () => { ## Coverage Goals -### Minimum Coverage Targets +### Targets vs. what is enforced -- **Utilities:** 95% coverage (pure functions) -- **Hooks:** 85% coverage (API interactions) -- **Components:** 75% coverage (UI behavior) -- **Overall:** 75% coverage +Aspirational targets: utilities 95%, hooks 85%, components 75%. What CI actually **enforces** is +narrower and honest — see "Coverage Thresholds" below and `TESTING.md` for current numbers. ### Viewing Coverage @@ -431,18 +429,13 @@ open coverage/lcov-report/index.html ### Coverage Thresholds -Set in `jest.config.js`: +Set in `jest.config.js` and enforced by `pnpm test:ci` in CI: -```javascript -coverageThreshold: { - global: { - statements: 75, - branches: 70, - functions: 75, - lines: 75, - }, -} -``` +- **Security-critical modules** (`lib/security/ssrf.ts`, `rate-limit.ts`, `workflow-graph.ts`, + `urw.ts`) and `app/api/execute-workflow/route.ts` are held near their current 90–100% coverage, + so a regression fails the build. +- **Global** is a ratchet floor at today's measured level (~13% for files without their own + threshold). Raise it as tests land; never lower it to make a PR pass. --- diff --git a/eslint.config.mjs b/eslint.config.mjs index 32364e1..320cc26 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -28,6 +28,9 @@ const eslintConfig = [ "build/**", "next-env.d.ts", "docs/**", + "coverage/**", + "playwright-report/**", + "test-results/**", ], }, { diff --git a/jest.config.js b/jest.config.js index a898e5f..9376cbe 100644 --- a/jest.config.js +++ b/jest.config.js @@ -43,14 +43,18 @@ const customJestConfig = { // Coverage directory coverageDirectory: 'coverage', - // Coverage thresholds (aligned with TDD strategy doc) + // Coverage thresholds — enforced in CI (`pnpm test:ci`), so they must be true. + // Strategy: hold the security-critical modules near their current coverage so a + // regression fails the build, and treat `global` as a ratchet floor (today's level, + // rounded down). Raise numbers as tests land; never lower them to pass a PR. + // Note: files matched by a path key are excluded from the `global` calculation. coverageThreshold: { - global: { - statements: 75, - branches: 70, - functions: 75, - lines: 75, - }, + global: { statements: 13, branches: 12, functions: 11, lines: 13 }, + './lib/security/ssrf.ts': { statements: 90, branches: 90, functions: 100, lines: 95 }, + './lib/security/rate-limit.ts': { statements: 95, branches: 85, functions: 100, lines: 95 }, + './lib/security/workflow-graph.ts': { statements: 95, branches: 85, functions: 100, lines: 95 }, + './lib/security/urw.ts': { statements: 95, branches: 60, functions: 100, lines: 95 }, + './app/api/execute-workflow/route.ts': { statements: 90, branches: 85, functions: 90, lines: 95 }, }, // Test match patterns