From c3a3259ae708b91c75280e6aefe77fedd2e70ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=91=E5=B8=83=E6=9E=97?= <11641432+heiheiyouyou@user.noreply.gitee.com> Date: Mon, 27 Apr 2026 20:55:04 +0800 Subject: [PATCH] Prevent common words from being involved in the calculation of the coverage score --- .../core/pipeline/memory-core.ts | 1 + .../core/skill/crystallize.ts | 6 +- .../memos-local-plugin/core/skill/verifier.ts | 60 ++++++++++++++++++- apps/memos-local-plugin/package.json | 2 +- .../tests/unit/skill/verifier.test.ts | 29 +++++++++ apps/memos-local-plugin/tsconfig.build.json | 14 +++++ 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 apps/memos-local-plugin/tsconfig.build.json diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 9ea444146..d5055d072 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1804,6 +1804,7 @@ export function createMemoryCore( status: dto.status, sourceEpisodeIds: [], inducedBy: "import", + decisionGuidance: (dto as any).decisionGuidance ?? { preference: [], antiPattern: [] }, vec: null, createdAt: dto.createdAt ?? Date.now(), updatedAt: dto.updatedAt ?? Date.now(), diff --git a/apps/memos-local-plugin/core/skill/crystallize.ts b/apps/memos-local-plugin/core/skill/crystallize.ts index 4cba94e51..cd9ed6767 100644 --- a/apps/memos-local-plugin/core/skill/crystallize.ts +++ b/apps/memos-local-plugin/core/skill/crystallize.ts @@ -96,7 +96,11 @@ export async function crystallizeDraft( ], { op: "skill.crystallize", - schemaHint: "skill-crystallize.v2", + schemaHint: + 'skill-crystallize.v2\n{"name":"snake_case","display_title":"string","summary":"string","parameters":[{"name":"string","type":"string|number|boolean|enum","required":true,"description":"string"}],"preconditions":["string"],"steps":[{"title":"string","body":"string"}],"examples":[{"input":"string","expected":"string"}],"decision_guidance":{"preference":["string"],"anti_pattern":["string"]},"tags":["string"]}', + malformedRetries: 2, + temperature: 0, + maxTokens: 4096, }, ); const draft = normaliseDraft(rsp.value, input); diff --git a/apps/memos-local-plugin/core/skill/verifier.ts b/apps/memos-local-plugin/core/skill/verifier.ts index 20a56b95a..29d85bd85 100644 --- a/apps/memos-local-plugin/core/skill/verifier.ts +++ b/apps/memos-local-plugin/core/skill/verifier.ts @@ -105,13 +105,16 @@ function collectCommandTokens(draft: SkillCrystallizationDraft): string[] { ...draft.steps.flatMap((s) => [s.title, s.body]), ...draft.examples.flatMap((e) => [e.input, e.expected]), ].join(" "); - const matches = fields.match(/`([^`]+)`|([a-z][a-z0-9_]{1,}\.[a-z][a-z0-9_]+|[a-z_]{3,}\b)/gi) ?? []; + const matches = + fields.match(/`([^`]+)`|([A-Za-z][A-Za-z0-9_]*\.[A-Za-z0-9_.]+|[A-Za-z0-9_./-]{3,})/g) ?? []; const out: string[] = []; const seen = new Set(); for (const raw of matches) { - const tok = raw.replace(/`/g, "").toLowerCase().trim(); + const rawTok = raw.replace(/`/g, "").trim(); + const tok = rawTok.toLowerCase(); if (tok.length < 3) continue; if (STOPWORDS.has(tok)) continue; + if (!isCoverageToken(rawTok)) continue; if (!seen.has(tok)) { seen.add(tok); out.push(tok); @@ -120,6 +123,28 @@ function collectCommandTokens(draft: SkillCrystallizationDraft): string[] { return out; } +function isCoverageToken(raw: string): boolean { + const tok = raw.trim(); + const lower = tok.toLowerCase(); + if (tok.length < 3) return false; + if (STOPWORDS.has(lower)) return false; + + // Coverage is meant to catch invented APIs / tools / paths, not every + // natural-language word in the generated skill prose. + if ( + tok.includes(".") || + tok.includes("/") || + tok.includes("_") || + tok.includes("-") || + /[0-9]/.test(tok) || + /[A-Z]/.test(tok) + ) { + return true; + } + + return ACTION_TOKENS.has(lower); +} + function computeResonance( draft: SkillCrystallizationDraft, evidence: TraceRow[], @@ -166,6 +191,37 @@ function tokensOf(s: string): Set { return out; } +const ACTION_TOKENS = new Set([ + "apk", + "build", + "cat", + "commit", + "curl", + "export", + "find", + "grep", + "import", + "install", + "load", + "ls", + "npm", + "parse", + "pip", + "pnpm", + "python", + "read", + "retry", + "rg", + "run", + "save", + "sqlite3", + "test", + "tree", + "validate", + "verify", + "write", +]); + const STOPWORDS = new Set([ "the", "and", "for", "with", "that", "this", "from", "will", "then", "into", "when", "what", "where", "your", "user", "agent", "null", "true", diff --git a/apps/memos-local-plugin/package.json b/apps/memos-local-plugin/package.json index 9e9244cf1..a0bc9093f 100644 --- a/apps/memos-local-plugin/package.json +++ b/apps/memos-local-plugin/package.json @@ -35,7 +35,7 @@ "docs" ], "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.build.json", "build:web": "vite build --config vite.config.ts", "build:site": "cd site && vite build", "build:all": "npm run build && npm run build:web && npm run build:site", diff --git a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts index 837aa2e51..b4ef66dc2 100644 --- a/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/verifier.test.ts @@ -61,6 +61,35 @@ describe("skill/verifier", () => { expect(r.reason).toBeTruthy(); }); + it("does not count plain prose words as coverage tokens", () => { + const draft = makeDraft({ + summary: "Create modular storage backends with consistent interface", + steps: [ + { + title: "Implement storage backend", + body: "Create a modular implementation that mirrors JSONStore with load and save methods.", + }, + ], + }); + const evidence = [ + trace( + "tr_1", + "Add a YAML storage backend", + "Wrote YAMLStore mirroring JSONStore with load and save methods.", + ), + trace( + "tr_2", + "Verify storage files", + "Checked task_cli/storage/yaml_store.py and __init__.py exports.", + ), + ]; + const r = verifyDraft({ draft, evidence }, { log }); + expect(r.ok).toBe(true); + expect(r.unmappedTokens).not.toContain("modular"); + expect(r.unmappedTokens).not.toContain("interface"); + expect(r.unmappedTokens).not.toContain("implementation"); + }); + it("fails when there is no evidence at all", () => { const r = verifyDraft({ draft: makeDraft(), evidence: [] }, { log }); expect(r.ok).toBe(false); diff --git a/apps/memos-local-plugin/tsconfig.build.json b/apps/memos-local-plugin/tsconfig.build.json new file mode 100644 index 000000000..ed5c444f6 --- /dev/null +++ b/apps/memos-local-plugin/tsconfig.build.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "core/**/*.ts", + "agent-contract/**/*.ts", + "server/**/*.ts", + "bridge/**/*.ts", + "adapters/openclaw/**/*.ts", + "scripts/**/*.ts", + "site/scripts/**/*.ts", + "bridge.cts" + ], + "exclude": ["node_modules", "dist", "web", "site/src", "adapters/hermes", "tests"] +}