diff --git a/.gitignore b/.gitignore index 9a9f924c..02379e1d 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,8 @@ pnpm-debug.log* # Brainstorm assets (not tracked in source control) .superpowers/ .claude/ + +# Local Windows dev helpers +LLM Wiki Dev.lnk +launch-llm-wiki-dev.cmd +tools/ diff --git a/mcp-server/README.md b/mcp-server/README.md new file mode 100644 index 00000000..e7bc5304 --- /dev/null +++ b/mcp-server/README.md @@ -0,0 +1,77 @@ +# LLM Wiki MCP Server + +Local stdio MCP server for the LLM Wiki desktop app. It wraps the app's local +HTTP API at `http://127.0.0.1:19827/api/v1` so agents can discover and call +LLM Wiki as named MCP tools instead of raw HTTP endpoints. + +## Requirements + +- Start the LLM Wiki desktop app. +- Open the target project in the app. +- Keep the local API reachable at `127.0.0.1:19827`. +- Configure an LLM provider in LLM Wiki Settings before using + `llmwiki_chat`, `llmwiki_ingest_clip`, or `llmwiki_ingest_file`. + +## Run + +From this repository: + +```powershell +npm run mcp:llmwiki +``` + +Equivalent direct command: + +```powershell +node D:\Dev\llm_wiki\mcp-server\llmwiki-mcp.js +``` + +Optional environment variables: + +- `LLMWIKI_API_BASE`: defaults to `http://127.0.0.1:19827/api/v1` +- `LLMWIKI_TIMEOUT_MS`: defaults to `60000` for ordinary calls; chat and + ingest calls keep a 30 minute timeout to match the desktop bridge. + +## Codex Registration + +```powershell +codex mcp add llmwiki -- node D:\Dev\llm_wiki\mcp-server\llmwiki-mcp.js +``` + +Newly registered MCP servers may require a new Codex session before the tools +appear. + +## Claude Desktop Example + +Add a server entry like this to your Claude Desktop MCP config: + +```json +{ + "mcpServers": { + "llmwiki": { + "command": "node", + "args": ["D:\\Dev\\llm_wiki\\mcp-server\\llmwiki-mcp.js"] + } + } +} +``` + +## Tools + +- `llmwiki_status`: checks whether the local LLM Wiki API is reachable and + reports current project, recent projects, and capabilities. +- `llmwiki_search`: quick ranked search across an LLM Wiki project. +- `llmwiki_retrieve`: retrieves citation-ready context, relevant pages, + references, search hits, and graph expansions. +- `llmwiki_chat`: asks LLM Wiki to answer using the app's configured LLM and + project knowledge base. +- `llmwiki_graph`: returns the project knowledge graph; text summarizes counts + and `structuredContent` contains the full graph. +- `llmwiki_ingest_clip`: saves text as a raw source in the active project and + queues it for ingest. +- `llmwiki_ingest_file`: copies a local file into the active project's + `raw/sources` directory and queues it for ingest. + +Every tool returns a short text summary plus `structuredContent` containing the +raw LLM Wiki API response. API failures return `isError: true` with a concrete +hint to start the app and open the target project. diff --git a/mcp-server/llmwiki-mcp.js b/mcp-server/llmwiki-mcp.js new file mode 100644 index 00000000..e11479e3 --- /dev/null +++ b/mcp-server/llmwiki-mcp.js @@ -0,0 +1,406 @@ +#!/usr/bin/env node +import { fileURLToPath } from "node:url"; +import { resolve } from "node:path"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { z } from "zod"; + +export const DEFAULT_API_BASE = "http://127.0.0.1:19827/api/v1"; +export const DEFAULT_TIMEOUT_MS = 60_000; +export const LONG_TIMEOUT_MS = 30 * 60_000; + +const projectPathSchema = z + .string() + .trim() + .min(1) + .optional() + .describe("Optional LLM Wiki project path. Omit to use the project currently open in the desktop app."); + +const limitSchema = z + .number() + .int() + .positive() + .max(100) + .optional() + .describe("Maximum number of results to return."); + +const historyMessageSchema = z.object({ + role: z.enum(["user", "assistant"]).describe("Conversation role."), + content: z.string().min(1).describe("Message text."), +}); + +function readPositiveInt(value, fallback) { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; +} + +function normalizeApiBase(value) { + const base = (value || DEFAULT_API_BASE).trim(); + return base.replace(/\/+$/, ""); +} + +function endpointUrl(apiBase, endpoint) { + return `${normalizeApiBase(apiBase)}/${endpoint.replace(/^\/+/, "")}`; +} + +function isAbortError(error) { + return error && typeof error === "object" && error.name === "AbortError"; +} + +function connectionHint() { + return "Start LLM Wiki, open the target project, and make sure the local API is listening on 127.0.0.1:19827."; +} + +function errorMessage(error, endpoint) { + if (isAbortError(error)) { + return `Timed out while calling LLM Wiki endpoint "${endpoint}".`; + } + const raw = error instanceof Error ? error.message : String(error); + return `Failed to reach LLM Wiki endpoint "${endpoint}": ${raw}. ${connectionHint()}`; +} + +export function createApiClient(options = {}) { + const apiBase = normalizeApiBase(options.apiBase ?? process.env.LLMWIKI_API_BASE); + const timeoutMs = readPositiveInt(options.timeoutMs ?? process.env.LLMWIKI_TIMEOUT_MS, DEFAULT_TIMEOUT_MS); + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + + if (typeof fetchImpl !== "function") { + throw new Error("A fetch implementation is required. Use Node.js 18+ or provide fetchImpl."); + } + + async function request(method, endpoint, body, requestOptions = {}) { + const timeout = requestOptions.long ? Math.max(timeoutMs, LONG_TIMEOUT_MS) : timeoutMs; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + timer.unref?.(); + + try { + const response = await fetchImpl(endpointUrl(apiBase, endpoint), { + method, + headers: method === "POST" ? { "Content-Type": "application/json" } : undefined, + body: method === "POST" ? JSON.stringify(body ?? {}) : undefined, + signal: controller.signal, + }); + + const text = await response.text(); + let data = {}; + if (text.trim()) { + try { + data = JSON.parse(text); + } catch (error) { + return { + ok: false, + endpoint, + status: response.status, + error: `LLM Wiki returned non-JSON response from "${endpoint}": ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + + if (!response.ok) { + return { + ok: false, + endpoint, + status: response.status, + error: extractApiError(data) || `HTTP ${response.status} from LLM Wiki endpoint "${endpoint}".`, + response: data, + }; + } + + if (data && typeof data === "object" && data.ok === false) { + return { + ok: false, + endpoint, + status: response.status, + error: extractApiError(data) || `LLM Wiki endpoint "${endpoint}" returned ok:false.`, + response: data, + }; + } + + return { ok: true, endpoint, status: response.status, response: data }; + } catch (error) { + return { + ok: false, + endpoint, + error: errorMessage(error, endpoint), + }; + } finally { + clearTimeout(timer); + } + } + + return { + apiBase, + get: (endpoint) => request("GET", endpoint), + post: (endpoint, body, options) => request("POST", endpoint, body, options), + }; +} + +function extractApiError(data) { + if (!data || typeof data !== "object") return ""; + if (typeof data.error === "string") return data.error; + if (data.error !== undefined) return JSON.stringify(data.error); + if (typeof data.message === "string") return data.message; + return ""; +} + +function textResult(text, structuredContent, isError = false) { + return { + content: [{ type: "text", text }], + structuredContent, + ...(isError ? { isError: true } : {}), + }; +} + +function apiErrorResult(response) { + const structuredContent = { + ok: false, + endpoint: response.endpoint, + status: response.status ?? null, + error: response.error || "Unknown LLM Wiki API error.", + response: response.response ?? null, + }; + return textResult( + `${structuredContent.error}\n\n${connectionHint()}`, + structuredContent, + true, + ); +} + +function apiSuccessResult(response, summarize) { + return textResult(summarize(response.response), response.response); +} + +function count(value) { + return Array.isArray(value) ? value.length : 0; +} + +function maybeList(value, max = 3) { + if (!Array.isArray(value) || value.length === 0) return ""; + const names = value.slice(0, max).map((item) => { + if (!item || typeof item !== "object") return String(item); + return item.title || item.name || item.path || item.file || item.slug || JSON.stringify(item).slice(0, 80); + }); + return names.length > 0 ? ` Top: ${names.join("; ")}` : ""; +} + +export function summarizeStatus(data) { + const projectPath = data?.project?.path || ""; + const projects = Array.isArray(data?.projects) ? data.projects : []; + const capabilities = Array.isArray(data?.capabilities) ? data.capabilities : []; + const current = projectPath ? `Current project: ${projectPath}` : "No current project reported."; + return [ + "LLM Wiki local API is reachable.", + current, + `Projects listed: ${projects.length}.`, + `Capabilities: ${capabilities.length > 0 ? capabilities.join(", ") : "none reported"}.`, + ].join(" "); +} + +export function summarizeSearch(data) { + const payload = data?.result ?? data ?? {}; + const results = Array.isArray(payload.results) ? payload.results : []; + return `Found ${results.length} search result(s) for "${payload.query ?? ""}".${maybeList(results)}`; +} + +export function summarizeRetrieve(data) { + const payload = data?.result ?? data ?? {}; + const pages = Array.isArray(payload.pages) ? payload.pages : []; + const references = Array.isArray(payload.references) ? payload.references : []; + const searchResults = Array.isArray(payload.searchResults) ? payload.searchResults : []; + const graphExpansions = Array.isArray(payload.graphExpansions) ? payload.graphExpansions : []; + return [ + `Retrieved context for "${payload.query ?? ""}".`, + `Pages: ${pages.length}.`, + `References: ${references.length}.`, + `Search results: ${searchResults.length}.`, + `Graph expansions: ${graphExpansions.length}.`, + ].join(" "); +} + +export function summarizeChat(data) { + const payload = data?.result ?? data ?? {}; + const answer = typeof payload.answer === "string" ? payload.answer : ""; + const references = Array.isArray(payload.references) ? payload.references : []; + if (!answer) return `LLM Wiki chat completed for "${payload.query ?? ""}" with ${references.length} reference(s).`; + return `${answer}\n\nReferences: ${references.length}.`; +} + +export function summarizeGraph(data) { + const payload = data?.result ?? data ?? {}; + return `Graph loaded. Nodes: ${count(payload.nodes)}. Edges: ${count(payload.edges)}.`; +} + +export function summarizeIngest(data) { + const payload = data?.result ?? data ?? {}; + const path = payload.path || payload.absolutePath || ""; + const taskId = payload.taskId || ""; + return `Source queued for ingest.${path ? ` Path: ${path}.` : ""}${taskId ? ` Task: ${taskId}.` : ""}`; +} + +async function runGetTool(apiClient, endpoint, summarize) { + const response = await apiClient.get(endpoint); + if (!response.ok) return apiErrorResult(response); + return apiSuccessResult(response, summarize); +} + +async function runPostTool(apiClient, endpoint, args, summarize, options = {}) { + const response = await apiClient.post(endpoint, args, options); + if (!response.ok) return apiErrorResult(response); + return apiSuccessResult(response, summarize); +} + +export const toolHandlers = { + status: (apiClient) => runGetTool(apiClient, "status", summarizeStatus), + search: (apiClient, args) => runPostTool(apiClient, "search", args, summarizeSearch), + retrieve: (apiClient, args) => runPostTool(apiClient, "retrieve", { includeContent: true, ...args }, summarizeRetrieve), + chat: (apiClient, args) => runPostTool(apiClient, "chat", args, summarizeChat, { long: true }), + graph: (apiClient, args) => runPostTool(apiClient, "graph", args, summarizeGraph), + ingestClip: (apiClient, args) => runPostTool(apiClient, "ingest/clip", args, summarizeIngest, { long: true }), + ingestFile: (apiClient, args) => runPostTool(apiClient, "ingest/file", args, summarizeIngest, { long: true }), +}; + +const readOnlyAnnotations = { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, +}; + +const writeAnnotations = { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: false, +}; + +export function createLlmWikiMcpServer(options = {}) { + const apiClient = options.apiClient ?? createApiClient(options); + const server = new McpServer({ + name: "llmwiki", + version: "0.1.0", + }); + + server.registerTool( + "llmwiki_status", + { + title: "LLM Wiki Status", + description: "Check whether the local LLM Wiki desktop API is reachable and report the current project, recent projects, and supported capabilities.", + inputSchema: {}, + annotations: readOnlyAnnotations, + }, + () => toolHandlers.status(apiClient), + ); + + server.registerTool( + "llmwiki_search", + { + title: "Search LLM Wiki", + description: "Search an LLM Wiki project for matching wiki pages or raw sources. Use this for a quick ranked lookup when full page content is not required.", + inputSchema: { + query: z.string().trim().min(1).describe("Search query."), + limit: limitSchema.default(20), + projectPath: projectPathSchema, + }, + annotations: readOnlyAnnotations, + }, + (args) => toolHandlers.search(apiClient, args), + ); + + server.registerTool( + "llmwiki_retrieve", + { + title: "Retrieve LLM Wiki Context", + description: "Retrieve citation-ready context from LLM Wiki, including relevant pages, references, search hits, and graph expansions for a query.", + inputSchema: { + query: z.string().trim().min(1).describe("Question or topic to retrieve context for."), + limit: limitSchema.default(10), + includeContent: z.boolean().optional().default(true).describe("Whether to include full page content in retrieved pages."), + projectPath: projectPathSchema, + }, + annotations: readOnlyAnnotations, + }, + (args) => toolHandlers.retrieve(apiClient, args), + ); + + server.registerTool( + "llmwiki_chat", + { + title: "Ask LLM Wiki", + description: "Ask LLM Wiki to answer using the currently configured LLM provider and the selected project knowledge base. Requires LLM Wiki Settings to have a usable model/API key.", + inputSchema: { + query: z.string().trim().min(1).describe("Question to answer from the LLM Wiki project."), + messages: z.array(historyMessageSchema).optional().describe("Optional recent conversation messages to include as chat history."), + maxHistoryMessages: z.number().int().positive().max(50).optional().default(10).describe("Maximum number of history messages to forward."), + projectPath: projectPathSchema, + }, + annotations: { + ...readOnlyAnnotations, + idempotentHint: false, + openWorldHint: true, + }, + }, + (args) => toolHandlers.chat(apiClient, args), + ); + + server.registerTool( + "llmwiki_graph", + { + title: "Get LLM Wiki Graph", + description: "Load the LLM Wiki knowledge graph for a project. The text result summarizes node and edge counts; structuredContent contains the full graph payload.", + inputSchema: { + projectPath: projectPathSchema, + }, + annotations: readOnlyAnnotations, + }, + (args) => toolHandlers.graph(apiClient, args), + ); + + server.registerTool( + "llmwiki_ingest_clip", + { + title: "Ingest Text Clip", + description: "Save text content as a new raw source in the active LLM Wiki desktop project and enqueue it for ingest. Open the target project in the desktop app before calling.", + inputSchema: { + title: z.string().trim().min(1).optional().describe("Source title. Defaults to Untitled in LLM Wiki if omitted."), + url: z.string().trim().optional().describe("Original source URL, if available."), + content: z.string().trim().min(1).describe("Markdown or plain text content to save and ingest."), + projectPath: projectPathSchema, + }, + annotations: writeAnnotations, + }, + (args) => toolHandlers.ingestClip(apiClient, args), + ); + + server.registerTool( + "llmwiki_ingest_file", + { + title: "Ingest Local File", + description: "Copy a local file into the active LLM Wiki desktop project raw/sources directory and enqueue it for ingest. Open the target project in the desktop app before calling.", + inputSchema: { + sourcePath: z.string().trim().min(1).describe("Absolute or app-readable local file path to import."), + projectPath: projectPathSchema, + }, + annotations: writeAnnotations, + }, + (args) => toolHandlers.ingestFile(apiClient, args), + ); + + return server; +} + +export async function main() { + const server = createLlmWikiMcpServer(); + const transport = new StdioServerTransport(); + await server.connect(transport); +} + +const currentFile = fileURLToPath(import.meta.url); +const invokedFile = process.argv[1] ? resolve(process.argv[1]) : ""; + +if (invokedFile && currentFile === invokedFile) { + main().catch((error) => { + console.error("LLM Wiki MCP server failed:", error); + process.exitCode = 1; + }); +} diff --git a/mcp-server/llmwiki-mcp.test.js b/mcp-server/llmwiki-mcp.test.js new file mode 100644 index 00000000..8d0611bc --- /dev/null +++ b/mcp-server/llmwiki-mcp.test.js @@ -0,0 +1,102 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createApiClient, + summarizeStatus, + toolHandlers, +} from "./llmwiki-mcp.js"; + +function jsonResponse(data, options = {}) { + return { + ok: options.ok ?? true, + status: options.status ?? 200, + text: vi.fn().mockResolvedValue(JSON.stringify(data)), + }; +} + +function makeClient(fetchImpl) { + return createApiClient({ + apiBase: "http://127.0.0.1:19827/api/v1", + timeoutMs: 1000, + fetchImpl, + }); +} + +describe("llmwiki MCP API wrapper", () => { + it("sends search payload to /api/v1/search", async () => { + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse({ ok: true, result: { query: "cache", results: [] } }), + ); + const client = makeClient(fetchImpl); + + const result = await toolHandlers.search(client, { + query: "cache", + limit: 5, + projectPath: "D:/wiki", + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + const [url, options] = fetchImpl.mock.calls[0]; + expect(url).toBe("http://127.0.0.1:19827/api/v1/search"); + expect(options.method).toBe("POST"); + expect(JSON.parse(options.body)).toEqual({ + query: "cache", + limit: 5, + projectPath: "D:/wiki", + }); + expect(result.isError).toBeUndefined(); + expect(result.structuredContent.result.results).toEqual([]); + }); + + it("defaults retrieve includeContent to true", async () => { + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse({ ok: true, result: { query: "rope", pages: [], references: [] } }), + ); + const client = makeClient(fetchImpl); + + await toolHandlers.retrieve(client, { query: "rope" }); + + const [, options] = fetchImpl.mock.calls[0]; + expect(JSON.parse(options.body)).toMatchObject({ + query: "rope", + includeContent: true, + }); + }); + + it("returns an MCP error result for ok:false chat responses", async () => { + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse({ ok: false, error: "LLM not configured -- set API key and model in Settings." }), + ); + const client = makeClient(fetchImpl); + + const result = await toolHandlers.chat(client, { query: "what changed?" }); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("LLM not configured"); + expect(result.structuredContent.ok).toBe(false); + }); + + it("summarizes status projects and capabilities", () => { + const summary = summarizeStatus({ + ok: true, + project: { path: "D:/wiki" }, + projects: [{ name: "A", path: "D:/wiki" }], + capabilities: ["search", "retrieve"], + }); + + expect(summary).toContain("LLM Wiki local API is reachable"); + expect(summary).toContain("Current project: D:/wiki"); + expect(summary).toContain("Projects listed: 1"); + expect(summary).toContain("search, retrieve"); + }); + + it("returns a helpful error when the local API is unreachable", async () => { + const fetchImpl = vi.fn().mockRejectedValue(new Error("ECONNREFUSED")); + const client = makeClient(fetchImpl); + + const result = await toolHandlers.status(client); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("Start LLM Wiki"); + expect(result.content[0].text).toContain("open the target project"); + }); +}); diff --git a/package-lock.json b/package-lock.json index ea1935cf..d8489765 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@milkdown/plugin-math": "^7.5.9", "@milkdown/react": "^7.20.0", "@milkdown/theme-nord": "^7.20.0", + "@modelcontextprotocol/sdk": "^1.29.0", "@react-sigma/core": "^5.0.6", "@tailwindcss/vite": "^4.2.2", "@tauri-apps/api": "^2.10.1", @@ -45,6 +46,7 @@ "tailwind-merge": "^3.5.0", "tailwindcss": "^4.2.2", "tw-animate-css": "^1.4.0", + "zod": "^3.25.76", "zustand": "^5.0.12" }, "devDependencies": { diff --git a/package.json b/package.json index 348203f4..3d266171 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "dev": "vite", "typecheck": "tsc --build --pretty", "build": "npm run typecheck && vite build", + "mcp:llmwiki": "node mcp-server/llmwiki-mcp.js", "preview": "vite preview", "test": "npm run test:mocks && npm run test:llm", "test:mocks": "vitest run --exclude='**/*.real-llm.test.ts'", @@ -20,6 +21,7 @@ "@milkdown/plugin-math": "^7.5.9", "@milkdown/react": "^7.20.0", "@milkdown/theme-nord": "^7.20.0", + "@modelcontextprotocol/sdk": "^1.29.0", "@react-sigma/core": "^5.0.6", "@tailwindcss/vite": "^4.2.2", "@tauri-apps/api": "^2.10.1", @@ -51,6 +53,7 @@ "tailwind-merge": "^3.5.0", "tailwindcss": "^4.2.2", "tw-animate-css": "^1.4.0", + "zod": "^3.25.76", "zustand": "^5.0.12" }, "devDependencies": { diff --git a/src-tauri/src/clip_server.rs b/src-tauri/src/clip_server.rs index 33a47f4c..ee96fda7 100644 --- a/src-tauri/src/clip_server.rs +++ b/src-tauri/src/clip_server.rs @@ -1,20 +1,45 @@ -use std::sync::Mutex; +use serde::Serialize; +use std::collections::HashMap; +use std::io::Read; +use std::sync::{Condvar, Mutex}; use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::LazyLock; use std::thread; -use tiny_http::{Header, Method, Response, Server}; +use std::time::{Duration, Instant}; +use tiny_http::{Header, Method, Request, Response, Server}; static CURRENT_PROJECT: Mutex = Mutex::new(String::new()); static ALL_PROJECTS: Mutex> = Mutex::new(Vec::new()); // (name, path) static PENDING_CLIPS: Mutex> = Mutex::new(Vec::new()); // (projectPath, filePath) +static API_BRIDGE_REQUESTS: Mutex> = Mutex::new(Vec::new()); +static API_BRIDGE_RESPONSES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); +static API_BRIDGE_CVAR: Condvar = Condvar::new(); /// Daemon status: 0=starting, 1=running, 2=port_conflict, 3=error static DAEMON_STATUS: AtomicU8 = AtomicU8::new(0); +static NEXT_API_REQUEST_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); const PORT: u16 = 19827; const MAX_BIND_RETRIES: u32 = 3; const MAX_RESTART_RETRIES: u32 = 10; const BIND_RETRY_DELAY_SECS: u64 = 2; const RESTART_DELAY_SECS: u64 = 5; +const API_SHORT_TIMEOUT_SECS: u64 = 60; +const API_LONG_TIMEOUT_SECS: u64 = 30 * 60; + +#[derive(Clone, Serialize)] +struct ApiBridgeRequest { + id: String, + endpoint: String, + payload: serde_json::Value, +} + +#[derive(Clone)] +struct ApiBridgeResponse { + ok: bool, + payload: serde_json::Value, +} /// Get current daemon status as a string pub fn get_daemon_status() -> &'static str { @@ -70,174 +95,9 @@ pub fn start_clip_server() { restart_count = 0; // Reset on successful bind println!("[Clip Server] Listening on http://127.0.0.1:{}", PORT); - for mut request in server.incoming_requests() { - let cors_headers = vec![ - Header::from_bytes("Access-Control-Allow-Origin", "*").unwrap(), - Header::from_bytes("Access-Control-Allow-Methods", "GET, POST, OPTIONS").unwrap(), - Header::from_bytes("Access-Control-Allow-Headers", "Content-Type").unwrap(), - Header::from_bytes("Content-Type", "application/json").unwrap(), - ]; - - // Handle CORS preflight - if request.method() == &Method::Options { - let mut response = Response::from_string("").with_status_code(204); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - continue; - } - - let url = request.url().to_string(); - - match (request.method(), url.as_str()) { - (&Method::Get, "/status") => { - let body = r#"{"ok":true,"version":"0.1.0"}"#; - let mut response = Response::from_string(body); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - } - (&Method::Get, "/project") => { - let path = CURRENT_PROJECT.lock().unwrap().clone(); - // serde_json handles backslash escaping so a Windows - // path that somehow still contains `\` won't break - // the JSON parser on the client. - let body = serde_json::json!({ - "ok": true, - "path": path, - }).to_string(); - let mut response = Response::from_string(body); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - } - (&Method::Post, "/project") => { - let mut body = String::new(); - if let Err(e) = request.as_reader().read_to_string(&mut body) { - let err = - format!(r#"{{"ok":false,"error":"Failed to read body: {}"}}"#, e); - let mut response = Response::from_string(err).with_status_code(400); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - continue; - } - - let result = handle_set_project(&body); - let status = if result.contains(r#""ok":true"#) { - 200 - } else { - 400 - }; - let mut response = Response::from_string(result).with_status_code(status); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - } - (&Method::Get, "/projects") => { - let projects = ALL_PROJECTS.lock().unwrap().clone(); - let current = CURRENT_PROJECT.lock().unwrap().clone(); - // serde_json for proper escaping of `\`, `"`, and any - // other characters that might appear in a project name - // or path. Previously only `"` was escaped by hand, - // which broke on Windows paths containing backslashes. - let items: Vec = projects.iter() - .map(|(name, path)| serde_json::json!({ - "name": name, - "path": path, - "current": path == ¤t, - })) - .collect(); - let body = serde_json::json!({ - "ok": true, - "projects": items, - }).to_string(); - let mut response = Response::from_string(body); - for h in &cors_headers { response.add_header(h.clone()); } - let _ = request.respond(response); - } - (&Method::Post, "/projects") => { - let mut body = String::new(); - if request.as_reader().read_to_string(&mut body).is_ok() { - if let Ok(parsed) = serde_json::from_str::(&body) { - if let Some(arr) = parsed["projects"].as_array() { - let mut projects = ALL_PROJECTS.lock().unwrap(); - projects.clear(); - for item in arr { - let name = item["name"].as_str().unwrap_or("").to_string(); - let path = item["path"].as_str().unwrap_or("").to_string(); - if !path.is_empty() { - projects.push((name, path)); - } - } - } - } - } - let mut response = Response::from_string(r#"{"ok":true}"#); - for h in &cors_headers { response.add_header(h.clone()); } - let _ = request.respond(response); - } - (&Method::Get, "/clips/pending") => { - let mut pending = PENDING_CLIPS.lock().unwrap(); - // Use serde_json for proper escaping of both quotes - // and backslashes — hand-rolled escaping previously - // produced invalid JSON on Windows paths containing - // \r, \s, etc. - let clips_json: Vec = pending.iter() - .map(|(proj, file)| serde_json::json!({ - "projectPath": proj, - "filePath": file, - })) - .collect(); - let body = serde_json::json!({ - "ok": true, - "clips": clips_json, - }).to_string(); - pending.clear(); - let mut response = Response::from_string(body); - for h in &cors_headers { response.add_header(h.clone()); } - let _ = request.respond(response); - } - (&Method::Post, "/clip") => { - let mut body = String::new(); - if let Err(e) = request.as_reader().read_to_string(&mut body) { - let err = - format!(r#"{{"ok":false,"error":"Failed to read body: {}"}}"#, e); - let mut response = Response::from_string(err).with_status_code(400); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - continue; - } - - let result = handle_clip(&body); - let status = if result.contains(r#""ok":true"#) { - 200 - } else { - 500 - }; - let mut response = Response::from_string(result).with_status_code(status); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - } - _ => { - let body = r#"{"ok":false,"error":"Not found"}"#; - let mut response = Response::from_string(body).with_status_code(404); - for h in &cors_headers { - response.add_header(h.clone()); - } - let _ = request.respond(response); - } + for request in server.incoming_requests() { + thread::spawn(move || handle_request(request)); } - } // Server loop exited (shouldn't happen normally) DAEMON_STATUS.store(3, Ordering::Relaxed); // error @@ -260,6 +120,373 @@ pub fn start_clip_server() { }); } +fn cors_headers() -> Vec
{ + vec![ + Header::from_bytes("Access-Control-Allow-Origin", "*").unwrap(), + Header::from_bytes("Access-Control-Allow-Methods", "GET, POST, OPTIONS").unwrap(), + Header::from_bytes("Access-Control-Allow-Headers", "Content-Type").unwrap(), + Header::from_bytes("Content-Type", "application/json").unwrap(), + ] +} + +fn respond_json(request: Request, status: u16, body: serde_json::Value) { + let mut response = Response::from_string(body.to_string()).with_status_code(status); + for h in cors_headers() { + response.add_header(h); + } + let _ = request.respond(response); +} + +fn read_body(request: &mut Request) -> Result { + let mut body = String::new(); + request + .as_reader() + .read_to_string(&mut body) + .map_err(|e| format!("Failed to read body: {e}"))?; + Ok(body) +} + +fn current_project_path() -> String { + CURRENT_PROJECT.lock().unwrap().clone() +} + +fn projects_json() -> serde_json::Value { + let projects = ALL_PROJECTS.lock().unwrap().clone(); + let current = current_project_path(); + let items: Vec = projects + .iter() + .map(|(name, path)| { + serde_json::json!({ + "name": name, + "path": path, + "current": path == ¤t, + }) + }) + .collect(); + serde_json::json!({ + "ok": true, + "projects": items, + }) +} + +fn api_capabilities() -> Vec<&'static str> { + vec![ + "status", + "search", + "retrieve", + "chat", + "graph", + "ingest.clip", + "ingest.file", + ] +} + +fn handle_request(mut request: Request) { + if request.method() == &Method::Options { + let mut response = Response::from_string("").with_status_code(204); + for h in cors_headers() { + response.add_header(h); + } + let _ = request.respond(response); + return; + } + + let url = request.url().to_string(); + let path = url.split('?').next().unwrap_or(url.as_str()); + + match (request.method(), path) { + (&Method::Get, "/status") => { + respond_json(request, 200, serde_json::json!({"ok": true, "version": "0.1.0"})); + } + (&Method::Get, "/project") => { + respond_json( + request, + 200, + serde_json::json!({ + "ok": true, + "path": current_project_path(), + }), + ); + } + (&Method::Post, "/project") => { + let body = match read_body(&mut request) { + Ok(body) => body, + Err(error) => { + respond_json(request, 400, serde_json::json!({"ok": false, "error": error})); + return; + } + }; + let result = handle_set_project(&body); + let parsed = serde_json::from_str::(&result) + .unwrap_or_else(|_| serde_json::json!({"ok": false, "error": result})); + let status = if parsed["ok"].as_bool().unwrap_or(false) { 200 } else { 400 }; + respond_json(request, status, parsed); + } + (&Method::Get, "/projects") => { + respond_json(request, 200, projects_json()); + } + (&Method::Post, "/projects") => { + let body = read_body(&mut request).unwrap_or_default(); + if let Ok(parsed) = serde_json::from_str::(&body) { + if let Some(arr) = parsed["projects"].as_array() { + let mut projects = ALL_PROJECTS.lock().unwrap(); + projects.clear(); + for item in arr { + let name = item["name"].as_str().unwrap_or("").to_string(); + let path = item["path"].as_str().unwrap_or("").to_string(); + if !path.is_empty() { + projects.push((name, path)); + } + } + } + } + respond_json(request, 200, serde_json::json!({"ok": true})); + } + (&Method::Get, "/clips/pending") => { + let mut pending = PENDING_CLIPS.lock().unwrap(); + let clips_json: Vec = pending + .iter() + .map(|(proj, file)| { + serde_json::json!({ + "projectPath": proj, + "filePath": file, + }) + }) + .collect(); + pending.clear(); + respond_json( + request, + 200, + serde_json::json!({ + "ok": true, + "clips": clips_json, + }), + ); + } + (&Method::Post, "/clip") => { + let body = match read_body(&mut request) { + Ok(body) => body, + Err(error) => { + respond_json(request, 400, serde_json::json!({"ok": false, "error": error})); + return; + } + }; + let result = handle_clip(&body); + let parsed = serde_json::from_str::(&result) + .unwrap_or_else(|_| serde_json::json!({"ok": false, "error": result})); + let status = if parsed["ok"].as_bool().unwrap_or(false) { 200 } else { 500 }; + respond_json(request, status, parsed); + } + (&Method::Get, "/api/v1/status") => { + respond_json( + request, + 200, + serde_json::json!({ + "ok": true, + "version": "0.1.0", + "apiVersion": "v1", + "project": { + "path": current_project_path(), + }, + "projects": projects_json()["projects"].clone(), + "capabilities": api_capabilities(), + "bridge": { + "mode": "frontend", + "pending": API_BRIDGE_REQUESTS.lock().map(|q| q.len()).unwrap_or(0), + }, + }), + ); + } + (&Method::Get, "/api/v1/bridge/pending") => { + let requests = { + let mut queue = API_BRIDGE_REQUESTS.lock().unwrap(); + queue.drain(..).collect::>() + }; + respond_json( + request, + 200, + serde_json::json!({ + "ok": true, + "requests": requests, + }), + ); + } + (&Method::Post, "/api/v1/bridge/respond") => { + let body = match read_body(&mut request) { + Ok(body) => body, + Err(error) => { + respond_json(request, 400, serde_json::json!({"ok": false, "error": error})); + return; + } + }; + let parsed = match serde_json::from_str::(&body) { + Ok(value) => value, + Err(e) => { + respond_json( + request, + 400, + serde_json::json!({"ok": false, "error": format!("Invalid JSON: {e}")}), + ); + return; + } + }; + let id = match parsed["id"].as_str() { + Some(id) if !id.is_empty() => id.to_string(), + _ => { + respond_json( + request, + 400, + serde_json::json!({"ok": false, "error": "id field is required"}), + ); + return; + } + }; + let ok = parsed["ok"].as_bool().unwrap_or(false); + let payload = if ok { + parsed.get("result").cloned().unwrap_or(serde_json::Value::Null) + } else { + parsed + .get("error") + .cloned() + .unwrap_or_else(|| serde_json::json!("Unknown bridge error")) + }; + { + let mut responses = API_BRIDGE_RESPONSES.lock().unwrap(); + responses.insert(id, ApiBridgeResponse { ok, payload }); + } + API_BRIDGE_CVAR.notify_all(); + respond_json(request, 200, serde_json::json!({"ok": true})); + } + (&Method::Post, p) if p.starts_with("/api/v1/") => { + let endpoint = p.trim_start_matches("/api/v1/").to_string(); + let body = match read_body(&mut request) { + Ok(body) => body, + Err(error) => { + respond_json(request, 400, serde_json::json!({"ok": false, "error": error})); + return; + } + }; + match forward_to_frontend_bridge(&endpoint, &body) { + Ok(value) => respond_json(request, 200, value), + Err((status, value)) => respond_json(request, status, value), + } + } + _ => { + respond_json( + request, + 404, + serde_json::json!({"ok": false, "error": "Not found"}), + ); + } + } +} + +fn forward_to_frontend_bridge( + endpoint: &str, + body: &str, +) -> Result { + let mut payload = if body.trim().is_empty() { + serde_json::json!({}) + } else { + serde_json::from_str::(body).map_err(|e| { + ( + 400, + serde_json::json!({"ok": false, "error": format!("Invalid JSON: {e}")}), + ) + })? + }; + + if let Some(obj) = payload.as_object_mut() { + if !obj.contains_key("projectPath") { + let current = current_project_path(); + if !current.is_empty() { + obj.insert("projectPath".to_string(), serde_json::json!(current)); + } + } + } + + let id = NEXT_API_REQUEST_ID + .fetch_add(1, Ordering::Relaxed) + .to_string(); + let request = ApiBridgeRequest { + id: id.clone(), + endpoint: endpoint.to_string(), + payload, + }; + + { + let mut queue = API_BRIDGE_REQUESTS.lock().map_err(|e| { + ( + 500, + serde_json::json!({"ok": false, "error": format!("Bridge queue lock error: {e}")}), + ) + })?; + queue.push(request); + } + + let timeout_secs = if endpoint == "chat" || endpoint.starts_with("ingest/") { + API_LONG_TIMEOUT_SECS + } else { + API_SHORT_TIMEOUT_SECS + }; + let deadline = Instant::now() + Duration::from_secs(timeout_secs); + let mut responses = API_BRIDGE_RESPONSES.lock().map_err(|e| { + ( + 500, + serde_json::json!({"ok": false, "error": format!("Bridge response lock error: {e}")}), + ) + })?; + + loop { + if let Some(response) = responses.remove(&id) { + if response.ok { + return Ok(serde_json::json!({ + "ok": true, + "result": response.payload, + })); + } + return Err(( + 500, + serde_json::json!({ + "ok": false, + "error": response.payload, + }), + )); + } + + let now = Instant::now(); + if now >= deadline { + return Err(( + 504, + serde_json::json!({ + "ok": false, + "error": format!("Timed out waiting for frontend bridge after {timeout_secs}s"), + }), + )); + } + + let remaining = deadline.saturating_duration_since(now); + let (guard, wait_result) = API_BRIDGE_CVAR + .wait_timeout(responses, remaining) + .map_err(|e| { + ( + 500, + serde_json::json!({"ok": false, "error": format!("Bridge wait error: {e}")}), + ) + })?; + responses = guard; + if wait_result.timed_out() { + return Err(( + 504, + serde_json::json!({ + "ok": false, + "error": format!("Timed out waiting for frontend bridge after {timeout_secs}s"), + }), + )); + } + } +} + fn handle_set_project(body: &str) -> String { let parsed: serde_json::Value = match serde_json::from_str(body) { Ok(v) => v, diff --git a/src-tauri/src/commands/fs.rs b/src-tauri/src/commands/fs.rs index e8b32e22..6ef88b14 100644 --- a/src-tauri/src/commands/fs.rs +++ b/src-tauri/src/commands/fs.rs @@ -1,8 +1,8 @@ use std::fs; use std::io::Read as IoRead; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::thread; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use calamine::{Reader, open_workbook_auto, Data}; @@ -899,12 +899,8 @@ pub async fn write_file(path: String, contents: String) -> Result<(), String> { tauri::async_runtime::spawn_blocking(move || { run_guarded("write_file", || { let p = Path::new(&path); - if let Some(parent) = p.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create parent dirs for '{}': {}", path, e))?; - } file_sync::mark_app_write_path(p); - fs::write(&path, contents) + write_file_atomic(p, &contents) .map_err(|e| format!("Failed to write file '{}': {}", path, e))?; file_sync::mark_app_write_path(p); Ok(()) @@ -914,6 +910,67 @@ pub async fn write_file(path: String, contents: String) -> Result<(), String> { .map_err(|e| format!("write_file blocking task join error: {e}"))? } +fn write_file_atomic(path: &Path, contents: &str) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create parent dirs for '{}': {}", path.display(), e))?; + } + + let tmp_path = atomic_temp_path(path); + fs::write(&tmp_path, contents) + .map_err(|e| format!("Failed to write temporary file '{}': {}", tmp_path.display(), e))?; + + match fs::rename(&tmp_path, path) { + Ok(()) => Ok(()), + Err(err) => { + #[cfg(windows)] + { + if path.exists() { + fs::remove_file(path) + .map_err(|remove_err| { + let _ = fs::remove_file(&tmp_path); + format!( + "Failed to replace existing file '{}': {}; original rename error: {}", + path.display(), + remove_err, + err + ) + })?; + return fs::rename(&tmp_path, path).map_err(|rename_err| { + let _ = fs::remove_file(&tmp_path); + format!( + "Failed to move temporary file '{}' to '{}': {}", + tmp_path.display(), + path.display(), + rename_err + ) + }); + } + } + + let _ = fs::remove_file(&tmp_path); + Err(format!( + "Failed to move temporary file '{}' to '{}': {}", + tmp_path.display(), + path.display(), + err + )) + } + } +} + +fn atomic_temp_path(path: &Path) -> PathBuf { + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_else(|| "write-file".to_string()); + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + path.with_file_name(format!(".{file_name}.{stamp}.tmp")) +} + #[tauri::command] pub async fn list_directory(path: String) -> Result, String> { tauri::async_runtime::spawn_blocking(move || { diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7d849c8b..d5df7ab8 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -5,6 +5,8 @@ mod proxy; mod types; use panic_guard::run_guarded; +use serde::Serialize; +use std::path::{Path, PathBuf}; #[tauri::command] fn clip_server_status() -> String { @@ -30,6 +32,77 @@ fn set_proxy_env(config: proxy::ProxyConfig) -> String { summary } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct McpServerConfig { + script_path: String, + codex_command: String, + json_config: String, +} + +fn normalize_display_path(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +fn shell_quote(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\\\"")) +} + +fn build_mcp_config(script_path: &Path) -> Result { + let script_path = normalize_display_path(script_path); + let codex_command = format!( + "codex mcp add llmwiki -- node {}", + shell_quote(&script_path) + ); + let json_config = serde_json::to_string_pretty(&serde_json::json!({ + "mcpServers": { + "llmwiki": { + "command": "node", + "args": [script_path.clone()], + }, + }, + })) + .map_err(|e| format!("Failed to build MCP JSON config: {e}"))?; + + Ok(McpServerConfig { + script_path, + codex_command, + json_config, + }) +} + +fn find_mcp_server_script() -> Result { + let manifest_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf); + let cwd = std::env::current_dir().ok(); + + let mut candidates = Vec::new(); + if let Some(root) = manifest_root { + candidates.push(root.join("mcp-server").join("llmwiki-mcp.js")); + } + if let Some(dir) = cwd { + candidates.push(dir.join("mcp-server").join("llmwiki-mcp.js")); + candidates.push(dir.join("..").join("mcp-server").join("llmwiki-mcp.js")); + } + + for candidate in candidates { + if candidate.is_file() { + return Ok(candidate); + } + } + + Err("Could not find mcp-server/llmwiki-mcp.js. Run from the LLM Wiki source tree or install the MCP server file next to this app.".to_string()) +} + +#[tauri::command] +fn mcp_server_config() -> Result { + run_guarded("mcp_server_config", || { + let script = find_mcp_server_script()?; + build_mcp_config(&script) + }) +} + #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { clip_server::start_clip_server(); @@ -109,6 +182,7 @@ pub fn run() { commands::extract_images::extract_office_images_cmd, commands::extract_images::extract_and_save_pdf_images_cmd, commands::extract_images::extract_and_save_office_images_cmd, + mcp_server_config, commands::file_sync::start_project_file_watcher, commands::file_sync::stop_project_file_watcher, commands::file_sync::rescan_project_files, @@ -163,3 +237,19 @@ pub fn run() { let _ = (app, event); // suppress unused warnings on non-macOS }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mcp_config_contains_node_and_script_path() { + let cfg = build_mcp_config(Path::new("D:/Dev/llm_wiki/mcp-server/llmwiki-mcp.js")) + .expect("config should build"); + + assert!(cfg.codex_command.contains("node")); + assert!(cfg.codex_command.contains("mcp-server/llmwiki-mcp.js")); + assert!(cfg.json_config.contains("\"command\": \"node\"")); + assert!(cfg.json_config.contains("mcp-server/llmwiki-mcp.js")); + } +} diff --git a/src/App.tsx b/src/App.tsx index 39e7137e..392d21fc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -5,10 +5,11 @@ import { useWikiStore } from "@/stores/wiki-store" import { useReviewStore } from "@/stores/review-store" import { useChatStore } from "@/stores/chat-store" import { listDirectory, openProject } from "@/commands/fs" -import { getLastProject, getRecentProjects, saveLastProject, loadLlmConfig, loadLanguage, loadSearchApiConfig, loadEmbeddingConfig, loadMultimodalConfig, loadOutputLanguage, loadProviderConfigs, loadActivePresetId, loadProxyConfig, loadProjectFileSyncEnabled } from "@/lib/project-store" +import { getLastProject, getRecentProjects, saveLastProject, loadLlmConfig, loadLanguage, loadSearchApiConfig, loadEmbeddingConfig, loadDocumentLlmConfig, loadMultimodalConfig, loadOutputLanguage, loadProviderConfigs, loadActivePresetId, loadProxyConfig, loadMcpAccessEnabled, loadProjectFileSyncEnabled } from "@/lib/project-store" import { loadReviewItems, loadChatHistory } from "@/lib/persist" import { setupAutoSave } from "@/lib/auto-save" import { startClipWatcher } from "@/lib/clip-watcher" +import { startLocalApiBridge } from "@/lib/local-api-bridge" import { AppLayout } from "@/components/layout/app-layout" import { WelcomeScreen } from "@/components/project/welcome-screen" import { CreateProjectDialog } from "@/components/project/create-project-dialog" @@ -27,6 +28,7 @@ function App() { useEffect(() => { setupAutoSave() startClipWatcher() + startLocalApiBridge() }, []) // Dev-only helper for visually testing the update-banner UX. @@ -214,6 +216,10 @@ function App() { if (savedEmbeddingConfig) { useWikiStore.getState().setEmbeddingConfig(savedEmbeddingConfig) } + const savedDocumentLlmConfig = await loadDocumentLlmConfig() + if (savedDocumentLlmConfig) { + useWikiStore.getState().setDocumentLlmConfig(savedDocumentLlmConfig) + } const savedMultimodalConfig = await loadMultimodalConfig() if (savedMultimodalConfig) { useWikiStore.getState().setMultimodalConfig(savedMultimodalConfig) @@ -222,6 +228,8 @@ function App() { if (savedProxy) { useWikiStore.getState().setProxyConfig(savedProxy) } + const savedMcpAccessEnabled = await loadMcpAccessEnabled() + useWikiStore.getState().setMcpAccessEnabled(savedMcpAccessEnabled) const savedLang = await loadLanguage() if (savedLang) { await i18n.changeLanguage(savedLang) @@ -313,11 +321,10 @@ function App() { // Load persisted review items try { const savedReview = await loadReviewItems(proj.path) - if (savedReview.length > 0) { - useReviewStore.getState().setItems(savedReview) - } + useReviewStore.getState().setItems(savedReview) } catch { // ignore, start fresh + useReviewStore.getState().setItems([]) } // Load persisted chat history try { diff --git a/src/commands/fs.ts b/src/commands/fs.ts index 4efe013e..8987afd1 100644 --- a/src/commands/fs.ts +++ b/src/commands/fs.ts @@ -97,3 +97,13 @@ export async function openProjectFolder(path: string): Promise { export async function clipServerStatus(): Promise { return invoke("clip_server_status") } + +export interface McpServerConfig { + scriptPath: string + codexCommand: string + jsonConfig: string +} + +export async function mcpServerConfig(): Promise { + return invoke("mcp_server_config") +} diff --git a/src/components/chat/chat-message.tsx b/src/components/chat/chat-message.tsx index ae23d8eb..dd8c2b8f 100644 --- a/src/components/chat/chat-message.tsx +++ b/src/components/chat/chat-message.tsx @@ -18,7 +18,7 @@ import type { FileNode } from "@/types/wiki" import { convertLatexToUnicode } from "@/lib/latex-to-unicode" import { normalizePath, getFileName } from "@/lib/path-utils" import { makeQueryFileName } from "@/lib/wiki-filename" -import { hasUsableLlm } from "@/lib/has-usable-llm" +import { hasUsableDocumentLlm } from "@/lib/has-usable-llm" import { resolveMarkdownImageSrc } from "@/lib/markdown-image-resolver" import { findRawSourceForImage, imageUrlToAbsolute } from "@/lib/raw-source-resolver" import { detectLanguage } from "@/lib/detect-language" @@ -234,8 +234,9 @@ function SaveToWikiButton({ content, visible }: { content: string; visible: bool setTimeout(() => setSaved(false), 2000) // Full auto-ingest: extract entities, concepts, cross-references from saved content - const llmConfig = useWikiStore.getState().llmConfig - if (hasUsableLlm(llmConfig)) { + const store = useWikiStore.getState() + const llmConfig = store.llmConfig + if (hasUsableDocumentLlm(llmConfig, store.documentLlmConfig)) { const { autoIngest } = await import("@/lib/ingest") autoIngest(pp, filePath, llmConfig).catch((err) => console.error("Failed to auto-ingest saved query:", err) diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsx index 65ef8ad5..68416c7a 100644 --- a/src/components/chat/chat-panel.tsx +++ b/src/components/chat/chat-panel.tsx @@ -7,13 +7,10 @@ import { useChatStore, chatMessagesToLLM } from "@/stores/chat-store" import { useWikiStore } from "@/stores/wiki-store" import { streamChat, type ChatMessage as LLMMessage } from "@/lib/llm-client" import { executeIngestWrites } from "@/lib/ingest" -import { listDirectory, readFile, deleteFile } from "@/commands/fs" -import { searchWiki } from "@/lib/search" -import { buildRetrievalGraph, getRelatedNodes } from "@/lib/graph-relevance" -import { normalizePath, getFileName, getRelativePath } from "@/lib/path-utils" -import { getOutputLanguage, buildLanguageReminder } from "@/lib/output-language" -import { isGreeting } from "@/lib/greeting-detector" -import { computeContextBudget } from "@/lib/context-budget" +import { listDirectory, deleteFile } from "@/commands/fs" +import { normalizePath } from "@/lib/path-utils" +import { buildChatRetrievalContext } from "@/lib/chat-retrieval" +import { resolveDocumentLlmConfig } from "@/lib/document-llm" // Store the page mapping from the last query so SourceFilesBar can show which pages were cited export let lastQueryPages: { title: string; path: string }[] = [] @@ -165,188 +162,18 @@ export function ChatPanel() { addMessage("user", text) setStreaming(true) - // Build system prompt with wiki context using graph-enhanced retrieval - const systemMessages: LLMMessage[] = [] - let queryRefs: { title: string; path: string }[] = [] - let langReminder: string | undefined - // Pure greetings ("hi", "你好", "嗨") don't warrant running the whole - // retrieval pipeline — it's slow, costs context, and drags in random - // wiki pages the user clearly didn't ask about. Short-circuit with a - // minimal system prompt and let the model reply conversationally. - const greetingOnly = isGreeting(text) - if (project && greetingOnly) { - const outLang = getOutputLanguage(text) - systemMessages.push({ - role: "system", - content: [ - `You are a wiki assistant for the project "${project.name}".`, - "The user sent a casual greeting — reply briefly and naturally, in one or two sentences.", - "Do NOT invent wiki content or pretend to have retrieved pages. Invite the user to ask a concrete question if they want information from the wiki.", - "", - `Respond in ${outLang}.`, - ].join("\n"), - }) - // Skip retrieval; queryRefs stays empty so no "Sources" chip is shown. - } else if (project) { - const pp = normalizePath(project.path) - const dataVersion = useWikiStore.getState().dataVersion - - // ── Budget allocation (see context-budget.ts) ───────── - // Page budget scales with the LLM's context window; we now - // also reserve ~15% as headroom for the response so the - // model isn't truncated mid-sentence on a packed prompt. - const { - indexBudget: INDEX_BUDGET, - pageBudget: PAGE_BUDGET, - maxPageSize: MAX_PAGE_SIZE, - } = computeContextBudget(llmConfig.maxContextSize) - - const [rawIndex, purpose] = await Promise.all([ - readFile(`${pp}/wiki/index.md`).catch(() => ""), - readFile(`${pp}/purpose.md`).catch(() => ""), - ]) - - // ── Phase 1: Tokenized search → top 10 ──────────────── - const searchResults = await searchWiki(pp, text) - const topSearchResults = searchResults.slice(0, 10) - - // ── Trim index by relevance if over budget ───────────── - let index = rawIndex - if (rawIndex.length > INDEX_BUDGET) { - const { tokenizeQuery } = await import("@/lib/search") - const tokens = tokenizeQuery(text) - const lines = rawIndex.split("\n") - const keptLines: string[] = [] - let keptSize = 0 - - for (const line of lines) { - const isHeader = line.startsWith("##") - const lower = line.toLowerCase() - const isRelevant = tokens.some((t) => lower.includes(t)) - - if (isHeader || isRelevant) { - if (keptSize + line.length + 1 <= INDEX_BUDGET) { - keptLines.push(line) - keptSize += line.length + 1 - } - } - } - index = keptLines.join("\n") - if (index.length < rawIndex.length) { - index += "\n\n[...index trimmed to relevant entries...]" - } - } - - // ── Phase 2: Graph 1-level expansion ─────────────────── - // Note: Vector search (if enabled) is already merged into searchResults - // by searchWiki() in search.ts — no duplicate code needed here. - const graph = await buildRetrievalGraph(pp, dataVersion) - const expandedIds = new Set() - const searchHitPaths = new Set(topSearchResults.map((r) => r.path)) - const graphExpansions: { title: string; path: string; relevance: number }[] = [] - - for (const result of topSearchResults) { - const fileName = getFileName(result.path) - const nodeId = fileName.replace(/\.md$/, "") - const related = getRelatedNodes(nodeId, graph, 3) - for (const { node, relevance } of related) { - if (relevance < 2.0) continue - if (searchHitPaths.has(node.path)) continue - if (expandedIds.has(node.id)) continue - expandedIds.add(node.id) - graphExpansions.push({ title: node.title, path: node.path, relevance }) - } - } - graphExpansions.sort((a, b) => b.relevance - a.relevance) - - // ── Phase 3 & 4: Page budget control ─────────────────── - let usedChars = 0 - type PageEntry = { title: string; path: string; content: string; priority: number } - const relevantPages: PageEntry[] = [] - - const tryAddPage = async (title: string, filePath: string, priority: number): Promise => { - if (usedChars >= PAGE_BUDGET) return false - try { - const raw = await readFile(filePath) - const relativePath = getRelativePath(filePath, pp) - const truncated = raw.length > MAX_PAGE_SIZE - ? raw.slice(0, MAX_PAGE_SIZE) + "\n\n[...truncated...]" - : raw - if (usedChars + truncated.length > PAGE_BUDGET) return false - usedChars += truncated.length - relevantPages.push({ title, path: relativePath, content: truncated, priority }) - return true - } catch { return false } - } - - // P0: Title matches - for (const r of topSearchResults.filter((r) => r.titleMatch)) { - await tryAddPage(r.title, r.path, 0) - } - // P1: Content matches - for (const r of topSearchResults.filter((r) => !r.titleMatch)) { - await tryAddPage(r.title, r.path, 1) - } - // P2: Graph expansions - for (const exp of graphExpansions) { - await tryAddPage(exp.title, exp.path, 2) - } - // P3: Overview fallback - if (relevantPages.length === 0) { - await tryAddPage("Overview", `${pp}/wiki/overview.md`, 3) - } - - const pagesContext = relevantPages.length > 0 - ? relevantPages.map((p, i) => - `### [${i + 1}] ${p.title}\nPath: ${p.path}\n\n${p.content}` - ).join("\n\n---\n\n") - : "(No wiki pages found)" - - const pageList = relevantPages.map((p, i) => - `[${i + 1}] ${p.title} (${p.path})` - ).join("\n") - - const outLang = getOutputLanguage(text) - - systemMessages.push({ - role: "system", - content: [ - "You are a knowledgeable wiki assistant. Answer questions based on the wiki content provided below.", - "", - "## Rules", - "- Answer based ONLY on the numbered wiki pages provided below.", - "- If the provided pages don't contain enough information, say so honestly.", - "- Use [[wikilink]] syntax to reference wiki pages.", - "- When citing information, use the page number in brackets, e.g. [1], [2].", - "- At the VERY END of your response, add a hidden comment listing which page numbers you used:", - " ", - "", - "Use markdown formatting for clarity.", - "", - purpose ? `## Wiki Purpose\n${purpose}` : "", - index ? `## Wiki Index\n${index}` : "", - relevantPages.length > 0 ? `## Page List\n${pageList}` : "", - `## Wiki Pages\n\n${pagesContext}`, - "", - "---", - "", - `## ⚠️ MANDATORY OUTPUT LANGUAGE: ${outLang}`, - "", - `You MUST write your entire response in **${outLang}**.`, - `The wiki content above may be in a different language, but this is IRRELEVANT to your output language.`, - `Ignore the language of the wiki content. Write in ${outLang} only.`, - `Even proper nouns should use standard ${outLang} transliteration when appropriate.`, - `DO NOT use any other language. This overrides all other instructions.`, - ].filter(Boolean).join("\n"), - }) - - // Reminder injected later, right before the user's current message - // (after history so it's the last system instruction the LLM sees). - langReminder = buildLanguageReminder(text) - - lastQueryPages = relevantPages.map((p) => ({ title: p.title, path: p.path })) - queryRefs = [...lastQueryPages] - } + const retrieval = project + ? await buildChatRetrievalContext({ + project, + query: text, + llmConfig, + dataVersion: useWikiStore.getState().dataVersion, + }) + : null + const systemMessages = retrieval?.systemMessages ?? [] + const queryRefs = retrieval?.references ?? [] + const langReminder = retrieval?.langReminder + lastQueryPages = [...queryRefs] // ── Conversation history with count limit ──────────────── // Only include messages from the active conversation, last N messages @@ -460,7 +287,11 @@ export function ChatPanel() { if (!project) return const pp = normalizePath(project.path) try { - await executeIngestWrites(pp, llmConfig, undefined, undefined) + const ingestLlm = resolveDocumentLlmConfig( + llmConfig, + useWikiStore.getState().documentLlmConfig, + ) + await executeIngestWrites(pp, ingestLlm, undefined, undefined) try { const tree = await listDirectory(pp) setFileTree(tree) diff --git a/src/components/layout/icon-sidebar.tsx b/src/components/layout/icon-sidebar.tsx index a95f7546..ed382a74 100644 --- a/src/components/layout/icon-sidebar.tsx +++ b/src/components/layout/icon-sidebar.tsx @@ -10,6 +10,8 @@ import { useUpdateStore, hasAvailableUpdate } from "@/stores/update-store" import { useTranslation } from "react-i18next" import logoImg from "@/assets/logo.jpg" import type { WikiState } from "@/stores/wiki-store" +import { normalizePath } from "@/lib/path-utils" +import { bucketReviewItems } from "@/lib/review-utils" type NavView = WikiState["activeView"] @@ -30,9 +32,14 @@ export function IconSidebar({ onSwitchProject }: IconSidebarProps) { const { t } = useTranslation() const activeView = useWikiStore((s) => s.activeView) const setActiveView = useWikiStore((s) => s.setActiveView) - const pendingCount = useReviewStore((s) => s.items.filter((i) => !i.resolved).length) + const currentProjectPath = useWikiStore((s) => s.project?.path ? normalizePath(s.project.path) : "") + const pendingCount = useReviewStore((s) => + bucketReviewItems(s.items, currentProjectPath).currentPending.length, + ) const researchPanelOpen = useResearchStore((s) => s.panelOpen) - const researchActiveCount = useResearchStore((s) => s.tasks.filter((t) => t.status !== "done" && t.status !== "error").length) + const researchActiveCount = useResearchStore((s) => + s.tasks.filter((t) => t.status !== "done" && t.status !== "error").length, + ) const toggleResearchPanel = useResearchStore((s) => s.setPanelOpen) // Use `hasAvailableUpdate` (ignores dismiss state) rather than // `shouldShowUpdateBanner`. The dot is a passive signpost — it diff --git a/src/components/layout/research-panel.tsx b/src/components/layout/research-panel.tsx index e3542124..6fa0535b 100644 --- a/src/components/layout/research-panel.tsx +++ b/src/components/layout/research-panel.tsx @@ -6,14 +6,14 @@ import rehypeKatex from "rehype-katex" import "katex/dist/katex.min.css" import { Search, Loader2, CheckCircle2, AlertCircle, ChevronRight, ChevronDown, X, - FileText, Send, + FileText, Send, RotateCcw, Trash2, } from "lucide-react" import { Button } from "@/components/ui/button" import { useResearchStore, type ResearchTask } from "@/stores/research-store" import { useWikiStore } from "@/stores/wiki-store" import { readFile } from "@/commands/fs" -import { queueResearch } from "@/lib/deep-research" -import { normalizePath } from "@/lib/path-utils" +import { queueResearch, retryResearchTask } from "@/lib/deep-research" +import { getFileName, normalizePath } from "@/lib/path-utils" import { isImeComposing } from "@/lib/keyboard-utils" import { detectLanguage } from "@/lib/detect-language" import { getHtmlLang, getTextDirection } from "@/lib/language-metadata" @@ -22,6 +22,7 @@ import { MermaidDiagram, unwrapMermaidPre } from "@/components/mermaid-diagram" export function ResearchPanel() { const tasks = useResearchStore((s) => s.tasks) const removeTask = useResearchStore((s) => s.removeTask) + const clearFinished = useResearchStore((s) => s.clearFinished) const setPanelOpen = useResearchStore((s) => s.setPanelOpen) const project = useWikiStore((s) => s.project) const llmConfig = useWikiStore((s) => s.llmConfig) @@ -55,12 +56,25 @@ export function ResearchPanel() { )} - +
+ {done.length > 0 && ( + + )} + +
{/* Research input */} @@ -220,8 +234,11 @@ function ResearchTaskCard({ task, onRemove }: { task: ResearchTask; onRemove: (i ) const setSelectedFile = useWikiStore((s) => s.setSelectedFile) const setFileContent = useWikiStore((s) => s.setFileContent) - const project = useWikiStore((s) => s.project) - + const llmConfig = useWikiStore((s) => s.llmConfig) + const searchApiConfig = useWikiStore((s) => s.searchApiConfig) + const currentProjectPath = useWikiStore((s) => s.project?.path ? normalizePath(s.project.path) : "") + const isCurrentProject = normalizePath(task.projectPath) === currentProjectPath + const projectLabel = getFileName(task.projectPath.replace(/[\\/]+$/, "")) || task.projectPath const statusIcon = { queued:
, searching: , @@ -241,8 +258,8 @@ function ResearchTaskCard({ task, onRemove }: { task: ResearchTask; onRemove: (i }[task.status] async function handleOpenSaved() { - if (!project || !task.savedPath) return - const path = `${normalizePath(project.path)}/${task.savedPath}` + if (!task.savedPath) return + const path = `${normalizePath(task.projectPath)}/${task.savedPath}` try { const content = await readFile(path) setSelectedFile(path) @@ -265,7 +282,20 @@ function ResearchTaskCard({ task, onRemove }: { task: ResearchTask; onRemove: (i )} {statusIcon} - {task.topic} +
+
{task.topic}
+
+ + {projectLabel} + +
+
{statusText} @@ -310,6 +340,17 @@ function ResearchTaskCard({ task, onRemove }: { task: ResearchTask; onRemove: (i Open )} + {task.status === "error" && ( + + )} {(task.status === "done" || task.status === "error") && (

- All AI-generated content (wiki pages, chat replies, research - output) will use this language. You can change it later in - Settings → Output. + This controls how AI writes across chat, wiki generation, and + research. The default mode is Chinese-first while preserving + necessary English terms. You can change it later in Settings → + Output.

diff --git a/src/components/review/review-view.tsx b/src/components/review/review-view.tsx index 343f6ab4..9a14151a 100644 --- a/src/components/review/review-view.tsx +++ b/src/components/review/review-view.tsx @@ -1,4 +1,4 @@ -import { useCallback } from "react" +import { useCallback, useEffect, useState } from "react" import { queueResearch } from "@/lib/deep-research" import { AlertTriangle, @@ -16,6 +16,9 @@ import { useReviewStore, type ReviewItem } from "@/stores/review-store" import { useWikiStore } from "@/stores/wiki-store" import { writeFile, readFile, listDirectory, deleteFile } from "@/commands/fs" import { normalizePath } from "@/lib/path-utils" +import { getRecentProjects } from "@/lib/project-store" +import type { WikiProject } from "@/types/wiki" +import { bucketReviewItems, needsProjectAssignment } from "@/lib/review-utils" const typeConfig: Record = { contradiction: { icon: AlertTriangle, label: "Contradiction", color: "text-amber-500" }, @@ -32,17 +35,37 @@ export function ReviewView() { const clearResolved = useReviewStore((s) => s.clearResolved) const project = useWikiStore((s) => s.project) const setFileTree = useWikiStore((s) => s.setFileTree) + const currentProjectPath = project ? normalizePath(project.path) : "" + const [activeTab, setActiveTab] = useState<"current" | "orphan">("current") + const [recentProjects, setRecentProjects] = useState([]) + + useEffect(() => { + getRecentProjects().then(setRecentProjects).catch(() => {}) + }, []) + + function resolveTargetProject(item: ReviewItem): { id: string; path: string } | null { + if (!project) return null + const itemPath = normalizePath(item.projectPath) + if (itemPath !== currentProjectPath) return null + return { id: item.projectId, path: itemPath } + } const handleResolve = useCallback(async (id: string, action: string) => { - const pp = project ? normalizePath(project.path) : "" + const item = items.find((i) => i.id === id) + const targetProject = item ? resolveTargetProject(item) : null + const pp = targetProject?.path ?? "" + + if (item && !targetProject) { + window.alert("This review item belongs to a different project. Open that project first.") + return + } // Deep Research — must be checked FIRST before any fuzzy matching - if (action === "__deep_research__" && project) { + if (action === "__deep_research__" && targetProject) { const searchConfig = useWikiStore.getState().searchApiConfig if (searchConfig.provider === "none" || !searchConfig.apiKey) { window.alert("Web Search not configured. Go to Settings → Web Search to add a Tavily or SerpApi API key first.") return } - const item = items.find((i) => i.id === id) if (item) { const llmConfig = useWikiStore.getState().llmConfig // Use pre-generated search queries if available, otherwise fall back to title @@ -55,7 +78,7 @@ export function ReviewView() { return } - if (action.startsWith("save:") && project) { + if (action.startsWith("save:") && targetProject) { // Decode and save the content to wiki try { const encoded = action.slice(5) @@ -105,7 +128,7 @@ export function ReviewView() { console.error("Failed to save to wiki from review:", err) resolveItem(id, "Save failed") } - } else if (action.startsWith("open:") && project) { + } else if (action.startsWith("open:") && targetProject) { // Open a page for editing const page = action.slice(5) const candidates = [ @@ -124,7 +147,7 @@ export function ReviewView() { } } resolveItem(id, action) - } else if (action.startsWith("delete:") && project) { + } else if (action.startsWith("delete:") && targetProject) { // Delete a file const filePath = action.slice(7) try { @@ -136,7 +159,7 @@ export function ReviewView() { console.error("Failed to delete:", err) resolveItem(id, "Delete failed") } - } else if (actionLooksLikeResearch(action) && project) { + } else if (actionLooksLikeResearch(action) && targetProject) { // Actions with "research" trigger deep research, not just page creation const searchConfig = useWikiStore.getState().searchApiConfig if (searchConfig.provider === "none" || !searchConfig.apiKey) { @@ -147,7 +170,6 @@ export function ReviewView() { } return } - const item = items.find((i) => i.id === id) if (item) { const llmConfig = useWikiStore.getState().llmConfig const topic = action.replace(/^research\s*/i, "").trim() || item.description.split("\n")[0] @@ -158,7 +180,7 @@ export function ReviewView() { } } else if ( (action.startsWith("__create_page__:") || actionLooksLikeCreate(action)) - && project + && targetProject ) { // Create a wiki page from the review item's content. Accepts both // the `__create_page__:` sentinel (forced via the "no search API" @@ -167,7 +189,6 @@ export function ReviewView() { const realAction = action.startsWith("__create_page__:") ? action.slice("__create_page__:".length) : action - const item = items.find((i) => i.id === id) if (item) { try { const title = item.title.replace(/^(Create|Save|Add)[:\s]*/i, "").trim() || "Untitled" @@ -219,24 +240,52 @@ export function ReviewView() { } else { resolveItem(id, action) } - }, [project, items, resolveItem, setFileTree]) + }, [currentProjectPath, items, project, resolveItem, setFileTree]) - const pending = items.filter((i) => !i.resolved) - const resolved = items.filter((i) => i.resolved) + const { currentPending: pending, currentResolved: resolved, unassigned: orphanItems } = + bucketReviewItems(items, currentProjectPath) + + const assignOrphanToProject = useCallback((itemId: string, target: WikiProject) => { + useReviewStore.setState((state) => ({ + items: state.items.map((item) => + item.id === itemId + ? { ...item, projectId: target.id, projectPath: normalizePath(target.path) } + : item, + ), + })) + }, []) return (
-

- Review - {pending.length > 0 && ( - - {pending.length} - - )} -

+
+

+ Review + {pending.length > 0 && ( + + {pending.length} + + )} +

+
+ + +
+
{resolved.length > 0 && ( - @@ -244,7 +293,36 @@ export function ReviewView() {
- {items.length === 0 ? ( + {activeTab === "orphan" ? ( + orphanItems.length === 0 ? ( +
+ +

No unassigned review items

+
+ ) : ( +
+ {orphanItems.map((item) => ( +
+
{item.title}
+

{item.description}

+
+ {recentProjects.map((proj) => ( + + ))} +
+
+ ))} +
+ ) + ) : pending.length === 0 && resolved.length === 0 ? (

All clear — nothing to review

@@ -257,6 +335,8 @@ export function ReviewView() { item={item} onResolve={handleResolve} onDismiss={dismissItem} + currentProject={project} + onAssignToProject={assignOrphanToProject} /> ))} {resolved.length > 0 && pending.length > 0 && ( @@ -270,6 +350,8 @@ export function ReviewView() { item={item} onResolve={handleResolve} onDismiss={dismissItem} + currentProject={project} + onAssignToProject={assignOrphanToProject} /> ))}
@@ -283,13 +365,18 @@ function ReviewCard({ item, onResolve, onDismiss, + currentProject, + onAssignToProject, }: { item: ReviewItem onResolve: (id: string, action: string) => void onDismiss: (id: string) => void + currentProject: WikiProject | null + onAssignToProject: (itemId: string, target: WikiProject) => void }) { const config = typeConfig[item.type] const Icon = config.icon + const showAssignCurrent = currentProject && needsProjectAssignment(item, currentProject.path) return (
+ {showAssignCurrent && currentProject && ( + + )} {(item.type === "suggestion" || item.type === "missing-page") && ( +
+ 
+        {value}
+
+
+ ) +} diff --git a/src/components/settings/sections/document-llm-section.tsx b/src/components/settings/sections/document-llm-section.tsx new file mode 100644 index 00000000..3536e2f3 --- /dev/null +++ b/src/components/settings/sections/document-llm-section.tsx @@ -0,0 +1,201 @@ +import { useTranslation } from "react-i18next" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { ContextSizeSelector } from "../context-size-selector" +import type { SettingsDraft, DraftSetter } from "../settings-types" + +interface Props { + draft: SettingsDraft + setDraft: DraftSetter +} + +const PROVIDER_OPTIONS: Array<{ value: SettingsDraft["documentProvider"]; label: string }> = [ + { value: "custom", label: "Custom (OpenAI-compat)" }, + { value: "openai", label: "OpenAI" }, + { value: "anthropic", label: "Anthropic" }, + { value: "google", label: "Google (Gemini)" }, + { value: "ollama", label: "Ollama" }, + { value: "claude-code", label: "Claude Code CLI" }, + { value: "minimax", label: "MiniMax" }, +] + +const REASONING_OPTIONS = [ + { value: "auto", label: "Auto" }, + { value: "off", label: "Off" }, + { value: "low", label: "Low" }, + { value: "medium", label: "Medium" }, + { value: "high", label: "High" }, + { value: "max", label: "Max" }, + { value: "custom", label: "Custom" }, +] as const + +export function DocumentLlmSection({ draft, setDraft }: Props) { + const { t } = useTranslation() + const reasoning = draft.documentReasoning ?? { mode: "auto" as const } + + return ( +
+
+

{t("settings.sections.document.title")}

+

+ {t("settings.sections.document.description")} +

+
+ +
+
+
+ {t("settings.sections.document.useMainLabel")} +
+
+ {t("settings.sections.document.useMainHint")} +
+
+ +
+ + {!draft.documentUseMainLlm && ( +
+
+ {t("settings.sections.document.dedicatedHeading")} +
+ +
+ + +
+ + {draft.documentProvider === "ollama" && ( +
+ + setDraft("documentOllamaUrl", e.target.value)} + placeholder="http://localhost:11434" + /> +
+ )} + + {draft.documentProvider === "custom" && ( +
+ + setDraft("documentCustomEndpoint", e.target.value)} + placeholder="http://localhost:1234/v1" + /> +

+ {t("settings.sections.document.customEndpointHint")} +

+
+ )} + + {draft.documentProvider !== "ollama" && draft.documentProvider !== "claude-code" && ( +
+ + setDraft("documentApiKey", e.target.value)} + placeholder={t("settings.sections.document.apiKeyPlaceholder")} + /> +
+ )} + +
+ + setDraft("documentModel", e.target.value)} + placeholder={t("settings.sections.document.modelPlaceholder")} + /> +

+ {t("settings.sections.document.modelHint")} +

+
+ +
+ + setDraft("documentMaxContextSize", v)} + /> +
+ +
+ +
+ {REASONING_OPTIONS.map((m) => { + const active = reasoning.mode === m.value + return ( + + ) + })} +
+ {reasoning.mode === "custom" && ( +
+ { + const raw = e.target.value.trim() + const n = Number(raw) + setDraft("documentReasoning", { + ...reasoning, + budgetTokens: raw === "" || !Number.isFinite(n) ? undefined : Math.max(0, n), + }) + }} + placeholder="1024" + /> + + {t("settings.sections.document.reasoningBudget")} + +
+ )} +

+ {t("settings.sections.document.reasoningHint")} +

+
+
+ )} +
+ ) +} diff --git a/src/components/settings/sections/maintenance-section.tsx b/src/components/settings/sections/maintenance-section.tsx index 4b5151ad..a4a278e9 100644 --- a/src/components/settings/sections/maintenance-section.tsx +++ b/src/components/settings/sections/maintenance-section.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from "react" +import { useCallback, useEffect, useMemo, useRef, useState } from "react" import { useTranslation } from "react-i18next" import { Wrench, @@ -64,6 +64,8 @@ export function MaintenanceSection({ draft, setDraft }: Props) { // that completed while the user was on a different settings tab). // Same pattern activity-panel uses for ingest-queue. const [tasks, setTasks] = useState([]) + const lastSeenTaskKeysRef = useRef>(new Set()) + const cancelledTaskKeysRef = useRef>(new Set()) useEffect(() => { setTasks([...getQueue()]) const id = setInterval(() => setTasks([...getQueue()]), 1000) @@ -109,6 +111,7 @@ export function MaintenanceSection({ draft, setDraft }: Props) { async (entry: GroupUiEntry) => { if (!project) return try { + cancelledTaskKeysRef.current.delete(groupKey(entry.group.slugs)) await enqueueMerge(project.id, entry.group, entry.canonicalSlug) // Refresh immediately so the card flips to "queued" without // waiting for the next 1s poll tick. @@ -121,6 +124,10 @@ export function MaintenanceSection({ draft, setDraft }: Props) { ) const handleCancel = useCallback(async (taskId: string) => { + const task = getQueue().find((t) => t.id === taskId) + if (task) { + cancelledTaskKeysRef.current.add(groupKey(task.group.slugs)) + } await cancelTask(taskId) setTasks([...getQueue()]) }, []) @@ -147,43 +154,31 @@ export function MaintenanceSection({ draft, setDraft }: Props) { [project, groups], ) - // Drive each card's status from the queue. - // - Card not in queue + not skipped → idle, can merge / dismiss - // - Task pending → "Queued (N ahead)" - // - Task processing → "Merging…" - // - Task gone (after success) → "Merged" (queue removes done tasks - // immediately, so we only know it succeeded if we observed it - // in-flight before. Track that with a session-local set.) - // - Task failed → show error + retry / delete. - const [recentlyMergedKeys, setRecentlyMergedKeys] = useState>( - () => new Set(), - ) - + // Queue state drives each card: + // - pending / processing / failed stay visible with their task status. + // - explicitly cancelled tasks stay in the current scan result. + // - tasks that vanish without a cancel were completed successfully, + // so remove their candidate group from the current scan result. useEffect(() => { - // Detect transitions out of the queue: a slug-set we saw last - // tick is now gone → it completed (cancelled paths also remove, - // but only with explicit user action that re-renders separately). - setRecentlyMergedKeys((prev) => { - const currentKeys = new Set(tasks.map((t) => groupKey(t.group.slugs))) - let changed = false - const next = new Set(prev) - for (const g of groups) { - const k = groupKey(g.group.slugs) - const wasInFlight = lastSeenTaskKeysRef.current.has(k) - if (wasInFlight && !currentKeys.has(k) && !next.has(k)) { - next.add(k) - changed = true - } + const currentKeys = new Set(tasks.map((t) => groupKey(t.group.slugs))) + const completedKeys = new Set() + + for (const key of lastSeenTaskKeysRef.current) { + if (currentKeys.has(key)) continue + if (cancelledTaskKeysRef.current.has(key)) { + cancelledTaskKeysRef.current.delete(key) + } else { + completedKeys.add(key) } - lastSeenTaskKeysRef.current = currentKeys - return changed ? next : prev - }) - // We intentionally only re-run when tasks change — the closure - // over `groups` is fine because newly-scanned groups can't be - // "recently merged" until they've been observed in-flight first. - // eslint-disable-next-line react-hooks/exhaustive-deps + } + + lastSeenTaskKeysRef.current = currentKeys + if (completedKeys.size > 0) { + setGroups((prev) => + prev.filter((g) => !completedKeys.has(groupKey(g.group.slugs))), + ) + } }, [tasks]) - const lastSeenTaskKeysRef = useRefInit>(() => new Set()) // Pending position helper: "queued (N ahead)" — count pending tasks // before this one in arrival order. @@ -324,13 +319,11 @@ export function MaintenanceSection({ draft, setDraft }: Props) { {groups.map((entry, idx) => { const task = findTaskForGroup(tasks, entry.group.slugs) - const merged = recentlyMergedKeys.has(groupKey(entry.group.slugs)) return ( (init: () => T): { current: T } { - // `useState` returning a ref-shaped object lets us mutate `.current` - // without triggering re-renders, which is exactly the ref semantics - // we want for the "last seen task keys" tracking above. - // eslint-disable-next-line react-hooks/rules-of-hooks - const [ref] = useState<{ current: T }>(() => ({ current: init() })) - return ref -} - interface QueueOrphanListProps { tasks: readonly DedupTask[] groups: GroupUiEntry[] @@ -503,7 +485,6 @@ function TaskStatusChip({ task, pendingPosition }: ChipProps) { interface CardProps { entry: GroupUiEntry task: DedupTask | undefined - merged: boolean pendingPosition: number onCanonicalChange: (slug: string) => void onEnqueue: () => void @@ -515,7 +496,6 @@ interface CardProps { function DuplicateGroupCard({ entry, task, - merged, pendingPosition, onCanonicalChange, onEnqueue, @@ -528,7 +508,7 @@ function DuplicateGroupCard({ const inFlight = !!task && (task.status === "pending" || task.status === "processing") const failed = !!task && task.status === "failed" - const finished = merged || skipped + const finished = skipped const confidenceClass = group.confidence === "high" @@ -553,12 +533,6 @@ function DuplicateGroupCard({ n: group.slugs.length, })} - {merged && ( - - - {t("settings.sections.maintenance.dedup.merged", { defaultValue: "Merged" })} - - )} {skipped && ( {t("settings.sections.maintenance.dedup.skipped", { defaultValue: "Marked not duplicates" })} diff --git a/src/components/settings/settings-types.ts b/src/components/settings/settings-types.ts index 30030386..e14aa6ca 100644 --- a/src/components/settings/settings-types.ts +++ b/src/components/settings/settings-types.ts @@ -28,6 +28,17 @@ export interface SettingsDraft { /** Overlap characters between adjacent chunks. Empty = default (200). */ embeddingOverlapChunkChars: number | undefined + // Document processing (ingest / merge / Save to Wiki) + documentUseMainLlm: boolean + documentProvider: "openai" | "anthropic" | "google" | "ollama" | "custom" | "minimax" | "claude-code" + documentApiKey: string + documentModel: string + documentOllamaUrl: string + documentCustomEndpoint: string + documentMaxContextSize: number + documentApiMode: CustomApiMode | undefined + documentReasoning: ReasoningConfig | undefined + // Multimodal (image captioning at ingest time) multimodalEnabled: boolean multimodalUseMainLlm: boolean diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsx index 1d15de0b..12b87f81 100644 --- a/src/components/settings/settings-view.tsx +++ b/src/components/settings/settings-view.tsx @@ -22,6 +22,7 @@ import { loadProjectFileSyncEnabled, saveLanguage } from "@/lib/project-store" import type { SettingsDraft, DraftSetter } from "./settings-types" import { LlmProviderSection } from "./sections/llm-provider-section" import { EmbeddingSection } from "./sections/embedding-section" +import { DocumentLlmSection } from "./sections/document-llm-section" import { MultimodalSection } from "./sections/multimodal-section" import { WebSearchSection } from "./sections/web-search-section" import { OutputSection } from "./sections/output-section" @@ -30,13 +31,16 @@ import { NetworkSection } from "./sections/network-section" import { ChangelogSection } from "./sections/changelog-section" import { MaintenanceSection } from "./sections/maintenance-section" import { AboutSection } from "./sections/about-section" +import { AgentAccessSection } from "./sections/agent-access-section" type CategoryId = | "llm" | "embedding" + | "document" | "multimodal" | "web-search" | "network" + | "agent" | "output" | "interface" | "maintenance" @@ -55,9 +59,11 @@ interface Category { const CATEGORIES: Category[] = [ { id: "llm", labelKey: "settings.categories.llm", icon: Bot }, { id: "embedding", labelKey: "settings.categories.embedding", icon: Binary }, + { id: "document", labelKey: "settings.categories.document", icon: Bot }, { id: "multimodal", labelKey: "settings.categories.multimodal", icon: ImageIcon }, { id: "web-search", labelKey: "settings.categories.webSearch", icon: Globe }, { id: "network", labelKey: "settings.categories.network", icon: Network }, + { id: "agent", labelKey: "settings.categories.agent", icon: Bot }, { id: "output", labelKey: "settings.categories.output", icon: Languages }, { id: "interface", labelKey: "settings.categories.interface", icon: Palette }, { id: "maintenance", labelKey: "settings.categories.maintenance", icon: Wrench }, @@ -68,6 +74,7 @@ const CATEGORIES: Category[] = [ function initialDraft( llm: ReturnType["llmConfig"], embed: ReturnType["embeddingConfig"], + documentLlm: ReturnType["documentLlmConfig"], multimodal: ReturnType["multimodalConfig"], outputLanguage: ReturnType["outputLanguage"], proxy: ReturnType["proxyConfig"], @@ -86,11 +93,20 @@ function initialDraft( reasoning: llm.reasoning, embeddingEnabled: embed.enabled, embeddingEndpoint: embed.endpoint, - embeddingApiKey: embed.apiKey, - embeddingModel: embed.model, - embeddingMaxChunkChars: embed.maxChunkChars, - embeddingOverlapChunkChars: embed.overlapChunkChars, - multimodalEnabled: multimodal.enabled, + embeddingApiKey: embed.apiKey, + embeddingModel: embed.model, + embeddingMaxChunkChars: embed.maxChunkChars, + embeddingOverlapChunkChars: embed.overlapChunkChars, + documentUseMainLlm: documentLlm.useMainLlm, + documentProvider: documentLlm.provider, + documentApiKey: documentLlm.apiKey, + documentModel: documentLlm.model, + documentOllamaUrl: documentLlm.ollamaUrl, + documentCustomEndpoint: documentLlm.customEndpoint, + documentMaxContextSize: documentLlm.maxContextSize ?? 204800, + documentApiMode: documentLlm.apiMode, + documentReasoning: documentLlm.reasoning, + multimodalEnabled: multimodal.enabled, multimodalUseMainLlm: multimodal.useMainLlm, multimodalProvider: multimodal.provider, multimodalApiKey: multimodal.apiKey, @@ -115,6 +131,8 @@ export function SettingsView() { const setLlmConfig = useWikiStore((s) => s.setLlmConfig) const embeddingConfig = useWikiStore((s) => s.embeddingConfig) const setEmbeddingConfig = useWikiStore((s) => s.setEmbeddingConfig) + const documentLlmConfig = useWikiStore((s) => s.documentLlmConfig) + const setDocumentLlmConfig = useWikiStore((s) => s.setDocumentLlmConfig) const multimodalConfig = useWikiStore((s) => s.multimodalConfig) const setMultimodalConfig = useWikiStore((s) => s.setMultimodalConfig) const outputLanguage = useWikiStore((s) => s.outputLanguage) @@ -141,6 +159,7 @@ export function SettingsView() { initialDraft( llmConfig, embeddingConfig, + documentLlmConfig, multimodalConfig, outputLanguage, proxyConfig, @@ -179,6 +198,7 @@ export function SettingsView() { initialDraft( llmConfig, embeddingConfig, + documentLlmConfig, multimodalConfig, outputLanguage, proxyConfig, @@ -190,6 +210,7 @@ export function SettingsView() { }, [ llmConfig, embeddingConfig, + documentLlmConfig, multimodalConfig, outputLanguage, proxyConfig, @@ -205,6 +226,7 @@ export function SettingsView() { const { saveLlmConfig, saveEmbeddingConfig, + saveDocumentLlmConfig, saveMultimodalConfig, saveOutputLanguage, saveProxyConfig, @@ -229,6 +251,17 @@ export function SettingsView() { maxChunkChars: draft.embeddingMaxChunkChars, overlapChunkChars: draft.embeddingOverlapChunkChars, } + const newDocumentLlm = { + useMainLlm: draft.documentUseMainLlm, + provider: draft.documentProvider, + apiKey: draft.documentApiKey, + model: draft.documentModel, + ollamaUrl: draft.documentOllamaUrl, + customEndpoint: draft.documentCustomEndpoint, + maxContextSize: draft.documentMaxContextSize, + apiMode: draft.documentProvider === "custom" ? draft.documentApiMode : undefined, + reasoning: draft.documentReasoning, + } const newMultimodal = { enabled: draft.multimodalEnabled, useMainLlm: draft.multimodalUseMainLlm, @@ -257,6 +290,8 @@ export function SettingsView() { await saveLlmConfig(newLlm) setEmbeddingConfig(newEmbed) await saveEmbeddingConfig(newEmbed) + setDocumentLlmConfig(newDocumentLlm) + await saveDocumentLlmConfig(newDocumentLlm) setMultimodalConfig(newMultimodal) await saveMultimodalConfig(newMultimodal) setOutputLanguage(draft.outputLanguage as typeof outputLanguage) @@ -297,6 +332,7 @@ export function SettingsView() { draft, setLlmConfig, setEmbeddingConfig, + setDocumentLlmConfig, setOutputLanguage, setProxyConfig, setMaxHistoryMessages, @@ -313,12 +349,16 @@ export function SettingsView() { return case "embedding": return + case "document": + return case "multimodal": return case "web-search": return case "network": return + case "agent": + return case "output": return case "interface": @@ -391,8 +431,9 @@ export function SettingsView() { {/* Global Save bar hidden for sections that persist inline: - "llm" saves per-row on every edit (independent per-preset state) + - "agent" saves its MCP access toggle inline - "about" has no draft-bound fields */} - {active !== "about" && active !== "llm" && ( + {active !== "about" && active !== "llm" && active !== "agent" && (

diff --git a/src/components/sources/sources-view.tsx b/src/components/sources/sources-view.tsx index f9b87a7c..95d4b039 100644 --- a/src/components/sources/sources-view.tsx +++ b/src/components/sources/sources-view.tsx @@ -28,6 +28,7 @@ export function SourcesView() { const setFileContent = useWikiStore((s) => s.setFileContent) const setFileTree = useWikiStore((s) => s.setFileTree) const llmConfig = useWikiStore((s) => s.llmConfig) + const documentLlmConfig = useWikiStore((s) => s.documentLlmConfig) const [sources, setSources] = useState([]) const [importing, setImporting] = useState(false) const [ingestingPath, setIngestingPath] = useState(null) @@ -115,7 +116,7 @@ export function SourcesView() { setImporting(true) const paths = Array.isArray(selected) ? selected : [selected] try { - await importSourceFiles(project, paths, llmConfig) + await importSourceFiles(project, paths, llmConfig, documentLlmConfig) await loadSources() } finally { setImporting(false) @@ -134,7 +135,7 @@ export function SourcesView() { setImporting(true) try { - await importSourceFolder(project, selected, llmConfig) + await importSourceFolder(project, selected, llmConfig, documentLlmConfig) await loadSources() } catch (err) { console.error(`Failed to import folder:`, err) @@ -226,7 +227,7 @@ export function SourcesView() { // who expected a fresh-import re-run. One button, one path now. setIngestingPath(node.path) try { - await enqueueSourceIngest(project, [node.path], llmConfig) + await enqueueSourceIngest(project, [node.path], llmConfig, documentLlmConfig) } catch (err) { console.error("Failed to enqueue ingest:", err) } finally { @@ -235,7 +236,7 @@ export function SourcesView() { } return ( -

+

{t("sources.title")}

@@ -253,7 +254,7 @@ export function SourcesView() {
- + {sources.length === 0 ? (

{t("sources.noSources")}

diff --git a/src/i18n/en.json b/src/i18n/en.json index 3d13c3cc..49bba182 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -102,12 +102,14 @@ "changeHint": "Changes apply after saving.", "language": "Language", "languageHint": "Interface language (wiki content follows source material language)", - "categories": { - "llm": "LLM Models", - "embedding": "Embeddings", - "multimodal": "Image Captioning", - "webSearch": "Web Search", + "categories": { + "llm": "LLM Models", + "embedding": "Embeddings", + "document": "Document Processing", + "multimodal": "Image Captioning", + "webSearch": "Web Search", "network": "Network", + "agent": "Agent Access", "output": "Output", "interface": "Interface", "maintenance": "Maintenance", @@ -157,6 +159,26 @@ "dropLegacy": "Drop legacy index", "dropLegacyDone": "Legacy index removed." }, + "document": { + "title": "Document Processing", + "description": "Controls the model used for source ingest, wiki file generation, page merging, and Save to Wiki. It does not affect normal chat answers or deep-research synthesis.", + "useMainLabel": "Use main LLM for document processing", + "useMainHint": "Keep using the main chat/research model for ingest and page merges. Turn this off if you want a faster dedicated model for document-heavy workflows.", + "dedicatedHeading": "Dedicated document-processing model", + "provider": "Provider", + "ollamaUrl": "Ollama URL", + "customEndpoint": "Endpoint URL", + "customEndpointHint": "OpenAI-compatible /v1 base. LM Studio, llama.cpp server, vLLM, LocalAI all work.", + "apiKey": "API key", + "apiKeyPlaceholder": "Leave blank for local / no-auth endpoints", + "model": "Model", + "modelPlaceholder": "e.g. gpt-4.1-mini, gemini-2.5-flash, kimi-k2.5", + "modelHint": "Balanced fast model recommended. This setting is used by ingest analysis, wiki generation, and page merge calls.", + "contextWindow": "Context window", + "reasoning": "Reasoning / thinking", + "reasoningBudget": "thinking budget tokens", + "reasoningHint": "Document-processing prompts often work best with low or off reasoning to avoid slow or reasoning-only responses." + }, "multimodal": { "title": "Image Captioning", "description": "Generate factual captions for images extracted from PDFs / DOCX / PPTX during ingest. Captions are inserted as alt text inside the source markdown — they're what semantic search matches when you search for image content. Cached by image hash so duplicate logos / charts only call the LLM once.", @@ -196,11 +218,30 @@ "bypassLocal": "Bypass proxy for local addresses (recommended)", "bypassLocalHelp": "Requests to localhost, 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, and *.local don't go through the proxy. Keep this on if you use Ollama / LM Studio / other local or LAN-deployed LLMs." }, + "agentAccess": { + "title": "Agent Access", + "description": "Allow local agents to use this app through the LLM Wiki MCP server.", + "enable": "Enable MCP access for local agents", + "enabledHint": "Agents can call the local LLM Wiki API after you add the MCP server to the agent client.", + "disabledHint": "Agent API requests will get a clear disabled error instead of waiting for a timeout.", + "clipServer": "Clip server", + "mcpAccess": "MCP access", + "enabled": "Enabled", + "disabled": "Disabled", + "currentProject": "Current project", + "noProject": "Open a project first.", + "disabledNotice": "Turn this on before using the MCP tools from Codex or another local agent.", + "codexCommand": "Codex command", + "jsonConfig": "MCP JSON config", + "restartHint": "After adding the MCP server to an agent client, start a new agent session so the tools are discovered.", + "copy": "Copy", + "copied": "Copied" + }, "output": { "title": "Output Preferences", "description": "Controls for generated content — language, conversation history length.", - "aiLanguage": "AI Output Language", - "aiLanguageHint": "Force AI-generated content (chat replies, wiki pages, research output, lint reports) to use the selected language. Pick \"Auto\" to let the AI follow the user's input or source document language.", + "aiLanguage": "AI Output Mode", + "aiLanguageHint": "Controls how AI-generated content (chat replies, wiki pages, research output, lint reports) should be written. Recommended: \"Chinese-first (preserve English terms)\" for Simplified Chinese prose that keeps English technical terms, paper titles, model names, APIs, commands, code, and paths. \"Auto\" remains strict single-language detection and is not recommended for mixed Chinese-English source material.", "historyLength": "Conversation History Length", "historyHint": "Number of prior messages sent to the AI on each request. More = richer context but more tokens.", "historyCurrent": "Currently {{count}} messages (about {{turns}} turns)" diff --git a/src/i18n/zh.json b/src/i18n/zh.json index fd5b06f4..584cf491 100644 --- a/src/i18n/zh.json +++ b/src/i18n/zh.json @@ -102,12 +102,14 @@ "changeHint": "改动会在保存后应用。", "language": "语言", "languageHint": "界面语言（Wiki 内容语言以原始资料为准）", - "categories": { - "llm": "LLM 模型", - "embedding": "向量嵌入", - "multimodal": "图片描述", - "webSearch": "网页搜索", + "categories": { + "llm": "LLM 模型", + "embedding": "向量嵌入", + "document": "文档处理", + "multimodal": "图片描述", + "webSearch": "网页搜索", "network": "网络", + "agent": "智能体访问", "output": "输出偏好", "interface": "界面", "maintenance": "维护", @@ -157,6 +159,26 @@ "dropLegacy": "删除旧版索引", "dropLegacyDone": "旧版索引已删除。" }, + "document": { + "title": "文档处理", + "description": "控制 source ingest、wiki 文件生成、页面合并、以及 Save to Wiki 使用的模型。不影响普通 chat 回答，也不影响 deep research 的研究结果合成。", + "useMainLabel": "文档处理复用主 LLM", + "useMainHint": "继续用主 chat/research 模型处理 ingest 和页面合并。如果你想给文档工作流单独换一个更快的模型，就把它关掉。", + "dedicatedHeading": "独立文档处理模型", + "provider": "Provider", + "ollamaUrl": "Ollama URL", + "customEndpoint": "端点 URL", + "customEndpointHint": "OpenAI 兼容的 /v1 根地址。LM Studio、llama.cpp server、vLLM、LocalAI 都可以。", + "apiKey": "API Key", + "apiKeyPlaceholder": "本地 / 无鉴权端点留空", + "model": "模型", + "modelPlaceholder": "例如 gpt-4.1-mini、gemini-2.5-flash、kimi-k2.5", + "modelHint": "推荐均衡型快模型。这个设置会用于 ingest 分析、wiki 生成、以及页面合并。", + "contextWindow": "上下文窗口", + "reasoning": "Reasoning / thinking", + "reasoningBudget": "thinking budget tokens", + "reasoningHint": "文档处理 prompt 往往更适合 low 或 off，能避免过慢或只输出 reasoning 的响应。" + }, "multimodal": { "title": "图片描述", "description": "导入时调用视觉模型为 PDF / DOCX / PPTX 中提取的图片生成事实性描述。描述会写入源 markdown 的 alt 属性，语义搜索按图片内容查询时就靠它命中。按图片字节 SHA-256 缓存，跨文档重复出现的 logo / 图表模板只调用一次。", @@ -196,11 +218,30 @@ "bypassLocal": "本地地址不走代理（推荐）", "bypassLocalHelp": "对 localhost、127.0.0.0/8、10.0.0.0/8、172.16.0.0/12、192.168.0.0/16 以及 *.local 的请求会绕过代理。如果你用 Ollama / LM Studio 或局域网内的 LLM 服务，请保持开启。" }, + "agentAccess": { + "title": "智能体访问", + "description": "允许本机智能体通过 LLM Wiki MCP server 使用这个应用。", + "enable": "允许本机智能体使用 MCP", + "enabledHint": "把 MCP server 添加到智能体客户端后，智能体就能调用本地 LLM Wiki API。", + "disabledHint": "智能体 API 请求会收到明确的 disabled 错误，不会一直等到超时。", + "clipServer": "Clip 服务", + "mcpAccess": "MCP 访问", + "enabled": "已开启", + "disabled": "已关闭", + "currentProject": "当前项目", + "noProject": "请先打开一个项目。", + "disabledNotice": "使用 Codex 或其他本机智能体调用 MCP 工具前，请先打开这个开关。", + "codexCommand": "Codex 命令", + "jsonConfig": "MCP JSON 配置", + "restartHint": "把 MCP server 添加到智能体客户端后，开启一个新的智能体会话才能发现这些工具。", + "copy": "复制", + "copied": "已复制" + }, "output": { "title": "输出偏好", "description": "影响生成内容的语言、对话历史长度等参数。", - "aiLanguage": "AI 输出语言", - "aiLanguageHint": "强制 AI 生成内容（chat 回复、wiki 页面、research 结果、lint 报告）使用指定语言。选 \"Auto\" 让 AI 跟随用户输入或源文档的语言。", + "aiLanguage": "AI 输出模式", + "aiLanguageHint": "控制 AI 生成内容（chat 回复、wiki 页面、research 结果、lint 报告）的输出方式。推荐“中文为主（保留必要英文）”：主体用简体中文，但保留英文术语、论文标题、模型名、API、命令、代码和路径。\"Auto\" 仍是严格单语自动检测，更适合明确的单语言资料，不适合中英混合资料。", "historyLength": "对话历史长度", "historyHint": "每次请求发给 AI 的历史消息条数。多 = 上下文更完整但更费 token。", "historyCurrent": "当前 {{count}} 条消息（约 {{turns}} 轮对话）" diff --git a/src/lib/auto-save.ts b/src/lib/auto-save.ts index 6281cf52..717250fa 100644 --- a/src/lib/auto-save.ts +++ b/src/lib/auto-save.ts @@ -2,6 +2,7 @@ import { useReviewStore } from "@/stores/review-store" import { useChatStore } from "@/stores/chat-store" import { useWikiStore } from "@/stores/wiki-store" import { saveReviewItems, saveChatHistory } from "./persist" +import { normalizePath } from "@/lib/path-utils" let reviewTimer: ReturnType | null = null let chatTimer: ReturnType | null = null @@ -13,7 +14,9 @@ export function setupAutoSave(): void { reviewTimer = setTimeout(() => { const project = useWikiStore.getState().project if (project) { - saveReviewItems(project.path, state.items).catch(() => {}) + const pp = normalizePath(project.path) + const scoped = state.items.filter((item) => normalizePath(item.projectPath) === pp) + saveReviewItems(project.path, scoped).catch(() => {}) } }, 1000) }) diff --git a/src/lib/chat-retrieval.ts b/src/lib/chat-retrieval.ts new file mode 100644 index 00000000..436c998a --- /dev/null +++ b/src/lib/chat-retrieval.ts @@ -0,0 +1,226 @@ +import { readFile } from "@/commands/fs" +import type { WikiProject } from "@/types/wiki" +import type { ChatMessage as LLMMessage } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" +import { searchWiki, tokenizeQuery, type SearchResult } from "@/lib/search" +import { buildRetrievalGraph, getRelatedNodes } from "@/lib/graph-relevance" +import { normalizePath, getFileName, getRelativePath } from "@/lib/path-utils" +import { + getOutputLanguage, + buildLanguageDirectiveFromLanguage, + buildLanguageReminderFromLanguage, +} from "@/lib/output-language" +import { isGreeting } from "@/lib/greeting-detector" +import { computeContextBudget } from "@/lib/context-budget" + +export interface RetrievedPage { + title: string + path: string + content: string + priority: number +} + +export interface GraphExpansion { + title: string + path: string + relevance: number +} + +export interface ChatRetrievalResult { + systemMessages: LLMMessage[] + references: Array<{ title: string; path: string }> + relevantPages: RetrievedPage[] + searchResults: SearchResult[] + graphExpansions: GraphExpansion[] + budget: ReturnType + langReminder?: string + outputLanguage: string + greetingOnly: boolean +} + +export interface BuildChatRetrievalInput { + project: Pick + query: string + llmConfig: Pick + dataVersion: number + searchLimit?: number + pageLimit?: number +} + +export async function buildChatRetrievalContext({ + project, + query, + llmConfig, + dataVersion, + searchLimit = 10, + pageLimit, +}: BuildChatRetrievalInput): Promise { + const systemMessages: LLMMessage[] = [] + const budget = computeContextBudget(llmConfig.maxContextSize) + const outputLanguage = getOutputLanguage(query) + const greetingOnly = isGreeting(query) + + if (greetingOnly) { + const greetingDirective = buildLanguageDirectiveFromLanguage(outputLanguage) + systemMessages.push({ + role: "system", + content: [ + `You are a wiki assistant for the project "${project.name}".`, + "The user sent a casual greeting -- reply briefly and naturally, in one or two sentences.", + "Do NOT invent wiki content or pretend to have retrieved pages. Invite the user to ask a concrete question if they want information from the wiki.", + "", + greetingDirective, + ].join("\n"), + }) + + return { + systemMessages, + references: [], + relevantPages: [], + searchResults: [], + graphExpansions: [], + budget, + outputLanguage, + greetingOnly, + } + } + + const pp = normalizePath(project.path) + const [rawIndex, purpose] = await Promise.all([ + readFile(`${pp}/wiki/index.md`).catch(() => ""), + readFile(`${pp}/purpose.md`).catch(() => ""), + ]) + + const searchResults = await searchWiki(pp, query) + const topSearchResults = searchResults.slice(0, Math.max(1, searchLimit)) + + let index = rawIndex + if (rawIndex.length > budget.indexBudget) { + const tokens = tokenizeQuery(query) + const lines = rawIndex.split("\n") + const keptLines: string[] = [] + let keptSize = 0 + + for (const line of lines) { + const isHeader = line.startsWith("##") + const lower = line.toLowerCase() + const isRelevant = tokens.some((t) => lower.includes(t)) + + if (isHeader || isRelevant) { + if (keptSize + line.length + 1 <= budget.indexBudget) { + keptLines.push(line) + keptSize += line.length + 1 + } + } + } + index = keptLines.join("\n") + if (index.length < rawIndex.length) { + index += "\n\n[...index trimmed to relevant entries...]" + } + } + + const graph = await buildRetrievalGraph(pp, dataVersion) + const expandedIds = new Set() + const searchHitPaths = new Set(topSearchResults.map((r) => r.path)) + const graphExpansions: GraphExpansion[] = [] + + for (const result of topSearchResults) { + const fileName = getFileName(result.path) + const nodeId = fileName.replace(/\.md$/, "") + const related = getRelatedNodes(nodeId, graph, 3) + for (const { node, relevance } of related) { + if (relevance < 2.0) continue + if (searchHitPaths.has(node.path)) continue + if (expandedIds.has(node.id)) continue + expandedIds.add(node.id) + graphExpansions.push({ title: node.title, path: node.path, relevance }) + } + } + graphExpansions.sort((a, b) => b.relevance - a.relevance) + + let usedChars = 0 + const relevantPages: RetrievedPage[] = [] + const maxPages = pageLimit && pageLimit > 0 ? pageLimit : Number.POSITIVE_INFINITY + + const tryAddPage = async ( + title: string, + filePath: string, + priority: number, + ): Promise => { + if (usedChars >= budget.pageBudget || relevantPages.length >= maxPages) return false + try { + const raw = await readFile(filePath) + const relativePath = getRelativePath(filePath, pp) + const truncated = raw.length > budget.maxPageSize + ? raw.slice(0, budget.maxPageSize) + "\n\n[...truncated...]" + : raw + if (usedChars + truncated.length > budget.pageBudget) return false + usedChars += truncated.length + relevantPages.push({ title, path: relativePath, content: truncated, priority }) + return true + } catch { + return false + } + } + + for (const r of topSearchResults.filter((r) => r.titleMatch)) { + await tryAddPage(r.title, r.path, 0) + } + for (const r of topSearchResults.filter((r) => !r.titleMatch)) { + await tryAddPage(r.title, r.path, 1) + } + for (const exp of graphExpansions) { + await tryAddPage(exp.title, exp.path, 2) + } + if (relevantPages.length === 0) { + await tryAddPage("Overview", `${pp}/wiki/overview.md`, 3) + } + + const pagesContext = relevantPages.length > 0 + ? relevantPages.map((p, i) => + `### [${i + 1}] ${p.title}\nPath: ${p.path}\n\n${p.content}` + ).join("\n\n---\n\n") + : "(No wiki pages found)" + + const pageList = relevantPages.map((p, i) => + `[${i + 1}] ${p.title} (${p.path})` + ).join("\n") + + systemMessages.push({ + role: "system", + content: [ + "You are a knowledgeable wiki assistant. Answer questions based on the wiki content provided below.", + "", + "## Rules", + "- Answer based ONLY on the numbered wiki pages provided below.", + "- If the provided pages don't contain enough information, say so honestly.", + "- Use [[wikilink]] syntax to reference wiki pages.", + "- When citing information, use the page number in brackets, e.g. [1], [2].", + "- At the VERY END of your response, add a hidden comment listing which page numbers you used:", + " ", + "", + "Use markdown formatting for clarity.", + "", + purpose ? `## Wiki Purpose\n${purpose}` : "", + index ? `## Wiki Index\n${index}` : "", + relevantPages.length > 0 ? `## Page List\n${pageList}` : "", + `## Wiki Pages\n\n${pagesContext}`, + "", + "---", + "", + buildLanguageDirectiveFromLanguage(outputLanguage), + ].filter(Boolean).join("\n"), + }) + + return { + systemMessages, + references: relevantPages.map((p) => ({ title: p.title, path: p.path })), + relevantPages, + searchResults, + graphExpansions, + budget, + langReminder: buildLanguageReminderFromLanguage(outputLanguage), + outputLanguage, + greetingOnly, + } +} diff --git a/src/lib/clip-watcher.ts b/src/lib/clip-watcher.ts index 3e5923e1..e8cb4984 100644 --- a/src/lib/clip-watcher.ts +++ b/src/lib/clip-watcher.ts @@ -1,7 +1,7 @@ import { useWikiStore } from "@/stores/wiki-store" import { enqueueIngest } from "./ingest-queue" import { listDirectory } from "@/commands/fs" -import { hasUsableLlm } from "@/lib/has-usable-llm" +import { hasUsableDocumentLlm } from "@/lib/has-usable-llm" const POLL_INTERVAL = 3000 // Check every 3 seconds let intervalId: ReturnType | null = null @@ -41,7 +41,7 @@ export function startClipWatcher() { // a UI refresh. Same path used by file imports from sources-view. // Pass the project's stable UUID — the queue looks up the // current filesystem path from the registry at run time. - if (hasUsableLlm(store.llmConfig)) { + if (hasUsableDocumentLlm(store.llmConfig, store.documentLlmConfig)) { enqueueIngest(project.id, clipFilePath).catch((err) => { console.error("Failed to enqueue web clip:", err) }) diff --git a/src/lib/deep-research.test.ts b/src/lib/deep-research.test.ts new file mode 100644 index 00000000..d4d303c4 --- /dev/null +++ b/src/lib/deep-research.test.ts @@ -0,0 +1,110 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +const updateTask = vi.fn() +const setPanelOpen = vi.fn() +const getNextQueued = vi.fn(() => undefined) + +vi.mock("./web-search", () => ({ + webSearch: vi.fn(), +})) + +vi.mock("./llm-client", () => ({ + streamChat: vi.fn(), +})) + +vi.mock("./ingest", () => ({ + autoIngest: vi.fn(), +})) + +vi.mock("@/commands/fs", () => ({ + writeFile: vi.fn(), + readFile: vi.fn(), + listDirectory: vi.fn(), +})) + +vi.mock("@/lib/output-language", () => ({ + buildLanguageDirective: vi.fn(() => ""), +})) + +vi.mock("@/lib/document-llm", () => ({ + resolveDocumentLlmConfig: vi.fn((cfg) => cfg), +})) + +vi.mock("@/stores/wiki-store", () => ({ + useWikiStore: { + getState: () => ({ + project: { id: "proj-1", path: "/project-1" }, + setFileTree: vi.fn(), + bumpDataVersion: vi.fn(), + documentLlmConfig: null, + }), + }, +})) + +vi.mock("@/stores/research-store", () => ({ + useResearchStore: { + getState: () => ({ + tasks: [ + { + id: "research-1", + projectId: "proj-1", + projectPath: "/project-1", + topic: "Verkle", + searchQueries: ["verkle trees"], + status: "error", + webResults: [{ title: "old", url: "https://x", source: "x", snippet: "y" }], + synthesis: "partial", + savedPath: "wiki/queries/old.md", + error: "boom", + createdAt: 1, + }, + ], + maxConcurrent: 3, + updateTask, + setPanelOpen, + getRunningCount: () => 0, + getNextQueued, + }), + }, +})) + +import { retryResearchTask } from "./deep-research" + +describe("retryResearchTask", () => { + beforeEach(() => { + vi.useFakeTimers() + updateTask.mockReset() + setPanelOpen.mockReset() + getNextQueued.mockReset() + getNextQueued.mockReturnValue(undefined) + }) + + it("requeues a failed task in place and clears stale fields", () => { + const ok = retryResearchTask( + "research-1", + { provider: "openai", model: "gpt-4.1" } as never, + { provider: "tavily", apiKey: "k" } as never, + ) + + expect(ok).toBe(true) + expect(updateTask).toHaveBeenCalledWith("research-1", { + status: "queued", + error: null, + webResults: [], + synthesis: "", + savedPath: null, + }) + expect(setPanelOpen).toHaveBeenCalledWith(true) + vi.runAllTimers() + }) + + it("returns false for a missing or non-error task", () => { + const ok = retryResearchTask( + "missing", + { provider: "openai", model: "gpt-4.1" } as never, + { provider: "tavily", apiKey: "k" } as never, + ) + expect(ok).toBe(false) + expect(updateTask).not.toHaveBeenCalled() + }) +}) diff --git a/src/lib/deep-research.ts b/src/lib/deep-research.ts index d850ea50..4017ca3d 100644 --- a/src/lib/deep-research.ts +++ b/src/lib/deep-research.ts @@ -6,6 +6,7 @@ import { useWikiStore, type LlmConfig, type SearchApiConfig } from "@/stores/wik import { useResearchStore } from "@/stores/research-store" import { normalizePath } from "@/lib/path-utils" import { buildLanguageDirective } from "@/lib/output-language" +import { resolveDocumentLlmConfig } from "@/lib/document-llm" /** * Queue a deep research task. Automatically starts processing if under concurrency limit. @@ -18,7 +19,12 @@ export function queueResearch( searchQueries?: string[], ): string { const store = useResearchStore.getState() - const taskId = store.addTask(topic) + const project = useWikiStore.getState().project + const taskId = store.addTask( + topic, + project?.id ?? normalizePath(projectPath), + normalizePath(projectPath), + ) // Store search queries on the task if (searchQueries && searchQueries.length > 0) { store.updateTask(taskId, { searchQueries }) @@ -27,16 +33,38 @@ export function queueResearch( store.setPanelOpen(true) // Start processing on next tick to ensure React has rendered the panel setTimeout(() => { - processQueue(projectPath, llmConfig, searchConfig) + processQueue(llmConfig, searchConfig) }, 50) return taskId } +export function retryResearchTask( + taskId: string, + llmConfig: LlmConfig, + searchConfig: SearchApiConfig, +): boolean { + const store = useResearchStore.getState() + const task = store.tasks.find((t) => t.id === taskId) + if (!task || task.status !== "error") return false + + store.updateTask(taskId, { + status: "queued", + error: null, + webResults: [], + synthesis: "", + savedPath: null, + }) + store.setPanelOpen(true) + setTimeout(() => { + processQueue(llmConfig, searchConfig) + }, 50) + return true +} + /** * Process queued tasks up to maxConcurrent limit. */ function processQueue( - projectPath: string, llmConfig: LlmConfig, searchConfig: SearchApiConfig, ) { @@ -47,7 +75,7 @@ function processQueue( for (let i = 0; i < available; i++) { const next = useResearchStore.getState().getNextQueued() if (!next) break - executeResearch(projectPath, next.id, next.topic, llmConfig, searchConfig) + executeResearch(next.projectPath, next.id, next.topic, llmConfig, searchConfig) } } @@ -217,7 +245,11 @@ async function executeResearch( } // Auto-ingest the research result to generate entities, concepts, cross-references - autoIngest(pp, `${pp}/${savedPath}`, llmConfig).catch((err) => { + const ingestLlm = resolveDocumentLlmConfig( + llmConfig, + useWikiStore.getState().documentLlmConfig, + ) + autoIngest(pp, `${pp}/${savedPath}`, ingestLlm).catch((err) => { console.error("Failed to auto-ingest research result:", err) }) } catch (err) { @@ -237,5 +269,6 @@ function onTaskFinished( searchConfig: SearchApiConfig, ) { // Process next queued task - setTimeout(() => processQueue(projectPath, llmConfig, searchConfig), 100) + void projectPath + setTimeout(() => processQueue(llmConfig, searchConfig), 100) } diff --git a/src/lib/document-llm.test.ts b/src/lib/document-llm.test.ts new file mode 100644 index 00000000..b00a43a0 --- /dev/null +++ b/src/lib/document-llm.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest" +import { resolveDocumentLlmConfig } from "./document-llm" +import type { DocumentLlmConfig, LlmConfig } from "@/stores/wiki-store" + +function mainConfig(overrides: Partial = {}): LlmConfig { + return { + provider: "openai", + apiKey: "sk-main", + model: "gpt-4.1", + ollamaUrl: "http://localhost:11434", + customEndpoint: "https://api.example.com/v1", + maxContextSize: 128000, + apiMode: undefined, + reasoning: { mode: "medium" }, + ...overrides, + } +} + +function documentConfig(overrides: Partial = {}): DocumentLlmConfig { + return { + useMainLlm: true, + provider: "custom", + apiKey: "", + model: "gpt-4.1-mini", + ollamaUrl: "http://localhost:11434", + customEndpoint: "http://localhost:1234/v1", + maxContextSize: 64000, + apiMode: "chat_completions", + reasoning: { mode: "off" }, + ...overrides, + } +} + +describe("resolveDocumentLlmConfig", () => { + it("returns the main config verbatim when useMainLlm is true", () => { + const main = mainConfig() + expect(resolveDocumentLlmConfig(main, documentConfig({ useMainLlm: true }))).toEqual(main) + }) + + it("returns the dedicated document-processing config when useMainLlm is false", () => { + const resolved = resolveDocumentLlmConfig( + mainConfig(), + documentConfig({ + useMainLlm: false, + provider: "anthropic", + apiKey: "sk-doc", + model: "claude-3-5-haiku", + customEndpoint: "ignored", + apiMode: "anthropic_messages", + reasoning: { mode: "low" }, + }), + ) + + expect(resolved).toEqual({ + provider: "anthropic", + apiKey: "sk-doc", + model: "claude-3-5-haiku", + ollamaUrl: "http://localhost:11434", + customEndpoint: "ignored", + maxContextSize: 64000, + apiMode: undefined, + reasoning: { mode: "low" }, + }) + }) + + it("keeps custom apiMode only for custom providers", () => { + const resolved = resolveDocumentLlmConfig( + mainConfig(), + documentConfig({ + useMainLlm: false, + provider: "custom", + apiMode: "anthropic_messages", + }), + ) + + expect(resolved.apiMode).toBe("anthropic_messages") + }) +}) diff --git a/src/lib/document-llm.ts b/src/lib/document-llm.ts new file mode 100644 index 00000000..f3a53479 --- /dev/null +++ b/src/lib/document-llm.ts @@ -0,0 +1,18 @@ +import type { DocumentLlmConfig, LlmConfig } from "@/stores/wiki-store" + +export function resolveDocumentLlmConfig( + mainLlm: LlmConfig, + documentLlm: DocumentLlmConfig, +): LlmConfig { + if (documentLlm.useMainLlm) return mainLlm + return { + provider: documentLlm.provider, + apiKey: documentLlm.apiKey, + model: documentLlm.model, + ollamaUrl: documentLlm.ollamaUrl, + customEndpoint: documentLlm.customEndpoint, + maxContextSize: documentLlm.maxContextSize, + apiMode: documentLlm.provider === "custom" ? documentLlm.apiMode : undefined, + reasoning: documentLlm.reasoning, + } +} diff --git a/src/lib/has-usable-llm.test.ts b/src/lib/has-usable-llm.test.ts index 35ce7534..e4300d4b 100644 --- a/src/lib/has-usable-llm.test.ts +++ b/src/lib/has-usable-llm.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest" import { hasUsableLlm, + hasUsableDocumentLlm, PROVIDERS_WITHOUT_KEY, type LlmProvider, } from "./has-usable-llm" @@ -107,3 +108,32 @@ describe("hasUsableLlm", () => { } }) }) + +describe("hasUsableDocumentLlm", () => { + it("reuses the main model readiness when document processing uses the main LLM", () => { + expect( + hasUsableDocumentLlm( + { provider: "openai", apiKey: "sk-main" }, + { useMainLlm: true, provider: "custom", apiKey: "" }, + ), + ).toBe(true) + }) + + it("checks the dedicated document model when useMainLlm is false", () => { + expect( + hasUsableDocumentLlm( + { provider: "openai", apiKey: "sk-main" }, + { useMainLlm: false, provider: "anthropic", apiKey: "" }, + ), + ).toBe(false) + }) + + it("accepts no-key local providers for the dedicated document model", () => { + expect( + hasUsableDocumentLlm( + { provider: "openai", apiKey: "" }, + { useMainLlm: false, provider: "ollama", apiKey: "" }, + ), + ).toBe(true) + }) +}) diff --git a/src/lib/has-usable-llm.ts b/src/lib/has-usable-llm.ts index 36f69be7..564207bf 100644 --- a/src/lib/has-usable-llm.ts +++ b/src/lib/has-usable-llm.ts @@ -1,4 +1,4 @@ -import type { LlmConfig } from "@/stores/wiki-store" +import type { DocumentLlmConfig, LlmConfig } from "@/stores/wiki-store" export type LlmProvider = LlmConfig["provider"] @@ -42,3 +42,11 @@ export function hasUsableLlm( if (PROVIDERS_WITHOUT_KEY.has(cfg.provider)) return true return cfg.apiKey.trim().length > 0 } + +export function hasUsableDocumentLlm( + mainCfg: Pick, + documentCfg: Pick, +): boolean { + if (documentCfg.useMainLlm) return hasUsableLlm(mainCfg) + return hasUsableLlm(documentCfg) +} diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.ts index 78a80dd3..ba15aa5e 100644 --- a/src/lib/ingest-queue.test.ts +++ b/src/lib/ingest-queue.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest" -import { flushMicrotasks } from "@/test-helpers/deferred" +import { createDeferred, flushMicrotasks } from "@/test-helpers/deferred" // Mock autoIngest so tests control success/failure timing. vi.mock("./ingest", () => ({ @@ -65,12 +65,14 @@ import { } from "./ingest-queue" import { autoIngest } from "./ingest" import { readFile, writeFile } from "@/commands/fs" +import { getProjectPathById } from "@/lib/project-identity" import { sweepResolvedReviews } from "./sweep-reviews" import { useWikiStore } from "@/stores/wiki-store" const mockAutoIngest = vi.mocked(autoIngest) const mockReadFile = vi.mocked(readFile) const mockWriteFile = vi.mocked(writeFile) +const mockGetProjectPathById = vi.mocked(getProjectPathById) const mockSweep = vi.mocked(sweepResolvedReviews) /** Simulate the app having opened `TEST_ID` at `TEST_PATH` so the queue @@ -86,6 +88,7 @@ beforeEach(async () => { mockAutoIngest.mockReset() mockReadFile.mockReset() mockWriteFile.mockReset() + mockGetProjectPathById.mockReset() mockSweep.mockReset() mockSweep.mockResolvedValue(0) removePageEmbeddingMock.mockReset() @@ -93,6 +96,7 @@ beforeEach(async () => { // Default: persisted queue file doesn't exist mockReadFile.mockRejectedValue(new Error("ENOENT")) mockWriteFile.mockResolvedValue(undefined as unknown as void) + mockGetProjectPathById.mockImplementation(async (id: string) => idToPath[id] ?? null) // Default: a valid LLM config so processNext doesn't reject. useWikiStore.getState().setLlmConfig({ @@ -148,6 +152,28 @@ describe("ingest-queue — enqueue & basic processing", () => { expect(mockAutoIngest).toHaveBeenCalledTimes(3) expect(getQueue()).toHaveLength(0) }) + + it("claims the worker before async project lookup so parallel enqueues stay serial", async () => { + const pathLookup = createDeferred() + mockGetProjectPathById.mockReturnValueOnce(pathLookup.promise) + mockAutoIngest.mockImplementation(() => new Promise(() => {})) + + await enqueueIngest(TEST_ID, "a.md") + await flushMicrotasks(2) + await enqueueIngest(TEST_ID, "b.md") + await flushMicrotasks(5) + + expect(mockAutoIngest).not.toHaveBeenCalled() + expect(getQueue().filter((t) => t.status === "processing")).toHaveLength(1) + expect(getQueue().filter((t) => t.status === "pending")).toHaveLength(1) + + pathLookup.resolve(TEST_PATH) + await flushMicrotasks(5) + + expect(mockAutoIngest).toHaveBeenCalledTimes(1) + expect(getQueue().filter((t) => t.status === "processing")).toHaveLength(1) + expect(getQueue().filter((t) => t.status === "pending")).toHaveLength(1) + }) }) describe("ingest-queue — retry & failure", () => { diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.ts index d8d1b1b9..62633ccb 100644 --- a/src/lib/ingest-queue.ts +++ b/src/lib/ingest-queue.ts @@ -3,7 +3,7 @@ import { autoIngest } from "./ingest" import { useWikiStore } from "@/stores/wiki-store" import { normalizePath, isAbsolutePath } from "@/lib/path-utils" import { getProjectPathById } from "@/lib/project-identity" -import { hasUsableLlm } from "@/lib/has-usable-llm" +import { hasUsableDocumentLlm } from "@/lib/has-usable-llm" // ── Types ───────────────────────────────────────────────────────────────── @@ -491,6 +491,14 @@ async function processNext(projectId: string): Promise { return } + // Claim the worker before the first await below. Several enqueue/retry + // calls can schedule processNext in the same tick; leaving this until after + // getProjectPathById allowed multiple tasks to enter "processing". + processing = true + next.status = "processing" + await saveQueue(currentProjectPath) + if (currentProjectId !== projectId) return + // Look up the project's current filesystem path from the registry — // it may have moved since the task was enqueued. If the project isn't // in the registry (was deleted or never registered), mark as failed. @@ -508,17 +516,16 @@ async function processNext(projectId: string): Promise { return } - processing = true - next.status = "processing" - await saveQueue(pp) - if (currentProjectId !== projectId) return - - const llmConfig = useWikiStore.getState().llmConfig + const store = useWikiStore.getState() + const llmConfig = store.llmConfig + const documentLlmConfig = store.documentLlmConfig // Check if LLM is configured - if (!hasUsableLlm(llmConfig)) { + if (!hasUsableDocumentLlm(llmConfig, documentLlmConfig)) { next.status = "failed" - next.error = "LLM not configured — set API key in Settings" + next.error = documentLlmConfig.useMainLlm + ? "LLM not configured — set API key in Settings" + : "Document-processing model not configured — check Settings → Document Processing" processing = false await saveQueue(pp) processNext(projectId) diff --git a/src/lib/ingest.prompt.test.ts b/src/lib/ingest.prompt.test.ts index fcf99718..bc8cfe46 100644 --- a/src/lib/ingest.prompt.test.ts +++ b/src/lib/ingest.prompt.test.ts @@ -65,6 +65,12 @@ describe("buildGenerationPrompt language directive", () => { expect(prompt).toContain("my-paper.pdf") }) + it("does not contradict optional review output in the strict instructions", () => { + const prompt = buildGenerationPrompt("", "", "", "my-paper.pdf") + expect(prompt).toContain("FILE blocks followed by optional REVIEW blocks") + expect(prompt).toContain("FILE blocks and any genuinely needed REVIEW blocks") + }) + it("respects user setting regardless of source content language", () => { useWikiStore.getState().setOutputLanguage("English") const prompt = buildGenerationPrompt("", "", "", "x.pdf", undefined, "私は日本語の文章を書きます") diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index bd79354d..5cd6b2f7 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -16,6 +16,9 @@ import { } from "@/lib/extract-source-images" import { captionMarkdownImages, loadCaptionCache } from "@/lib/image-caption-pipeline" import type { MultimodalConfig } from "@/stores/wiki-store" +import { resolveDocumentLlmConfig } from "@/lib/document-llm" +import { loadReviewItems, saveReviewItems } from "@/lib/persist" +import { ensureProjectId } from "@/lib/project-identity" /** * Resolve the LLM config that the caption pipeline should use. @@ -291,8 +294,12 @@ export async function autoIngest( signal?: AbortSignal, folderContext?: string, ): Promise { + const effectiveLlm = resolveDocumentLlmConfig( + llmConfig, + useWikiStore.getState().documentLlmConfig, + ) return withProjectLock(normalizePath(projectPath), () => - autoIngestImpl(projectPath, sourcePath, llmConfig, signal, folderContext), + autoIngestImpl(projectPath, sourcePath, effectiveLlm, signal, folderContext), ) } @@ -695,9 +702,30 @@ async function autoIngestImpl( } // ── Step 4: Parse review items ──────────────────────────────── - const reviewItems = parseReviewBlocks(generation, sp) + const stableProjectId = await ensureProjectId(pp).catch(() => pp) + const reviewItems = parseReviewBlocks( + generation, + stableProjectId, + pp, + sp, + ) if (reviewItems.length > 0) { - useReviewStore.getState().addItems(reviewItems) + const currentProjectPath = normalizePath(useWikiStore.getState().project?.path ?? "") + if (currentProjectPath === pp) { + const reviewStore = useReviewStore.getState() + reviewStore.addItems(reviewItems) + await saveReviewItems(pp, reviewStore.items).catch(() => {}) + } else { + const existing = await loadReviewItems(pp).catch(() => []) + const reviewStore = useReviewStore.getState() + reviewStore.addItems(existing) + reviewStore.addItems(reviewItems) + const merged = reviewStore.items.filter((item) => normalizePath(item.projectPath) === pp) + reviewStore.setItems( + reviewStore.items.filter((item) => normalizePath(item.projectPath) !== pp), + ) + await saveReviewItems(pp, merged).catch(() => {}) + } } // ── Step 5: Save to cache ─────────────────────────────────── @@ -910,6 +938,8 @@ const REVIEW_BLOCK_REGEX = /---REVIEW:\s*(\w[\w-]*)\s*\|\s*(.+?)\s*---\n([\s\S]* function parseReviewBlocks( text: string, + projectId: string, + projectPath: string, sourcePath: string, ): Omit[] { const items: Omit[] = [] @@ -958,6 +988,8 @@ function parseReviewBlocks( .trim() items.push({ + projectId, + projectPath, type, title, description, @@ -1152,7 +1184,7 @@ export function buildGenerationPrompt(schema: string, purpose: string, index: st "", "1. The FIRST character of your response MUST be `-` (the opening of `---FILE:`).", "2. DO NOT output any preamble such as \"Here are the files:\", \"Based on the analysis...\", or any introductory prose.", - "3. DO NOT echo or restate the analysis — that was stage 1's job. Your job is to emit FILE blocks.", + "3. DO NOT echo or restate the analysis — that was stage 1's job. Your job is to emit FILE blocks and any genuinely needed REVIEW blocks.", "4. DO NOT output markdown tables, bullet lists, or headings outside of FILE/REVIEW blocks.", "5. DO NOT output any trailing commentary after the last `---END FILE---` or `---END REVIEW---`.", "6. Between blocks, use only blank lines — no prose.", @@ -1397,6 +1429,10 @@ export async function startIngest( llmConfig: LlmConfig, signal?: AbortSignal, ): Promise { + const effectiveLlm = resolveDocumentLlmConfig( + llmConfig, + useWikiStore.getState().documentLlmConfig, + ) const pp = normalizePath(projectPath) const sp = normalizePath(sourcePath) const store = getStore() @@ -1460,7 +1496,7 @@ export async function startIngest( let accumulated = "" await streamChat( - llmConfig, + effectiveLlm, [ { role: "system", content: systemPrompt }, { role: "user", content: userMessage }, @@ -1487,6 +1523,10 @@ export async function executeIngestWrites( userGuidance?: string, signal?: AbortSignal, ): Promise { + const effectiveLlm = resolveDocumentLlmConfig( + llmConfig, + useWikiStore.getState().documentLlmConfig, + ) const pp = normalizePath(projectPath) const store = getStore() @@ -1546,7 +1586,7 @@ export async function executeIngestWrites( .join("\n\n") await streamChat( - llmConfig, + effectiveLlm, [{ role: "system", content: systemPrompt }, ...conversationHistory], { onToken: (token) => { diff --git a/src/lib/language-metadata.test.ts b/src/lib/language-metadata.test.ts index c54604e7..808a645f 100644 --- a/src/lib/language-metadata.test.ts +++ b/src/lib/language-metadata.test.ts @@ -17,6 +17,14 @@ describe("language metadata", () => { expect(sameScriptFamily("Persian", "Arabic")).toBe(true) }) + it("gives the mixed Chinese mode a stable prompt name and Chinese locale", () => { + expect(getLanguagePromptName("Chinese (preserve English terms)")).toBe( + "Chinese-first with preserved English technical terms", + ) + expect(getHtmlLang("Chinese (preserve English terms)")).toBe("zh-Hans") + expect(getTextDirection("Chinese (preserve English terms)")).toBe("ltr") + }) + it("defaults unknown languages to LTR with the original prompt name", () => { expect(getLanguagePromptName("Vietnamese")).toBe("Vietnamese") expect(getTextDirection("Vietnamese")).toBe("ltr") diff --git a/src/lib/language-metadata.ts b/src/lib/language-metadata.ts index 768b200a..30bf9748 100644 --- a/src/lib/language-metadata.ts +++ b/src/lib/language-metadata.ts @@ -38,6 +38,12 @@ const LANGUAGE_METADATA: Record = { direction: "ltr", scriptFamily: "cjk", }, + "Chinese (preserve English terms)": { + promptName: "Chinese-first with preserved English technical terms", + htmlLang: "zh-Hans", + direction: "ltr", + scriptFamily: "cjk", + }, "Traditional Chinese": { promptName: "Traditional Chinese", htmlLang: "zh-Hant", diff --git a/src/lib/local-api-bridge.test.ts b/src/lib/local-api-bridge.test.ts new file mode 100644 index 00000000..77e40e0e --- /dev/null +++ b/src/lib/local-api-bridge.test.ts @@ -0,0 +1,107 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import { useWikiStore } from "@/stores/wiki-store" +import { handleBridgeRequest, MCP_ACCESS_DISABLED_ERROR } from "./local-api-bridge" +import { searchWiki } from "@/lib/search" + +vi.mock("@/commands/fs", () => ({ + copyFile: vi.fn(), + createDirectory: vi.fn(), + fileExists: vi.fn(), + listDirectory: vi.fn().mockResolvedValue([]), + preprocessFile: vi.fn(), + writeFile: vi.fn(), +})) + +vi.mock("@/lib/chat-retrieval", () => ({ + buildChatRetrievalContext: vi.fn(), +})) + +vi.mock("@/lib/search", () => ({ + searchWiki: vi.fn(), +})) + +vi.mock("@/lib/wiki-graph", () => ({ + buildWikiGraph: vi.fn(), +})) + +vi.mock("@/lib/llm-client", () => ({ + streamChat: vi.fn(), +})) + +vi.mock("@/lib/ingest-queue", () => ({ + enqueueIngest: vi.fn(), +})) + +vi.mock("@/lib/has-usable-llm", () => ({ + hasUsableLlm: vi.fn(() => true), +})) + +describe("local-api-bridge MCP access gate", () => { + beforeEach(() => { + vi.resetAllMocks() + useWikiStore.setState({ + project: { id: "proj-1", name: "Wiki", path: "D:/wiki" }, + mcpAccessEnabled: false, + }) + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: true, + text: vi.fn().mockResolvedValue(JSON.stringify({ ok: true })), + json: vi.fn().mockResolvedValue({ ok: true }), + }), + ) + }) + + it("responds immediately with an error when MCP access is disabled", async () => { + await handleBridgeRequest({ + id: "req-1", + endpoint: "search", + payload: { query: "rope" }, + }) + + expect(searchWiki).not.toHaveBeenCalled() + expect(globalThis.fetch).toHaveBeenCalledWith( + "http://127.0.0.1:19827/api/v1/bridge/respond", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + id: "req-1", + ok: false, + error: MCP_ACCESS_DISABLED_ERROR, + }), + }), + ) + }) + + it("handles bridge requests normally when MCP access is enabled", async () => { + useWikiStore.setState({ mcpAccessEnabled: true }) + vi.mocked(searchWiki).mockResolvedValue([ + { + title: "Rope", + path: "D:/wiki/wiki/concepts/rope.md", + snippet: "Rope scales long context.", + titleMatch: true, + score: 1, + images: [], + }, + ]) + + await handleBridgeRequest({ + id: "req-2", + endpoint: "search", + payload: { query: "rope", projectPath: "D:/wiki", limit: 1 }, + }) + + expect(searchWiki).toHaveBeenCalledWith("D:/wiki", "rope") + const [, options] = vi.mocked(globalThis.fetch).mock.calls[0] + expect(JSON.parse(String(options?.body))).toMatchObject({ + id: "req-2", + ok: true, + result: { + projectPath: "D:/wiki", + query: "rope", + }, + }) + }) +}) diff --git a/src/lib/local-api-bridge.ts b/src/lib/local-api-bridge.ts new file mode 100644 index 00000000..17322f2d --- /dev/null +++ b/src/lib/local-api-bridge.ts @@ -0,0 +1,394 @@ +import { copyFile, createDirectory, fileExists, listDirectory, preprocessFile, writeFile } from "@/commands/fs" +import { buildChatRetrievalContext } from "@/lib/chat-retrieval" +import { searchWiki } from "@/lib/search" +import { buildWikiGraph } from "@/lib/wiki-graph" +import { streamChat, type ChatMessage as LLMMessage } from "@/lib/llm-client" +import { enqueueIngest } from "@/lib/ingest-queue" +import { hasUsableDocumentLlm, hasUsableLlm } from "@/lib/has-usable-llm" +import { normalizePath, getFileName, getRelativePath } from "@/lib/path-utils" +import { useWikiStore } from "@/stores/wiki-store" + +const API_BASE = "http://127.0.0.1:19827/api/v1" +const POLL_INTERVAL = 500 +export const MCP_ACCESS_DISABLED_ERROR = "MCP access is disabled in Settings." + +export interface BridgeRequest { + id: string + endpoint: string + payload: Record +} + +let intervalId: ReturnType | null = null +let polling = false + +export function startLocalApiBridge() { + if (intervalId) return + intervalId = setInterval(() => { + void pollBridge() + }, POLL_INTERVAL) + void pollBridge() +} + +export function stopLocalApiBridge() { + if (!intervalId) return + clearInterval(intervalId) + intervalId = null +} + +async function pollBridge(): Promise { + if (polling) return + polling = true + try { + const res = await fetch(`${API_BASE}/bridge/pending`, { method: "GET" }) + const data = await res.json() as { ok?: boolean; requests?: BridgeRequest[] } + if (!data.ok || !Array.isArray(data.requests) || data.requests.length === 0) return + + for (const request of data.requests) { + await handleBridgeRequest(request) + } + } catch { + // The local server may not be up yet or may be an older release. + } finally { + polling = false + } +} + +export async function handleBridgeRequest(request: BridgeRequest): Promise { + try { + if (!useWikiStore.getState().mcpAccessEnabled) { + throw new Error(MCP_ACCESS_DISABLED_ERROR) + } + const result = await handleRequest(request.endpoint, request.payload ?? {}) + await postBridgeResponse(request.id, true, result) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + await postBridgeResponse(request.id, false, message) + } +} + +async function postBridgeResponse(id: string, ok: boolean, payload: unknown): Promise { + await fetch(`${API_BASE}/bridge/respond`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(ok ? { id, ok: true, result: payload } : { id, ok: false, error: payload }), + }).catch(() => {}) +} + +async function handleRequest(endpoint: string, payload: Record): Promise { + switch (endpoint) { + case "search": + return handleSearch(payload) + case "retrieve": + return handleRetrieve(payload) + case "chat": + return handleChat(payload) + case "graph": + return handleGraph(payload) + case "ingest/file": + return handleIngestFile(payload) + case "ingest/clip": + return handleIngestClip(payload) + default: + throw new Error(`Unsupported local API endpoint: ${endpoint}`) + } +} + +function numberOrDefault(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? Math.floor(value) + : fallback +} + +function stringField(payload: Record, key: string): string { + const value = payload[key] + return typeof value === "string" ? value : "" +} + +function getProject(payload: Record) { + const explicit = stringField(payload, "projectPath") + const active = useWikiStore.getState().project + const path = normalizePath(explicit || active?.path || "") + if (!path) throw new Error("No projectPath provided and no active LLM Wiki project is open.") + + return { + id: active?.path && normalizePath(active.path) === path ? active.id : "", + name: active?.path && normalizePath(active.path) === path + ? active.name + : getFileName(path) || "LLM Wiki Project", + path, + isActive: !!active?.path && normalizePath(active.path) === path, + } +} + +async function handleSearch(payload: Record) { + const query = stringField(payload, "query").trim() + if (!query) throw new Error("query is required") + const limit = numberOrDefault(payload.limit, 20) + const project = getProject(payload) + const results = await searchWiki(project.path, query) + return { + projectPath: project.path, + query, + results: results.slice(0, limit), + } +} + +async function handleRetrieve(payload: Record) { + const query = stringField(payload, "query").trim() + if (!query) throw new Error("query is required") + const limit = numberOrDefault(payload.limit, 10) + const includeContent = payload.includeContent !== false + const project = getProject(payload) + const store = useWikiStore.getState() + const retrieval = await buildChatRetrievalContext({ + project, + query, + llmConfig: store.llmConfig, + dataVersion: store.dataVersion, + searchLimit: limit, + pageLimit: limit, + }) + + return { + projectPath: project.path, + query, + references: retrieval.references, + pages: retrieval.relevantPages.map((page) => includeContent ? page : { ...page, content: undefined }), + searchResults: retrieval.searchResults.slice(0, limit), + graphExpansions: retrieval.graphExpansions.slice(0, limit), + budget: retrieval.budget, + outputLanguage: retrieval.outputLanguage, + greetingOnly: retrieval.greetingOnly, + } +} + +async function handleChat(payload: Record) { + const query = stringField(payload, "query").trim() + if (!query) throw new Error("query is required") + const project = getProject(payload) + const store = useWikiStore.getState() + if (!hasUsableLlm(store.llmConfig)) { + throw new Error("LLM not configured -- set API key and model in Settings.") + } + + const maxHistoryMessages = numberOrDefault(payload.maxHistoryMessages, 10) + const retrieval = await buildChatRetrievalContext({ + project, + query, + llmConfig: store.llmConfig, + dataVersion: store.dataVersion, + }) + + const history = normalizeHistory(payload.messages).slice(-maxHistoryMessages) + const finalUserContent = retrieval.langReminder + ? `[${retrieval.langReminder}]\n\n${query}` + : query + const llmMessages: LLMMessage[] = [ + ...retrieval.systemMessages, + ...history, + { role: "user", content: finalUserContent }, + ] + + const answer = await streamToString(store.llmConfig, llmMessages) + return { + projectPath: project.path, + query, + answer, + references: retrieval.references, + outputLanguage: retrieval.outputLanguage, + } +} + +function normalizeHistory(value: unknown): LLMMessage[] { + if (!Array.isArray(value)) return [] + const out: LLMMessage[] = [] + for (const item of value) { + if (!item || typeof item !== "object") continue + const role = (item as { role?: unknown }).role + const content = (item as { content?: unknown }).content + if ((role === "user" || role === "assistant") && typeof content === "string") { + out.push({ role, content }) + } + } + return out +} + +function streamToString(llmConfig: Parameters[0], messages: LLMMessage[]): Promise { + return new Promise((resolve, reject) => { + let accumulated = "" + let thinkingOpen = false + + const appendReasoning = (token: string) => { + if (!token) return + if (!thinkingOpen) { + thinkingOpen = true + accumulated += "" + } + accumulated += token + } + + const closeReasoning = () => { + if (!thinkingOpen) return + thinkingOpen = false + accumulated += "" + } + + streamChat( + llmConfig, + messages, + { + onToken: (token) => { + closeReasoning() + accumulated += token + }, + onReasoningToken: appendReasoning, + onDone: () => { + closeReasoning() + resolve(accumulated) + }, + onError: reject, + }, + ).catch(reject) + }) +} + +async function handleGraph(payload: Record) { + const project = getProject(payload) + const graph = await buildWikiGraph(project.path) + return { + projectPath: project.path, + ...graph, + } +} + +async function handleIngestFile(payload: Record) { + const sourcePath = normalizePath(stringField(payload, "sourcePath") || stringField(payload, "path")) + if (!sourcePath) throw new Error("sourcePath is required") + const project = getActiveProjectForIngest(payload) + const store = useWikiStore.getState() + if (!hasUsableDocumentLlm(store.llmConfig, store.documentLlmConfig)) { + throw new Error( + store.documentLlmConfig.useMainLlm + ? "LLM not configured -- set API key and model in Settings." + : "Document-processing model not configured -- check Settings -> Document Processing.", + ) + } + + const pp = normalizePath(project.path) + const fileName = getFileName(sourcePath) || "source" + const destPath = await getUniqueDestPath(`${pp}/raw/sources`, fileName) + await createDirectory(`${pp}/raw/sources`).catch(() => {}) + await copyFile(sourcePath, destPath) + preprocessFile(destPath).catch(() => {}) + const taskId = await enqueueIngest(project.id, destPath) + await refreshFileTree(pp) + return { + projectPath: pp, + sourcePath, + path: getRelativePath(destPath, pp), + absolutePath: destPath, + taskId, + } +} + +async function handleIngestClip(payload: Record) { + const title = stringField(payload, "title").trim() || "Untitled" + const content = stringField(payload, "content") + const url = stringField(payload, "url") + if (!content.trim()) throw new Error("content is required") + const project = getActiveProjectForIngest(payload) + const store = useWikiStore.getState() + if (!hasUsableDocumentLlm(store.llmConfig, store.documentLlmConfig)) { + throw new Error( + store.documentLlmConfig.useMainLlm + ? "LLM not configured -- set API key and model in Settings." + : "Document-processing model not configured -- check Settings -> Document Processing.", + ) + } + + const pp = normalizePath(project.path) + const date = new Date().toISOString().slice(0, 10) + const dateCompact = date.replace(/-/g, "") + const slug = slugify(title).slice(0, 50) || "untitled" + const destPath = await getUniqueDestPath(`${pp}/raw/sources`, `${slug}-${dateCompact}.md`) + await createDirectory(`${pp}/raw/sources`).catch(() => {}) + await writeFile(destPath, buildClipMarkdown(title, url, date, content)) + const taskId = await enqueueIngest(project.id, destPath) + await refreshFileTree(pp) + return { + projectPath: pp, + path: getRelativePath(destPath, pp), + absolutePath: destPath, + taskId, + } +} + +function getActiveProjectForIngest(payload: Record) { + const project = getProject(payload) + if (!project.isActive || !project.id) { + throw new Error("Ingest API only supports the active desktop project. Open the project in LLM Wiki first.") + } + return project +} + +async function getUniqueDestPath(dir: string, fileName: string): Promise { + const basePath = `${dir}/${fileName}` + if (!(await fileExists(basePath))) return basePath + + const ext = fileName.includes(".") ? fileName.slice(fileName.lastIndexOf(".")) : "" + const nameWithoutExt = ext ? fileName.slice(0, -ext.length) : fileName + const date = new Date().toISOString().slice(0, 10).replace(/-/g, "") + const withDate = `${dir}/${nameWithoutExt}-${date}${ext}` + if (!(await fileExists(withDate))) return withDate + + for (let i = 2; i <= 99; i++) { + const withCounter = `${dir}/${nameWithoutExt}-${date}-${i}${ext}` + if (!(await fileExists(withCounter))) return withCounter + } + return `${dir}/${nameWithoutExt}-${Date.now()}${ext}` +} + +function slugify(title: string): string { + return title + .split("") + .map((ch) => /[\p{L}\p{N} -]/u.test(ch) ? ch : " ") + .join("") + .trim() + .split(/\s+/) + .join("-") + .toLowerCase() +} + +function yamlQuote(value: string): string { + return value.replace(/"/g, '\\"') +} + +function buildClipMarkdown(title: string, url: string, date: string, content: string): string { + return [ + "---", + "type: clip", + `title: "${yamlQuote(title)}"`, + `url: "${yamlQuote(url)}"`, + `clipped: ${date}`, + "origin: local-api", + "sources: []", + "tags: [web-clip]", + "---", + "", + `# ${title}`, + "", + `Source: ${url}`, + "", + content, + "", + ].join("\n") +} + +async function refreshFileTree(projectPath: string): Promise { + try { + const tree = await listDirectory(projectPath) + useWikiStore.getState().setFileTree(tree) + } catch { + // non-critical + } +} diff --git a/src/lib/output-language-options.ts b/src/lib/output-language-options.ts index f28c15bd..221d1e46 100644 --- a/src/lib/output-language-options.ts +++ b/src/lib/output-language-options.ts @@ -15,7 +15,8 @@ * the language. */ export const OUTPUT_LANGUAGE_OPTIONS = [ - { value: "auto", label: "Auto (detect from input/source)" }, + { value: "Chinese (preserve English terms)", label: "中文为主（保留必要英文）" }, + { value: "auto", label: "Auto (strict single-language detect from input/source)" }, { value: "English", label: "English" }, { value: "Chinese", label: "简体中文 (Simplified Chinese)" }, { value: "Traditional Chinese", label: "繁體中文 (Traditional Chinese)" }, diff --git a/src/lib/output-language.test.ts b/src/lib/output-language.test.ts index 51081d8b..6ffccdbf 100644 --- a/src/lib/output-language.test.ts +++ b/src/lib/output-language.test.ts @@ -1,5 +1,10 @@ import { describe, it, expect, beforeEach } from "vitest" -import { getOutputLanguage, buildLanguageDirective, buildLanguageReminder } from "./output-language" +import { + CHINESE_PRESERVE_ENGLISH_MODE, + getOutputLanguage, + buildLanguageDirective, + buildLanguageReminder, +} from "./output-language" import { useWikiStore } from "@/stores/wiki-store" // Reset the outputLanguage back to "auto" before each test so tests can't @@ -19,6 +24,11 @@ describe("getOutputLanguage", () => { expect(getOutputLanguage("This is clearly English text")).toBe("Japanese") }) + it("uses the explicit Chinese-first mixed mode verbatim", () => { + useWikiStore.getState().setOutputLanguage(CHINESE_PRESERVE_ENGLISH_MODE) + expect(getOutputLanguage("This is clearly English text")).toBe(CHINESE_PRESERVE_ENGLISH_MODE) + }) + it("auto mode falls back to detectLanguage on the fallback text", () => { useWikiStore.getState().setOutputLanguage("auto") expect(getOutputLanguage("注意力机制是什么")).toBe("Chinese") @@ -74,6 +84,15 @@ describe("buildLanguageDirective", () => { expect(directive).toContain("MANDATORY OUTPUT LANGUAGE: Persian (Farsi / فارسی)") }) + it("builds a Chinese-first mixed directive that preserves English terms", () => { + useWikiStore.getState().setOutputLanguage(CHINESE_PRESERVE_ENGLISH_MODE) + const directive = buildLanguageDirective("Transformer attention") + expect(directive).toContain("MANDATORY OUTPUT MODE: Chinese-first with preserved English technical terms") + expect(directive).toContain("Write the main narrative in **Simplified Chinese**.") + expect(directive).toContain("Preserve necessary English terms instead of translating them away") + expect(directive).toContain("Do NOT drift into a third language such as Korean, French, Arabic, or others") + }) + it("explicitly overrides source content language", () => { useWikiStore.getState().setOutputLanguage("English") const directive = buildLanguageDirective() @@ -100,6 +119,14 @@ describe("buildLanguageReminder", () => { expect(buildLanguageReminder("これは日本語です")).toContain("Japanese") }) + it("uses a Chinese-first mixed reminder without third-language drift", () => { + useWikiStore.getState().setOutputLanguage(CHINESE_PRESERVE_ENGLISH_MODE) + const reminder = buildLanguageReminder("ignored fallback") + expect(reminder).toContain("Use Simplified Chinese as the main language") + expect(reminder).toContain("preserve necessary English technical terms") + expect(reminder).toContain("do not drift into a third language") + }) + it("reminds Persian as Persian/Farsi", () => { useWikiStore.getState().setOutputLanguage("Persian") expect(buildLanguageReminder()).toContain("Persian (Farsi / فارسی)") diff --git a/src/lib/output-language.ts b/src/lib/output-language.ts index b2ee6c3d..a14f1df2 100644 --- a/src/lib/output-language.ts +++ b/src/lib/output-language.ts @@ -2,6 +2,8 @@ import { useWikiStore } from "@/stores/wiki-store" import { detectLanguage } from "./detect-language" import { getLanguagePromptName } from "./language-metadata" +export const CHINESE_PRESERVE_ENGLISH_MODE = "Chinese (preserve English terms)" as const + /** * Get the effective output language for LLM content generation. * @@ -16,12 +18,21 @@ export function getOutputLanguage(fallbackText: string = ""): string { return detectLanguage(fallbackText || "English") } -/** - * Build a strong language directive to inject into system prompts. - */ -export function buildLanguageDirective(fallbackText: string = ""): string { - const lang = getOutputLanguage(fallbackText) - const promptLang = getLanguagePromptName(lang) +export function buildLanguageDirectiveFromLanguage(language: string): string { + if (language === CHINESE_PRESERVE_ENGLISH_MODE) { + return [ + "## ⚠️ MANDATORY OUTPUT MODE: Chinese-first with preserved English technical terms", + "", + "Write the main narrative in **Simplified Chinese**.", + "Preserve necessary English terms instead of translating them away: technical terms, paper titles, model names, tool/API names, commands, code, paths, abbreviations, and cited source titles.", + "For concepts commonly used in both Chinese and English, prefer the first mention as `中文（English）` when it reads naturally.", + "Do NOT drift into a third language such as Korean, French, Arabic, or others unless the user explicitly asks for that language.", + "Do not turn the whole response into English. Keep the overall response Chinese-first while preserving necessary English.", + "This output mode overrides conflicting style preferences from source material.", + ].join("\n") + } + + const promptLang = getLanguagePromptName(language) return [ `## ⚠️ MANDATORY OUTPUT LANGUAGE: ${promptLang}`, "", @@ -33,10 +44,23 @@ export function buildLanguageDirective(fallbackText: string = ""): string { ].join("\n") } +/** + * Build a strong language directive to inject into system prompts. + */ +export function buildLanguageDirective(fallbackText: string = ""): string { + return buildLanguageDirectiveFromLanguage(getOutputLanguage(fallbackText)) +} + /** * Short reminder version — for placing right before user's current message. */ +export function buildLanguageReminderFromLanguage(language: string): string { + if (language === CHINESE_PRESERVE_ENGLISH_MODE) { + return "REMINDER: Use Simplified Chinese as the main language, preserve necessary English technical terms/titles/code/commands, and do not drift into a third language." + } + return `REMINDER: All output must be in ${getLanguagePromptName(language)}. Do not use any other language.` +} + export function buildLanguageReminder(fallbackText: string = ""): string { - const lang = getOutputLanguage(fallbackText) - return `REMINDER: All output must be in ${getLanguagePromptName(lang)}. Do not use any other language.` + return buildLanguageReminderFromLanguage(getOutputLanguage(fallbackText)) } diff --git a/src/lib/persist.integration.test.ts b/src/lib/persist.integration.test.ts index f8db30fb..0752b797 100644 --- a/src/lib/persist.integration.test.ts +++ b/src/lib/persist.integration.test.ts @@ -26,6 +26,8 @@ let tmp: { path: string; cleanup: () => Promise } function makeReview(overrides: Partial = {}): ReviewItem { return { id: "r-1", + projectId: "proj-1", + projectPath: tmp?.path ?? "/persist-project", type: "missing-page", title: "Attention", description: "", @@ -101,6 +103,29 @@ describe("review persistence — round-trip", () => { const loaded = await loadReviewItems(windowsy) expect(loaded).toHaveLength(1) }) + + it("deduplicates same-key review items before writing review.json", async () => { + await saveReviewItems(tmp.path, [ + makeReview({ id: "older", projectId: "", createdAt: 1 }), + makeReview({ id: "newer", projectId: "proj-1", createdAt: 2 }), + ]) + + const raw = await readFileRaw(`${tmp.path}/.llm-wiki/review.json`) + const parsed = JSON.parse(raw) as ReviewItem[] + expect(parsed).toHaveLength(1) + expect(parsed[0].id).toBe("newer") + }) + + it("does not auto-fill a missing projectId during save/load", async () => { + await saveReviewItems(tmp.path, [ + makeReview({ id: "orphanish", projectId: "", projectPath: tmp.path }), + ]) + + const loaded = await loadReviewItems(tmp.path) + expect(loaded).toHaveLength(1) + expect(loaded[0].projectId).toBe("") + expect(loaded[0].projectPath).toBe(tmp.path) + }) }) describe("chat persistence — round-trip (new format)", () => { diff --git a/src/lib/persist.ts b/src/lib/persist.ts index 443900de..d340a41b 100644 --- a/src/lib/persist.ts +++ b/src/lib/persist.ts @@ -2,6 +2,7 @@ import { writeFile, readFile, createDirectory } from "@/commands/fs" import type { ReviewItem } from "@/stores/review-store" import type { DisplayMessage, Conversation } from "@/stores/chat-store" import { normalizePath } from "@/lib/path-utils" +import { canonicalizeReviewItems } from "@/lib/review-utils" async function ensureDir(projectPath: string): Promise { await createDirectory(`${projectPath}/.llm-wiki`).catch(() => {}) @@ -11,14 +12,32 @@ async function ensureDir(projectPath: string): Promise { export async function saveReviewItems(projectPath: string, items: ReviewItem[]): Promise { const pp = normalizePath(projectPath) await ensureDir(pp) - await writeFile(`${pp}/.llm-wiki/review.json`, JSON.stringify(items, null, 2)) + const scoped = canonicalizeReviewItems( + items.filter((item) => normalizePath(item.projectPath) === pp), + ) + await writeFile(`${pp}/.llm-wiki/review.json`, JSON.stringify(scoped, null, 2)) } export async function loadReviewItems(projectPath: string): Promise { const pp = normalizePath(projectPath) try { const content = await readFile(`${pp}/.llm-wiki/review.json`) - return JSON.parse(content) as ReviewItem[] + const parsed = JSON.parse(content) as Array> + return parsed.map((item) => ({ + id: item.id ?? `review-legacy-${Math.random().toString(36).slice(2, 10)}`, + projectId: item.projectId ?? pp, + projectPath: normalizePath(item.projectPath ?? pp), + type: item.type ?? "confirm", + title: item.title ?? "", + description: item.description ?? "", + sourcePath: item.sourcePath, + affectedPages: item.affectedPages, + searchQueries: item.searchQueries, + options: item.options ?? [], + resolved: item.resolved ?? false, + resolvedAction: item.resolvedAction, + createdAt: item.createdAt ?? 0, + })) } catch { return [] } diff --git a/src/lib/project-file-sync.ts b/src/lib/project-file-sync.ts index d7848214..bc8c5ff5 100644 --- a/src/lib/project-file-sync.ts +++ b/src/lib/project-file-sync.ts @@ -149,7 +149,8 @@ async function enqueueRawSourceChanges(project: WikiProject, tasks: FileChangeTa if (paths.length === 0) return try { - await enqueueSourceIngest(project, paths, useWikiStore.getState().llmConfig) + const store = useWikiStore.getState() + await enqueueSourceIngest(project, paths, store.llmConfig, store.documentLlmConfig) } catch (err) { console.error("[file-sync] failed to enqueue raw source ingest:", err) } diff --git a/src/lib/project-store.ts b/src/lib/project-store.ts index 31a7c2f6..9c352799 100644 --- a/src/lib/project-store.ts +++ b/src/lib/project-store.ts @@ -1,6 +1,6 @@ import { load } from "@tauri-apps/plugin-store" import type { WikiProject } from "@/types/wiki" -import type { LlmConfig, SearchApiConfig, EmbeddingConfig, MultimodalConfig, OutputLanguage, ProviderConfigs, ProxyConfig } from "@/stores/wiki-store" +import type { LlmConfig, SearchApiConfig, EmbeddingConfig, DocumentLlmConfig, MultimodalConfig, OutputLanguage, ProviderConfigs, ProxyConfig } from "@/stores/wiki-store" const STORE_NAME = "app-state.json" const RECENT_PROJECTS_KEY = "recentProjects" @@ -96,6 +96,18 @@ export async function loadEmbeddingConfig(): Promise { return (await store.get(EMBEDDING_KEY)) ?? null } +const DOCUMENT_LLM_KEY = "documentLlmConfig" + +export async function saveDocumentLlmConfig(config: DocumentLlmConfig): Promise { + const store = await getStore() + await store.set(DOCUMENT_LLM_KEY, config) +} + +export async function loadDocumentLlmConfig(): Promise { + const store = await getStore() + return (await store.get(DOCUMENT_LLM_KEY)) ?? null +} + const MULTIMODAL_KEY = "multimodalConfig" export async function saveMultimodalConfig(config: MultimodalConfig): Promise { @@ -133,6 +145,18 @@ export async function loadProxyConfig(): Promise { return (await store.get(PROXY_CONFIG_KEY)) ?? null } +const MCP_ACCESS_ENABLED_KEY = "mcpAccessEnabled" + +export async function saveMcpAccessEnabled(enabled: boolean): Promise { + const store = await getStore() + await store.set(MCP_ACCESS_ENABLED_KEY, enabled) +} + +export async function loadMcpAccessEnabled(): Promise { + const store = await getStore() + return (await store.get(MCP_ACCESS_ENABLED_KEY)) ?? false +} + export async function removeFromRecentProjects( path: string ): Promise { diff --git a/src/lib/reset-project-state.test.ts b/src/lib/reset-project-state.test.ts index f02b52fb..00446266 100644 --- a/src/lib/reset-project-state.test.ts +++ b/src/lib/reset-project-state.test.ts @@ -63,6 +63,8 @@ describe("resetProjectState — Zustand stores", () => { items: [ { id: "r1", + projectId: "proj-1", + projectPath: "/project-1", type: "missing-page", title: "x", description: "", diff --git a/src/lib/review-utils.test.ts b/src/lib/review-utils.test.ts index 7537344a..f5f8bcb1 100644 --- a/src/lib/review-utils.test.ts +++ b/src/lib/review-utils.test.ts @@ -1,5 +1,26 @@ import { describe, it, expect } from "vitest" -import { normalizeReviewTitle } from "./review-utils" +import type { ReviewItem } from "@/stores/review-store" +import { + normalizeReviewTitle, + canonicalizeReviewItems, + bucketReviewItems, + needsProjectAssignment, +} from "./review-utils" + +function makeReview(overrides: Partial = {}): ReviewItem { + return { + id: "r-1", + projectId: "proj-1", + projectPath: "/project-1", + type: "missing-page", + title: "Attention", + description: "", + options: [], + resolved: false, + createdAt: 1, + ...overrides, + } +} describe("normalizeReviewTitle", () => { it("returns the title lowercased when no prefix", () => { @@ -72,3 +93,82 @@ describe("normalizeReviewTitle", () => { expect(normalizeReviewTitle("Missing page: 缺失页面: Foo")).toBe("缺失页面: foo") }) }) + +describe("canonicalizeReviewItems", () => { + it("keeps the more complete item when the visible key matches", () => { + const items = canonicalizeReviewItems([ + makeReview({ id: "a", projectId: "", createdAt: 1 }), + makeReview({ id: "b", projectId: "proj-1", createdAt: 2 }), + ]) + expect(items.map((item) => item.id)).toEqual(["b"]) + }) + + it("prefers pending over resolved for the same visible key", () => { + const items = canonicalizeReviewItems([ + makeReview({ id: "resolved", resolved: true, createdAt: 10 }), + makeReview({ id: "pending", resolved: false, createdAt: 1 }), + ]) + expect(items.map((item) => item.id)).toEqual(["pending"]) + }) + + it("keeps the newer item when assignment and resolution are tied", () => { + const items = canonicalizeReviewItems([ + makeReview({ id: "older", createdAt: 1 }), + makeReview({ id: "newer", createdAt: 5 }), + ]) + expect(items.map((item) => item.id)).toEqual(["newer"]) + }) +}) + +describe("bucketReviewItems", () => { + it("shows a same-path, missing-projectId item only in Current", () => { + const item = makeReview({ id: "r", projectId: "", projectPath: "/project-1" }) + const buckets = bucketReviewItems([item], "/project-1") + expect(buckets.currentPending.map((it) => it.id)).toEqual(["r"]) + expect(buckets.unassigned).toEqual([]) + }) + + it("does not show the same logical item in Current and Unassigned", () => { + const current = makeReview({ id: "current", projectId: "proj-1", projectPath: "/project-1" }) + const orphanTwin = makeReview({ id: "orphan", projectId: "", projectPath: "/project-1", createdAt: 2 }) + const buckets = bucketReviewItems([current, orphanTwin], "/project-1") + expect(buckets.currentPending).toHaveLength(1) + expect(buckets.currentPending[0].id).toBe("current") + expect(buckets.unassigned).toEqual([]) + }) + + it("puts truly unassigned items into Unassigned", () => { + const orphan = makeReview({ id: "orphan", projectId: "", projectPath: "" }) + const buckets = bucketReviewItems([orphan], "/project-1") + expect(buckets.currentPending).toEqual([]) + expect(buckets.unassigned.map((it) => it.id)).toEqual(["orphan"]) + }) + + it("separates resolved current items from pending count", () => { + const pending = makeReview({ id: "pending", resolved: false }) + const resolved = makeReview({ id: "resolved", title: "Beta", resolved: true }) + const buckets = bucketReviewItems([pending, resolved], "/project-1") + expect(buckets.currentPending.map((it) => it.id)).toEqual(["pending"]) + expect(buckets.currentResolved.map((it) => it.id)).toEqual(["resolved"]) + }) +}) + +describe("needsProjectAssignment", () => { + it("returns true when the item is visible in current project but missing projectId", () => { + expect( + needsProjectAssignment( + makeReview({ projectId: "", projectPath: "/project-1" }), + "/project-1", + ), + ).toBe(true) + }) + + it("returns false for already-assigned current items", () => { + expect( + needsProjectAssignment( + makeReview({ projectId: "proj-1", projectPath: "/project-1" }), + "/project-1", + ), + ).toBe(false) + }) +}) diff --git a/src/lib/review-utils.ts b/src/lib/review-utils.ts index 29e56760..2bd0d2d3 100644 --- a/src/lib/review-utils.ts +++ b/src/lib/review-utils.ts @@ -3,12 +3,50 @@ * Kept dependency-free so both the Zustand store and sweep logic can import * it without creating cycles or pulling heavy LLM modules into the store. */ +import { normalizePath } from "@/lib/path-utils" +import type { ReviewItem } from "@/stores/review-store" // Common prefixes LLM may prepend in English or Chinese review titles. // Kept in one place so dedupe and sweep agree on what "the same concept" means. const REVIEW_TITLE_PREFIX_RE = /^(missing[\s-]?page[:：]\s*|duplicate[\s-]?page[:：]\s*|possible[\s-]?duplicate[:：]\s*|缺失页面[:：]\s*|缺少页面[:：]\s*|重复页面[:：]\s*|疑似重复[:：]\s*)/i +export interface ReviewBuckets { + currentPending: ReviewItem[] + currentResolved: ReviewItem[] + unassigned: ReviewItem[] +} + +function normalizeProjectPath(projectPath: string | undefined): string { + return projectPath ? normalizePath(projectPath) : "" +} + +function normalizeProjectId(projectId: string | undefined): string { + return projectId?.trim() ?? "" +} + +function visibilityKey(item: Pick): string { + return `${normalizeProjectPath(item.projectPath)}::${item.type}::${normalizeReviewTitle(item.title)}` +} + +function isAssigned(item: Pick): boolean { + return normalizeProjectId(item.projectId).length > 0 && normalizeProjectPath(item.projectPath).length > 0 +} + +function pickMoreCanonical(a: ReviewItem, b: ReviewItem): ReviewItem { + const aAssigned = isAssigned(a) + const bAssigned = isAssigned(b) + if (aAssigned !== bAssigned) return aAssigned ? a : b + + if (a.resolved !== b.resolved) return a.resolved ? b : a + + if ((a.createdAt ?? 0) !== (b.createdAt ?? 0)) { + return (a.createdAt ?? 0) >= (b.createdAt ?? 0) ? a : b + } + + return a +} + /** * Normalize a review title for equality comparison: * - strip leading "Missing page:" / "缺失页面:" / etc. @@ -25,3 +63,56 @@ export function normalizeReviewTitle(title: string): string { .trim() .toLowerCase() } + +export function canonicalizeReviewItems(items: ReviewItem[]): ReviewItem[] { + const byKey = new Map() + + for (const item of items) { + const key = visibilityKey(item) + const existing = byKey.get(key) + byKey.set(key, existing ? pickMoreCanonical(existing, item) : item) + } + + const survivors = new Set(byKey.values()) + return items.filter((item) => survivors.has(item)) +} + +export function bucketReviewItems( + items: ReviewItem[], + currentProjectPath: string | null | undefined, +): ReviewBuckets { + const canonical = canonicalizeReviewItems(items) + const currentPath = normalizeProjectPath(currentProjectPath ?? "") + const currentPending: ReviewItem[] = [] + const currentResolved: ReviewItem[] = [] + const unassigned: ReviewItem[] = [] + const visibleInCurrent = new Set() + + for (const item of canonical) { + const itemPath = normalizeProjectPath(item.projectPath) + if (currentPath && itemPath === currentPath) { + visibleInCurrent.add(visibilityKey(item)) + if (item.resolved) currentResolved.push(item) + else currentPending.push(item) + } + } + + for (const item of canonical) { + const key = visibilityKey(item) + if (visibleInCurrent.has(key)) continue + if (!isAssigned(item)) { + unassigned.push(item) + } + } + + return { currentPending, currentResolved, unassigned } +} + +export function needsProjectAssignment( + item: Pick, + currentProjectPath: string | null | undefined, +): boolean { + const currentPath = normalizeProjectPath(currentProjectPath ?? "") + if (!currentPath) return false + return normalizeProjectPath(item.projectPath) === currentPath && normalizeProjectId(item.projectId).length === 0 +} diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts index 37d71cb0..bce29aef 100644 --- a/src/lib/source-lifecycle.ts +++ b/src/lib/source-lifecycle.ts @@ -9,9 +9,9 @@ import { writeFile, } from "@/commands/fs" import type { WikiProject, FileNode } from "@/types/wiki" -import type { LlmConfig } from "@/stores/wiki-store" +import type { DocumentLlmConfig, LlmConfig } from "@/stores/wiki-store" import { enqueueBatch } from "@/lib/ingest-queue" -import { hasUsableLlm } from "@/lib/has-usable-llm" +import { hasUsableDocumentLlm, hasUsableLlm } from "@/lib/has-usable-llm" import { getFileName, getFileStem, normalizePath } from "@/lib/path-utils" import { parseFrontmatterArray, @@ -90,9 +90,13 @@ export async function enqueueSourceIngest( project: WikiProject, sourcePaths: string[], llmConfig: LlmConfig, + documentLlmConfig?: DocumentLlmConfig, options: { sourceRoot?: string; rootContext?: string } = {}, ): Promise { - if (!hasUsableLlm(llmConfig)) return [] + const llmReady = documentLlmConfig + ? hasUsableDocumentLlm(llmConfig, documentLlmConfig) + : hasUsableLlm(llmConfig) + if (!llmReady) return [] const files = sourcePaths .filter(isIngestableSourcePath) .map((sourcePath) => ({ @@ -110,6 +114,7 @@ export async function importSourceFiles( project: WikiProject, sourcePaths: string[], llmConfig: LlmConfig, + documentLlmConfig?: DocumentLlmConfig, ): Promise { const pp = normalizePath(project.path) const importedPaths: string[] = [] @@ -126,7 +131,7 @@ export async function importSourceFiles( } } - await enqueueSourceIngest(project, importedPaths, llmConfig) + await enqueueSourceIngest(project, importedPaths, llmConfig, documentLlmConfig) return importedPaths } @@ -135,6 +140,7 @@ export async function importSourceFolder( project: WikiProject, selectedFolder: string, llmConfig: LlmConfig, + documentLlmConfig?: DocumentLlmConfig, ): Promise { const pp = normalizePath(project.path) const folderName = getFileName(selectedFolder) || "imported" @@ -145,8 +151,8 @@ export async function importSourceFolder( preprocessFile(filePath).catch(() => {}) } - if (hasUsableLlm(llmConfig)) { - await enqueueSourceIngest(project, copiedFiles, llmConfig, { + if (documentLlmConfig ? hasUsableDocumentLlm(llmConfig, documentLlmConfig) : hasUsableLlm(llmConfig)) { + await enqueueSourceIngest(project, copiedFiles, llmConfig, documentLlmConfig, { sourceRoot: destDir, rootContext: folderName, }) diff --git a/src/lib/sweep-reviews.race.test.ts b/src/lib/sweep-reviews.race.test.ts index 52f2defa..3727ff74 100644 --- a/src/lib/sweep-reviews.race.test.ts +++ b/src/lib/sweep-reviews.race.test.ts @@ -35,6 +35,8 @@ function fileNode(name: string): FileNode { function addPending(items: Array>) { const input = items.map((p) => ({ + projectId: "proj", + projectPath: useWikiStore.getState().project?.path ?? "/project", type: "missing-page" as ReviewItem["type"], title: "X", description: "", diff --git a/src/lib/sweep-reviews.real-llm.test.ts b/src/lib/sweep-reviews.real-llm.test.ts index ffffce19..c4f25b34 100644 --- a/src/lib/sweep-reviews.real-llm.test.ts +++ b/src/lib/sweep-reviews.real-llm.test.ts @@ -69,6 +69,8 @@ async function setup(scenario: typeof sweepScenarios[number]): Promise { useReviewStore.setState({ items: scenario.reviews.map((r) => ({ id: r.id, + projectId: "proj-real-llm", + projectPath: tmp.path, type: r.type, title: r.title, description: r.description ?? "", diff --git a/src/lib/sweep-reviews.scenarios.test.ts b/src/lib/sweep-reviews.scenarios.test.ts index 151c708a..a3cbcb91 100644 --- a/src/lib/sweep-reviews.scenarios.test.ts +++ b/src/lib/sweep-reviews.scenarios.test.ts @@ -85,6 +85,8 @@ async function setupScenario(scenario: SweepScenario): Promise { useReviewStore.setState({ items: reviewsRaw.map((r) => ({ id: r.id, + projectId: "proj-scenario", + projectPath: tmp.path, type: r.type, title: r.title, description: r.description ?? "", diff --git a/src/lib/sweep-reviews.ts b/src/lib/sweep-reviews.ts index 2a0506df..85478569 100644 --- a/src/lib/sweep-reviews.ts +++ b/src/lib/sweep-reviews.ts @@ -345,7 +345,10 @@ export async function sweepResolvedReviews( if (!matchesCurrentProject(projectPath)) return 0 const store = useReviewStore.getState() - const pending = store.items.filter((i) => !i.resolved) + const pp = normalizePath(projectPath) + const pending = store.items.filter( + (i) => !i.resolved && normalizePath(i.projectPath) === pp, + ) if (pending.length === 0) return 0 diff --git a/src/stores/research-store.test.ts b/src/stores/research-store.test.ts new file mode 100644 index 00000000..b9ece762 --- /dev/null +++ b/src/stores/research-store.test.ts @@ -0,0 +1,73 @@ +import { beforeEach, describe, expect, it } from "vitest" +import { useResearchStore } from "./research-store" + +describe("research-store", () => { + beforeEach(() => { + useResearchStore.setState({ + tasks: [], + panelOpen: false, + maxConcurrent: 3, + }) + }) + + it("clearFinished removes only done and error tasks", () => { + useResearchStore.setState({ + tasks: [ + { + id: "queued", + projectId: "proj", + projectPath: "/project", + topic: "queued", + status: "queued", + webResults: [], + synthesis: "", + savedPath: null, + error: null, + createdAt: 1, + }, + { + id: "searching", + projectId: "proj", + projectPath: "/project", + topic: "searching", + status: "searching", + webResults: [], + synthesis: "", + savedPath: null, + error: null, + createdAt: 2, + }, + { + id: "done", + projectId: "proj", + projectPath: "/project", + topic: "done", + status: "done", + webResults: [], + synthesis: "ok", + savedPath: "wiki/queries/a.md", + error: null, + createdAt: 3, + }, + { + id: "error", + projectId: "proj", + projectPath: "/project", + topic: "error", + status: "error", + webResults: [], + synthesis: "partial", + savedPath: null, + error: "boom", + createdAt: 4, + }, + ], + }) + + useResearchStore.getState().clearFinished() + expect(useResearchStore.getState().tasks.map((task) => task.id)).toEqual([ + "queued", + "searching", + ]) + }) +}) diff --git a/src/stores/research-store.ts b/src/stores/research-store.ts index 954cb2f6..d7962ec7 100644 --- a/src/stores/research-store.ts +++ b/src/stores/research-store.ts @@ -3,6 +3,8 @@ import type { WebSearchResult } from "@/lib/web-search" export interface ResearchTask { id: string + projectId: string + projectPath: string topic: string searchQueries?: string[] status: "queued" | "searching" | "synthesizing" | "saving" | "done" | "error" @@ -18,9 +20,10 @@ interface ResearchState { panelOpen: boolean maxConcurrent: number - addTask: (topic: string) => string + addTask: (topic: string, projectId: string, projectPath: string) => string updateTask: (id: string, updates: Partial) => void removeTask: (id: string) => void + clearFinished: () => void setPanelOpen: (open: boolean) => void getRunningCount: () => number getNextQueued: () => ResearchTask | undefined @@ -33,13 +36,15 @@ export const useResearchStore = create((set, get) => ({ panelOpen: false, maxConcurrent: 3, - addTask: (topic) => { + addTask: (topic, projectId, projectPath) => { const id = `research-${++counter}` set((state) => ({ tasks: [ ...state.tasks, { id, + projectId, + projectPath, topic, status: "queued", webResults: [], @@ -64,12 +69,17 @@ export const useResearchStore = create((set, get) => ({ tasks: state.tasks.filter((t) => t.id !== id), })), + clearFinished: () => + set((state) => ({ + tasks: state.tasks.filter((t) => t.status !== "done" && t.status !== "error"), + })), + setPanelOpen: (panelOpen) => set({ panelOpen }), getRunningCount: () => { const { tasks } = get() return tasks.filter((t) => - t.status === "searching" || t.status === "synthesizing" || t.status === "saving" + t.status === "searching" || t.status === "synthesizing" || t.status === "saving", ).length }, diff --git a/src/stores/review-store.property.test.ts b/src/stores/review-store.property.test.ts index a30132b4..aaf61c14 100644 --- a/src/stores/review-store.property.test.ts +++ b/src/stores/review-store.property.test.ts @@ -19,6 +19,8 @@ const typeArb = fc.constantFrom( ) const reviewInputArb = fc.record({ + projectId: fc.constant("proj-1"), + projectPath: fc.constant("/project-1"), type: typeArb, title: fc.string({ minLength: 1, maxLength: 60 }), description: fc.string({ maxLength: 100 }), @@ -39,6 +41,8 @@ describe("review-store addItems — dedupe invariants", () => { for (const batch of batches) { const input = batch.map((b) => ({ + projectId: b.projectId, + projectPath: b.projectPath, type: b.type, title: b.title, description: b.description, @@ -69,6 +73,8 @@ describe("review-store addItems — dedupe invariants", () => { for (const pages of affectedBatches) { useReviewStore.getState().addItems([ { + projectId: "proj-1", + projectPath: "/project-1", type, title, description: "", @@ -99,13 +105,13 @@ describe("review-store addItems — dedupe invariants", () => { useReviewStore.setState({ items: [] }) useReviewStore.getState().addItems([ - { type, title, description: "", options: [], affectedPages: ["first.md"] }, + { projectId: "proj-1", projectPath: "/project-1", type, title, description: "", options: [], affectedPages: ["first.md"] }, ]) const firstId = useReviewStore.getState().items[0].id useReviewStore.getState().resolveItem(firstId, "auto-resolved") useReviewStore.getState().addItems([ - { type, title, description: "", options: [], affectedPages: ["second.md"] }, + { projectId: "proj-1", projectPath: "/project-1", type, title, description: "", options: [], affectedPages: ["second.md"] }, ]) const all = useReviewStore.getState().items diff --git a/src/stores/review-store.test.ts b/src/stores/review-store.test.ts index afd9d3f5..b8e92b5c 100644 --- a/src/stores/review-store.test.ts +++ b/src/stores/review-store.test.ts @@ -4,6 +4,8 @@ import { useReviewStore, type ReviewItem } from "./review-store" // Minimal builder so each test only specifies what it cares about. function makeInput(overrides: Partial> = {}) { return { + projectId: "proj-1", + projectPath: "/project-1", type: "missing-page" as ReviewItem["type"], title: "Attention", description: "description", @@ -67,6 +69,14 @@ describe("review-store addItems dedupe", () => { expect(useReviewStore.getState().items).toHaveLength(2) }) + it("does NOT merge items from different projects even with the same normalized title", () => { + useReviewStore.getState().addItems([ + makeInput({ title: "Attention", projectId: "proj-1", projectPath: "/project-1" }), + makeInput({ title: "Attention", projectId: "proj-2", projectPath: "/project-2" }), + ]) + expect(useReviewStore.getState().items).toHaveLength(2) + }) + it("does NOT merge into a resolved item (creates a new one)", () => { const store = useReviewStore.getState() store.addItems([makeInput({ title: "Attention" })]) diff --git a/src/stores/review-store.ts b/src/stores/review-store.ts index 33ed2e8c..62ccbfbd 100644 --- a/src/stores/review-store.ts +++ b/src/stores/review-store.ts @@ -1,5 +1,6 @@ import { create } from "zustand" -import { normalizeReviewTitle } from "@/lib/review-utils" +import { normalizePath } from "@/lib/path-utils" +import { canonicalizeReviewItems, normalizeReviewTitle } from "@/lib/review-utils" export interface ReviewOption { label: string @@ -8,6 +9,8 @@ export interface ReviewOption { export interface ReviewItem { id: string + projectId: string + projectPath: string type: "contradiction" | "duplicate" | "missing-page" | "confirm" | "suggestion" title: string description: string @@ -27,7 +30,7 @@ interface ReviewState { setItems: (items: ReviewItem[]) => void resolveItem: (id: string, action: string) => void dismissItem: (id: string) => void - clearResolved: () => void + clearResolved: (projectPath?: string) => void } let counter = 0 @@ -55,18 +58,19 @@ export const useReviewStore = create((set) => ({ // from multiple files). // Merge affectedPages / searchQueries / sourcePath instead of duplicating. const result = [...state.items] - const keyFor = (t: string, title: string) => `${t}::${normalizeReviewTitle(title)}` + const keyFor = (projectId: string, t: string, title: string) => + `${projectId}::${t}::${normalizeReviewTitle(title)}` // Build index of existing pending items for fast lookup const pendingIndex = new Map() result.forEach((it, idx) => { if (!it.resolved) { - pendingIndex.set(keyFor(it.type, it.title), idx) + pendingIndex.set(keyFor(it.projectId, it.type, it.title), idx) } }) for (const incoming of items) { - const k = keyFor(incoming.type, incoming.title) + const k = keyFor(incoming.projectId, incoming.type, incoming.title) const existingIdx = pendingIndex.get(k) if (existingIdx !== undefined) { @@ -78,6 +82,7 @@ export const useReviewStore = create((set) => ({ ...old, description: incoming.description || old.description, // prefer newer description sourcePath: incoming.sourcePath ?? old.sourcePath, + projectPath: incoming.projectPath || old.projectPath, affectedPages: mergedPages.length > 0 ? mergedPages : undefined, searchQueries: mergedQueries.length > 0 ? mergedQueries : undefined, } @@ -93,10 +98,10 @@ export const useReviewStore = create((set) => ({ } } - return { items: result } + return { items: canonicalizeReviewItems(result) } }), - setItems: (items) => set({ items }), + setItems: (items) => set({ items: canonicalizeReviewItems(items) }), resolveItem: (id, action) => set((state) => ({ @@ -110,8 +115,12 @@ export const useReviewStore = create((set) => ({ items: state.items.filter((item) => item.id !== id), })), - clearResolved: () => + clearResolved: (projectPath) => set((state) => ({ - items: state.items.filter((item) => !item.resolved), + items: state.items.filter((item) => { + if (!item.resolved) return true + if (!projectPath) return false + return normalizePath(item.projectPath) !== normalizePath(projectPath) + }), })), })) diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts index ec445ac5..717c38ca 100644 --- a/src/stores/wiki-store.ts +++ b/src/stores/wiki-store.ts @@ -147,6 +147,12 @@ interface MultimodalConfig { concurrency: number } +interface DocumentLlmConfig extends LlmConfig { + /** Reuse `llmConfig` for document ingest / merge calls. When true, + * the fields below are ignored. */ + useMainLlm: boolean +} + /** * Output language for LLM-generated content (wiki pages, chat responses, research). * "auto" = detect from user input / source document language. @@ -154,6 +160,7 @@ interface MultimodalConfig { */ type OutputLanguage = | "auto" + | "Chinese (preserve English terms)" | "English" | "Chinese" | "Traditional Chinese" @@ -222,9 +229,11 @@ interface WikiState { activePresetId: string | null searchApiConfig: SearchApiConfig embeddingConfig: EmbeddingConfig + documentLlmConfig: DocumentLlmConfig multimodalConfig: MultimodalConfig outputLanguage: OutputLanguage proxyConfig: ProxyConfig + mcpAccessEnabled: boolean dataVersion: number setProject: (project: WikiProject | null) => void @@ -239,9 +248,11 @@ interface WikiState { setActivePresetId: (id: string | null) => void setSearchApiConfig: (config: SearchApiConfig) => void setEmbeddingConfig: (config: EmbeddingConfig) => void + setDocumentLlmConfig: (config: DocumentLlmConfig) => void setMultimodalConfig: (config: MultimodalConfig) => void setOutputLanguage: (lang: OutputLanguage) => void setProxyConfig: (config: ProxyConfig) => void + setMcpAccessEnabled: (enabled: boolean) => void bumpDataVersion: () => void } @@ -290,6 +301,18 @@ export const useWikiStore = create((set) => ({ model: "", }, + documentLlmConfig: { + useMainLlm: true, + provider: "custom", + apiKey: "", + model: "", + ollamaUrl: "http://localhost:11434", + customEndpoint: "", + maxContextSize: 204800, + apiMode: "chat_completions", + reasoning: { mode: "auto" }, + }, + multimodalConfig: { // Off by default — captioning is a non-trivial token spend // (one VLM call per extracted image), and silently turning it @@ -315,15 +338,19 @@ export const useWikiStore = create((set) => ({ bypassLocal: true, }, + mcpAccessEnabled: false, + setLlmConfig: (llmConfig) => set({ llmConfig }), setProviderConfigs: (providerConfigs) => set({ providerConfigs }), setActivePresetId: (activePresetId) => set({ activePresetId }), setSearchApiConfig: (searchApiConfig) => set({ searchApiConfig }), setEmbeddingConfig: (embeddingConfig) => set({ embeddingConfig }), + setDocumentLlmConfig: (documentLlmConfig) => set({ documentLlmConfig }), setMultimodalConfig: (multimodalConfig) => set({ multimodalConfig }), setOutputLanguage: (outputLanguage) => set({ outputLanguage }), setProxyConfig: (proxyConfig) => set({ proxyConfig }), + setMcpAccessEnabled: (mcpAccessEnabled) => set({ mcpAccessEnabled }), bumpDataVersion: () => set((state) => ({ dataVersion: state.dataVersion + 1 })), })) -export type { WikiState, LlmConfig, SearchApiConfig, EmbeddingConfig, MultimodalConfig, OutputLanguage, ProxyConfig } +export type { WikiState, LlmConfig, SearchApiConfig, EmbeddingConfig, DocumentLlmConfig, MultimodalConfig, OutputLanguage, ProxyConfig } diff --git a/src/test-helpers/fs-temp.ts b/src/test-helpers/fs-temp.ts index 290926e4..6168be31 100644 --- a/src/test-helpers/fs-temp.ts +++ b/src/test-helpers/fs-temp.ts @@ -54,7 +54,12 @@ export const realFs = { }, writeFile: async (p: string, contents: string): Promise => { await fs.mkdir(path.dirname(p), { recursive: true }) - await fs.writeFile(p, contents, "utf-8") + const tmp = path.join( + path.dirname(p), + `.${path.basename(p)}.${Date.now()}-${Math.random().toString(36).slice(2)}.tmp`, + ) + await fs.writeFile(tmp, contents, "utf-8") + await fs.rename(tmp, p) }, listDirectory: async (p: string): Promise => { return buildTree(p) @@ -82,6 +87,18 @@ export const realFs = { throw new Error("openProject not supported in tests") }, clipServerStatus: async (): Promise => "ok", + mcpServerConfig: async () => ({ + scriptPath: "/repo/mcp-server/llmwiki-mcp.js", + codexCommand: 'codex mcp add llmwiki -- node "/repo/mcp-server/llmwiki-mcp.js"', + jsonConfig: JSON.stringify({ + mcpServers: { + llmwiki: { + command: "node", + args: ["/repo/mcp-server/llmwiki-mcp.js"], + }, + }, + }), + }), } /**