From e1148a1e26114db41ab9006f3c3a45a71fc0bd95 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Wed, 26 Nov 2025 19:06:45 +0200 Subject: [PATCH 01/17] Add Excel file handling support with file handler architecture - Implement ExcelFileHandler for .xlsx/.xls/.xlsm files using xlsx library - Add FileHandler interface with read(), write(), getInfo(), editRange() methods - Create file handler factory for automatic handler selection by extension - Support Excel-specific features: sheet selection, range queries, offset/length - Add edit_block range mode for Excel cell/range updates - Refactor handleGetFileInfo to be generic (handles any file type's nested data) - Remove .describe() calls from schemas (WriteFileArgsSchema, EditBlockArgsSchema) - Add technical debt comments documenting text edit architectural inconsistency --- manifest.template.json | 6 +- package-lock.json | 538 ++++++++++++++++++++++++--- package.json | 1 + src/handlers/filesystem-handlers.ts | 67 +++- src/handlers/index.ts | 1 + src/handlers/node-handlers.ts | 90 +++++ src/search-manager.ts | 208 ++++++++++- src/server.ts | 106 ++++-- src/tools/edit.ts | 86 ++++- src/tools/filesystem.ts | 181 ++++----- src/tools/schemas.ts | 30 +- src/utils/files/base.ts | 200 ++++++++++ src/utils/files/binary.ts | 173 +++++++++ src/utils/files/excel.ts | 486 ++++++++++++++++++++++++ src/utils/files/factory.ts | 89 +++++ src/utils/files/image.ts | 93 +++++ src/utils/files/index.ts | 16 + src/utils/files/text.ts | 441 ++++++++++++++++++++++ test/test-excel-files.js | 369 ++++++++++++++++++ test/test-file-handlers.js | 313 ++++++++++++++++ test/test_output/node_repl_debug.txt | 24 +- 21 files changed, 3307 insertions(+), 211 deletions(-) create mode 100644 src/handlers/node-handlers.ts create mode 100644 src/utils/files/base.ts create mode 100644 src/utils/files/binary.ts create mode 100644 src/utils/files/excel.ts create mode 100644 src/utils/files/factory.ts create mode 100644 src/utils/files/image.ts create mode 100644 src/utils/files/index.ts create mode 100644 src/utils/files/text.ts create mode 100644 test/test-excel-files.js create mode 100644 test/test-file-handlers.js diff --git a/manifest.template.json b/manifest.template.json index 9ff08da2..4da5a1de 100644 --- a/manifest.template.json +++ b/manifest.template.json @@ -88,7 +88,7 @@ }, { "name": "edit_block", - "description": "Apply surgical text replacements to files. Make small, focused edits with minimal context for precision." + "description": "Apply surgical edits to files. Supports text replacement (old_string/new_string) with fuzzy matching for text files, and range updates (range/content) for Excel files." }, { "name": "start_process", @@ -133,6 +133,10 @@ { "name": "get_prompts", "description": "Browse and retrieve curated Desktop Commander prompts for various tasks and workflows." + }, + { + "name": "execute_node", + "description": "Execute Node.js code directly using the MCP server's Node runtime. Supports ES modules with top-level await." } ], "keywords": [ diff --git a/package-lock.json b/package-lock.json index c7f194d8..9d15a9e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,19 @@ { "name": "@wonderwhy-er/desktop-commander", - "version": "0.2.21", + "version": "0.2.23", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@wonderwhy-er/desktop-commander", - "version": "0.2.21", + "version": "0.2.23", "hasInstallScript": true, "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.9.0", "@vscode/ripgrep": "^1.15.9", "cross-fetch": "^4.1.0", + "exceljs": "^4.4.0", "fastest-levenshtein": "^1.0.16", "glob": "^10.3.10", "isbinaryfile": "^5.0.4", @@ -85,6 +86,47 @@ "node": ">=14.17.0" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, "node_modules/@inquirer/checkbox": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-3.0.1.tgz", @@ -1019,7 +1061,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1171,7 +1212,6 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", - "dev": true, "license": "MIT", "dependencies": { "archiver-utils": "^2.1.0", @@ -1190,7 +1230,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", - "dev": true, "license": "MIT", "dependencies": { "glob": "^7.1.4", @@ -1212,7 +1251,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -1224,7 +1262,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -1245,7 +1282,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -1258,7 +1294,6 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -1274,14 +1309,12 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, "license": "MIT" }, "node_modules/archiver-utils/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" @@ -1301,7 +1334,6 @@ "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "dev": true, "license": "MIT" }, "node_modules/balanced-match": { @@ -1314,7 +1346,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -1331,6 +1362,15 @@ ], "license": "MIT" }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -1341,6 +1381,19 @@ "node": "*" } }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -1358,7 +1411,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, "license": "MIT", "dependencies": { "buffer": "^5.5.0", @@ -1366,6 +1418,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", @@ -1443,7 +1501,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001688", "electron-to-chromium": "^1.5.73", @@ -1461,7 +1518,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -1523,6 +1579,23 @@ "dev": true, "license": "MIT" }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1640,6 +1713,18 @@ "node": ">=4" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -1847,7 +1932,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", - "dev": true, "license": "MIT", "dependencies": { "buffer-crc32": "^0.2.13", @@ -1863,7 +1947,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/config-chain": { @@ -1920,7 +2003,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true, "license": "MIT" }, "node_modules/cors": { @@ -1940,7 +2022,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -1953,7 +2034,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", - "dev": true, "license": "MIT", "dependencies": { "crc-32": "^1.2.0", @@ -1986,6 +2066,12 @@ "node": ">= 8" } }, + "node_modules/dayjs": { + "version": "1.11.19", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", + "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -2580,6 +2666,45 @@ "node": ">= 0.4" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/duplexer3": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz", @@ -2635,7 +2760,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -2818,6 +2942,35 @@ "node": ">=18.0.0" } }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/exceljs/node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/express": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/express/-/express-5.0.1.tgz", @@ -2954,6 +3107,19 @@ "node": ">=0.10.0" } }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -3230,7 +3396,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true, "license": "MIT" }, "node_modules/fs-extra": { @@ -3252,7 +3417,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -3270,6 +3434,90 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/fstream/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/fstream/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fstream/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/fstream/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/fstream/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -3473,7 +3721,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, "license": "ISC" }, "node_modules/has-flag": { @@ -3599,7 +3846,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -3633,6 +3879,12 @@ "dev": true, "license": "ISC" }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/import-local": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", @@ -3658,7 +3910,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -3852,7 +4103,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true, "license": "MIT" }, "node_modules/isbinaryfile": { @@ -3999,6 +4249,48 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -4023,7 +4315,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "^2.0.5" @@ -4036,7 +4327,6 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -4052,19 +4342,32 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, "license": "MIT" }, "node_modules/lazystream/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "license": "ISC" + }, "node_modules/loader-runner": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", @@ -4114,35 +4417,79 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "dev": true, "license": "MIT" }, "node_modules/lodash.difference": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "license": "MIT" }, "node_modules/lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", "license": "MIT" }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", "license": "MIT" }, "node_modules/lodash.union": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", "license": "MIT" }, "node_modules/log-symbols": { @@ -4344,7 +4691,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4557,7 +4903,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4807,6 +5152,12 @@ "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", "license": "BlueOak-1.0.0" }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -4830,7 +5181,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4995,7 +5345,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true, "license": "MIT" }, "node_modules/proto-list": { @@ -5153,7 +5502,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -5168,7 +5516,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", - "dev": true, "license": "Apache-2.0", "dependencies": { "minimatch": "^5.1.0" @@ -5178,7 +5525,6 @@ "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -5470,6 +5816,18 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/schema-utils": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", @@ -5570,6 +5928,12 @@ "node": ">= 18" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -5899,7 +6263,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -6078,7 +6441,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, "license": "MIT", "dependencies": { "bl": "^4.0.3", @@ -6151,7 +6513,6 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6285,6 +6646,15 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/trim-repeated": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz", @@ -6438,7 +6808,6 @@ "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6491,6 +6860,54 @@ "node": ">= 0.8" } }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", @@ -6558,7 +6975,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/utils-merge": { @@ -6570,6 +6986,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -6615,7 +7040,6 @@ "integrity": "sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.6", @@ -6663,7 +7087,6 @@ "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -6784,7 +7207,6 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6988,6 +7410,12 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -7025,7 +7453,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", - "dev": true, "license": "MIT", "dependencies": { "archiver-utils": "^3.0.4", @@ -7040,7 +7467,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", - "dev": true, "license": "MIT", "dependencies": { "glob": "^7.2.3", @@ -7062,7 +7488,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -7074,7 +7499,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -7095,7 +7519,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -7109,7 +7532,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 1f363c78..71078891 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "fastest-levenshtein": "^1.0.16", "glob": "^10.3.10", "isbinaryfile": "^5.0.4", + "exceljs": "^4.4.0", "zod": "^3.24.1", "zod-to-json-schema": "^3.23.5" }, diff --git a/src/handlers/filesystem-handlers.ts b/src/handlers/filesystem-handlers.ts index bf09ad82..fea83a04 100644 --- a/src/handlers/filesystem-handlers.ts +++ b/src/handlers/filesystem-handlers.ts @@ -9,6 +9,7 @@ import { type FileResult, type MultiFileResult } from '../tools/filesystem.js'; +import type { ReadOptions } from '../utils/files/base.js'; import {ServerResult} from '../types.js'; import {withTimeout} from '../utils/withTimeout.js'; @@ -59,23 +60,26 @@ export async function handleReadFile(args: unknown): Promise { const defaultLimit = config.fileReadLineLimit ?? 1000; - // Use the provided limits or defaults - const offset = parsed.offset ?? 0; - const length = parsed.length ?? defaultLimit; - - const fileResult = await readFile(parsed.path, parsed.isUrl, offset, length); + const options: ReadOptions = { + isUrl: parsed.isUrl, + offset: parsed.offset ?? 0, + length: parsed.length ?? defaultLimit, + sheet: parsed.sheet, + range: parsed.range + }; + const fileResult = await readFile(parsed.path, options); - if (fileResult.isImage) { + if (fileResult.metadata?.isImage) { // For image files, return as an image content type return { content: [ - { - type: "text", - text: `Image file: ${parsed.path} (${fileResult.mimeType})\n` + { + type: "text", + text: `Image file: ${parsed.path} (${fileResult.mimeType})\n` }, { type: "image", - data: fileResult.content, + data: fileResult.content.toString(), mimeType: fileResult.mimeType } ], @@ -83,7 +87,7 @@ export async function handleReadFile(args: unknown): Promise { } else { // For all other files, return as text return { - content: [{ type: "text", text: fileResult.content }], + content: [{ type: "text", text: fileResult.content.toString() }], }; } }; @@ -241,6 +245,33 @@ export async function handleMoveFile(args: unknown): Promise { } } +/** + * Format a value for display, handling objects and arrays + */ +function formatValue(value: unknown, indent: string = ''): string { + if (value === null || value === undefined) { + return String(value); + } + if (Array.isArray(value)) { + if (value.length === 0) return '[]'; + // For arrays of objects (like sheets), format each item + const items = value.map((item, i) => { + if (typeof item === 'object' && item !== null) { + const props = Object.entries(item) + .map(([k, v]) => `${k}: ${v}`) + .join(', '); + return `${indent} [${i}] { ${props} }`; + } + return `${indent} [${i}] ${item}`; + }); + return `\n${items.join('\n')}`; + } + if (typeof value === 'object') { + return JSON.stringify(value); + } + return String(value); +} + /** * Handle get_file_info command */ @@ -248,12 +279,16 @@ export async function handleGetFileInfo(args: unknown): Promise { try { const parsed = GetFileInfoArgsSchema.parse(args); const info = await getFileInfo(parsed.path); + + // Generic formatting for any file type + const formattedText = Object.entries(info) + .map(([key, value]) => `${key}: ${formatValue(value)}`) + .join('\n'); + return { - content: [{ - type: "text", - text: Object.entries(info) - .map(([key, value]) => `${key}: ${value}`) - .join('\n') + content: [{ + type: "text", + text: formattedText }], }; } catch (error) { diff --git a/src/handlers/index.ts b/src/handlers/index.ts index 1ac19090..10f6eebd 100644 --- a/src/handlers/index.ts +++ b/src/handlers/index.ts @@ -5,3 +5,4 @@ export * from './process-handlers.js'; export * from './edit-search-handlers.js'; export * from './search-handlers.js'; export * from './history-handlers.js'; +export * from './node-handlers.js'; diff --git a/src/handlers/node-handlers.ts b/src/handlers/node-handlers.ts new file mode 100644 index 00000000..cbb76c7d --- /dev/null +++ b/src/handlers/node-handlers.ts @@ -0,0 +1,90 @@ +import { spawn } from 'child_process'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +import { ExecuteNodeArgsSchema } from '../tools/schemas.js'; +import { ServerResult } from '../types.js'; + +// Get the directory where the MCP is installed (for requiring packages like exceljs) +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const mcpRoot = path.resolve(__dirname, '..', '..'); + +/** + * Handle execute_node command + * Executes Node.js code using the same Node runtime as the MCP + */ +export async function handleExecuteNode(args: unknown): Promise { + const parsed = ExecuteNodeArgsSchema.parse(args); + const { code, timeout_ms } = parsed; + + // Create temp file IN THE MCP DIRECTORY so ES module imports resolve correctly + // (ES modules resolve packages relative to file location, not NODE_PATH or cwd) + const tempFile = path.join(mcpRoot, `.mcp-exec-${Date.now()}-${Math.random().toString(36).slice(2)}.mjs`); + + // User code runs directly - imports will resolve from mcpRoot/node_modules + const wrappedCode = code; + + try { + await fs.writeFile(tempFile, wrappedCode, 'utf8'); + + const result = await new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { + const proc = spawn(process.execPath, [tempFile], { + cwd: mcpRoot, + timeout: timeout_ms + }); + + let stdout = ''; + let stderr = ''; + + proc.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + proc.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + proc.on('close', (exitCode) => { + resolve({ stdout, stderr, exitCode: exitCode ?? 1 }); + }); + + proc.on('error', (err) => { + resolve({ stdout, stderr: stderr + '\n' + err.message, exitCode: 1 }); + }); + }); + + // Clean up temp file + await fs.unlink(tempFile).catch(() => {}); + + if (result.exitCode !== 0) { + return { + content: [{ + type: "text", + text: `Execution failed (exit code ${result.exitCode}):\n${result.stderr}\n${result.stdout}` + }], + isError: true + }; + } + + return { + content: [{ + type: "text", + text: result.stdout || '(no output)' + }] + }; + + } catch (error) { + // Clean up temp file on error + await fs.unlink(tempFile).catch(() => {}); + + return { + content: [{ + type: "text", + text: `Failed to execute Node.js code: ${error instanceof Error ? error.message : String(error)}` + }], + isError: true + }; + } +} diff --git a/src/search-manager.ts b/src/search-manager.ts index 980f1d3c..d2af0558 100644 --- a/src/search-manager.ts +++ b/src/search-manager.ts @@ -1,8 +1,10 @@ import { spawn, ChildProcess } from 'child_process'; import path from 'path'; +import fs from 'fs/promises'; import { validatePath } from './tools/filesystem.js'; import { capture } from './utils/capture.js'; import { getRipgrepPath } from './utils/ripgrep-resolver.js'; +import { isExcelFile } from './utils/files/index.js'; export interface SearchResult { file: string; @@ -144,6 +146,27 @@ export interface SearchSessionOptions { validatedPath: validPath }); + // For content searches, also search Excel files in parallel + let excelSearchPromise: Promise | null = null; + if (options.searchType === 'content') { + excelSearchPromise = this.searchExcelFiles( + validPath, + options.pattern, + options.ignoreCase !== false, + options.maxResults, + options.filePattern // Pass filePattern to filter Excel files too + ).then(excelResults => { + // Add Excel results to session + for (const result of excelResults) { + session.results.push(result); + session.totalMatches++; + } + }).catch((err) => { + // Log Excel search errors but don't fail the whole search + capture('excel_search_error', { error: err instanceof Error ? err.message : String(err) }); + }); + } + // Wait for first chunk of data or early completion instead of fixed delay const firstChunk = new Promise(resolve => { const onData = () => { @@ -153,7 +176,9 @@ export interface SearchSessionOptions { session.process.stdout?.once('data', onData); setTimeout(resolve, 40); // cap at 40ms instead of 50-100ms }); - await firstChunk; + + // Wait for both ripgrep first chunk and Excel search + await Promise.all([firstChunk, excelSearchPromise].filter(Boolean)); return { sessionId, @@ -275,6 +300,187 @@ export interface SearchSessionOptions { })); } + /** + * Search Excel files for content matches + * Called during content search to include Excel files alongside text files + * Searches ALL sheets in each Excel file (row-wise for cross-column matching) + * + * TODO: Refactor - Extract Excel search logic to separate module (src/utils/search/excel-search.ts) + * and inject into SearchManager, similar to how file handlers are structured in src/utils/files/ + * This would allow adding other file type searches (PDF, etc.) without bloating search-manager.ts + */ + private async searchExcelFiles( + rootPath: string, + pattern: string, + ignoreCase: boolean, + maxResults?: number, + filePattern?: string + ): Promise { + const results: SearchResult[] = []; + + // Build regex for matching content + const flags = ignoreCase ? 'i' : ''; + let regex: RegExp; + try { + regex = new RegExp(pattern, flags); + } catch { + // If pattern is not valid regex, escape it for literal matching + const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + regex = new RegExp(escaped, flags); + } + + // Find Excel files recursively + let excelFiles = await this.findExcelFiles(rootPath); + + // Filter by filePattern if provided + if (filePattern) { + const patterns = filePattern.split('|').map(p => p.trim()).filter(Boolean); + excelFiles = excelFiles.filter(filePath => { + const fileName = path.basename(filePath); + return patterns.some(pat => { + // Support glob-like patterns + if (pat.includes('*')) { + const regexPat = pat.replace(/\./g, '\\.').replace(/\*/g, '.*'); + return new RegExp(`^${regexPat}$`, 'i').test(fileName); + } + // Exact match (case-insensitive) + return fileName.toLowerCase() === pat.toLowerCase(); + }); + }); + } + + // Dynamically import ExcelJS to search all sheets + const ExcelJS = await import('exceljs'); + + for (const filePath of excelFiles) { + if (maxResults && results.length >= maxResults) break; + + try { + const workbook = new ExcelJS.default.Workbook(); + await workbook.xlsx.readFile(filePath); + + // Search ALL sheets in the workbook (row-wise for speed and cross-column matching) + for (const worksheet of workbook.worksheets) { + if (maxResults && results.length >= maxResults) break; + + const sheetName = worksheet.name; + + // Iterate through rows (faster than cell-by-cell) + worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => { + if (maxResults && results.length >= maxResults) return; + + // Build a concatenated string of all cell values in the row + const rowValues: string[] = []; + row.eachCell({ includeEmpty: false }, (cell) => { + if (cell.value === null || cell.value === undefined) return; + + let cellStr: string; + if (typeof cell.value === 'object') { + if ('result' in cell.value) { + cellStr = String(cell.value.result ?? ''); + } else if ('richText' in cell.value) { + cellStr = (cell.value as any).richText.map((rt: any) => rt.text).join(''); + } else if ('text' in cell.value) { + cellStr = String((cell.value as any).text); + } else { + cellStr = String(cell.value); + } + } else { + cellStr = String(cell.value); + } + + if (cellStr.trim()) { + rowValues.push(cellStr); + } + }); + + // Join all cell values with space for cross-column matching + const rowText = rowValues.join(' '); + + if (regex.test(rowText)) { + // Extract the matching portion for display + const match = rowText.match(regex); + const matchContext = match + ? this.getMatchContext(rowText, match.index || 0, match[0].length) + : rowText.substring(0, 150); + + results.push({ + file: `${filePath}:${sheetName}!Row${rowNumber}`, + line: rowNumber, + match: matchContext, + type: 'content' + }); + } + }); + } + } catch (error) { + // Skip files that can't be read (permission issues, corrupted, etc.) + continue; + } + } + + return results; + } + + /** + * Find all Excel files in a directory recursively + */ + private async findExcelFiles(rootPath: string): Promise { + const excelFiles: string[] = []; + + async function walk(dir: string): Promise { + try { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + // Skip node_modules, .git, etc. + if (!entry.name.startsWith('.') && entry.name !== 'node_modules') { + await walk(fullPath); + } + } else if (entry.isFile() && isExcelFile(entry.name)) { + excelFiles.push(fullPath); + } + } + } catch { + // Skip directories we can't read + } + } + + // Check if rootPath is a file or directory + try { + const stats = await fs.stat(rootPath); + if (stats.isFile() && isExcelFile(rootPath)) { + return [rootPath]; + } else if (stats.isDirectory()) { + await walk(rootPath); + } + } catch { + // Path doesn't exist or can't be accessed + } + + return excelFiles; + } + + /** + * Extract context around a match for display (show surrounding text) + */ + private getMatchContext(text: string, matchStart: number, matchLength: number): string { + const contextChars = 50; // chars before and after match + const start = Math.max(0, matchStart - contextChars); + const end = Math.min(text.length, matchStart + matchLength + contextChars); + + let context = text.substring(start, end); + + // Add ellipsis if truncated + if (start > 0) context = '...' + context; + if (end < text.length) context = context + '...'; + + return context; + } + /** * Clean up completed sessions older than specified time * Called automatically by cleanup interval diff --git a/src/server.ts b/src/server.ts index 5f032199..008e4cd1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -46,6 +46,7 @@ import { ListSearchesArgsSchema, GetPromptsArgsSchema, GetRecentToolCallsArgsSchema, + ExecuteNodeArgsSchema, } from './tools/schemas.js'; import {getConfig, setConfigValue} from './tools/config.js'; import {getUsageStats} from './tools/usage.js'; @@ -261,9 +262,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { Can fetch content from URLs when isUrl parameter is set to true (URLs are always read in full regardless of offset/length). - Handles text files normally and image files are returned as viewable images. - Recognized image types: PNG, JPEG, GIF, WebP. - + FORMAT HANDLING (by extension): + - Text: Uses offset/length for line-based pagination + - Excel (.xlsx, .xls, .xlsm): Returns JSON 2D array + * Use sheet param: name (string) or index (number, 0-based) + * Use range param: A1 notation, e.g., "A1:D100" + * offset/length work as row pagination (optional fallback) + - Images (PNG, JPEG, GIF, WebP): Base64 encoded viewable content + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(ReadFileArgsSchema), @@ -296,7 +302,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { { name: "write_file", description: ` - Write or append to file contents. + Write or append to file contents. CHUNKING IS STANDARD PRACTICE: Always write files in chunks of 25-30 lines maximum. This is the normal, recommended way to write files - not an emergency measure. @@ -312,16 +318,21 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { 1. Any file expected to be longer than 25-30 lines 2. When writing multiple files in sequence 3. When creating documentation, code files, or configuration files - + HANDLING CONTINUATION ("Continue" prompts): If user asks to "Continue" after an incomplete operation: 1. Read the file to see what was successfully written 2. Continue writing ONLY the remaining content using {mode: 'append'} 3. Keep chunks to 25-30 lines each - + + FORMAT HANDLING (by extension): + - Text files: String content + - Excel (.xlsx, .xls, .xlsm): JSON 2D array or {"SheetName": [[...]]} + Example: '[["Name","Age"],["Alice",30]]' + Files over 50 lines will generate performance notes but are still written successfully. Only works within allowed directories. - + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(WriteFileArgsSchema), @@ -550,13 +561,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { Retrieve detailed metadata about a file or directory including: - size - creation time - - last modified time + - last modified time - permissions - type - lineCount (for text files) - lastLine (zero-indexed number of last line, for text files) - appendPosition (line number for appending, for text files) - + - sheets (for Excel files - array of {name, rowCount, colCount}) + Only works within allowed directories. ${PATH_GUIDANCE} @@ -569,45 +581,54 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { }, // Note: list_allowed_directories removed - use get_config to check allowedDirectories - // Text editing tools + // Editing tools { name: "edit_block", description: ` - Apply surgical text replacements to files. - + Apply surgical edits to files. + BEST PRACTICE: Make multiple small, focused edits rather than one large edit. - Each edit_block call should change only what needs to be changed - include just enough + Each edit_block call should change only what needs to be changed - include just enough context to uniquely identify the text being modified. - + + FORMAT HANDLING (by extension): + + EXCEL FILES (.xlsx, .xls, .xlsm) - Range Update mode: + Takes: + - file_path: Path to the Excel file + - range: "SheetName!A1:C10" or "SheetName" for whole sheet + - content: 2D array, e.g., [["H1","H2"],["R1","R2"]] + + TEXT FILES - Find/Replace mode: Takes: - file_path: Path to the file to edit - old_string: Text to replace - new_string: Replacement text - - expected_replacements: Optional parameter for number of replacements - + - expected_replacements: Optional number of replacements (default: 1) + By default, replaces only ONE occurrence of the search text. - To replace multiple occurrences, provide the expected_replacements parameter with + To replace multiple occurrences, provide expected_replacements with the exact number of matches expected. - + UNIQUENESS REQUIREMENT: When expected_replacements=1 (default), include the minimal amount of context necessary (typically 1-3 lines) before and after the change point, with exact whitespace and indentation. - + When editing multiple sections, make separate edit_block calls for each distinct change rather than one large replacement. - + When a close but non-exact match is found, a character-level diff is shown in the format: common_prefix{-removed-}{+added+}common_suffix to help you identify what's different. - + Similar to write_file, there is a configurable line limit (fileWriteLineLimit) that warns if the edited file exceeds this limit. If this happens, consider breaking your edits into smaller, more focused changes. - + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(EditBlockArgsSchema), annotations: { - title: "Edit Text Block", + title: "Edit Block", readOnlyHint: false, destructiveHint: true, openWorldHint: false, @@ -964,9 +985,34 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { If unclear from context, use: "exploring tool capabilities" The prompt content will be injected and execution begins immediately. - + ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(GetPromptsArgsSchema), + }, + { + name: "execute_node", + description: ` + Execute Node.js code directly using the MCP server's Node runtime. + + Code runs as ES module (.mjs) with top-level await support. + Uses the same Node.js environment that runs the MCP server. + + Available libraries: exceljs (for Excel file manipulation), and all Node.js built-ins. + + Use cases: Data transformations, bulk file operations, complex calculations, + Excel manipulation, JSON processing, or any task better suited to code than tools. + + Output: Use console.log() to return results. Stdout is captured and returned. + + ${PATH_GUIDANCE} + ${CMD_PREFIX_DESCRIPTION}`, + inputSchema: zodToJsonSchema(ExecuteNodeArgsSchema), + annotations: { + title: "Execute Node.js Code", + readOnlyHint: false, + destructiveHint: true, + openWorldHint: true, + }, } ]; @@ -1153,6 +1199,18 @@ server.setRequestHandler(CallToolRequestSchema, async (request: CallToolRequest) result = await handlers.handleKillProcess(args); break; + case "execute_node": + try { + result = await handlers.handleExecuteNode(args); + } catch (error) { + capture('server_request_error', {message: `Error in execute_node handler: ${error}`}); + result = { + content: [{type: "text", text: `Error executing Node.js code: ${error instanceof Error ? error.message : String(error)}`}], + isError: true, + }; + } + break; + // Note: REPL functionality removed in favor of using general terminal commands // Filesystem tools diff --git a/src/tools/edit.ts b/src/tools/edit.ts index 5847a16c..05453cf0 100644 --- a/src/tools/edit.ts +++ b/src/tools/edit.ts @@ -1,3 +1,20 @@ +/** + * Text file editing via search/replace with fuzzy matching support. + * + * TECHNICAL DEBT / ARCHITECTURAL NOTE: + * This file contains text editing logic that should ideally live in TextFileHandler.editRange() + * to be consistent with how Excel editing works (ExcelFileHandler.editRange()). + * + * Current inconsistency: + * - Excel: edit_block → ExcelFileHandler.editRange() ✓ uses file handler + * - Text: edit_block → performSearchReplace() here → bypasses TextFileHandler + * + * Future refactor should: + * 1. Move performSearchReplace() + fuzzy logic into TextFileHandler.editRange() + * 2. Make this file a thin dispatch layer that routes to appropriate FileHandler + * 3. Unify the editRange() signature to handle both text search/replace and structured edits + */ + import { readFile, writeFile, readFileInternal, validatePath } from './filesystem.js'; import fs from 'fs/promises'; import { ServerResult } from '../types.js'; @@ -337,17 +354,72 @@ function highlightDifferences(expected: string, actual: string): string { } /** - * Handle edit_block command with enhanced functionality - * - Supports multiple replacements - * - Validates expected replacements count - * - Provides detailed error messages + * Handle edit_block command + * + * 1. Text files: String replacement (old_string/new_string) + * - Uses fuzzy matching for resilience + * - Handles expected_replacements parameter + * + * 2. Structured files (Excel): Range rewrite (range + content) + * - Bulk updates to cell ranges (e.g., "Sheet1!A1:C10") + * - Whole sheet replacement (e.g., "Sheet1") + * - More powerful and simpler than surgical location-based edits + * - Supports chunking for large datasets (e.g., 1000 rows at a time) + */ export async function handleEditBlock(args: unknown): Promise { const parsed = EditBlockArgsSchema.parse(args); - + + // Structured files: Range rewrite + if (parsed.range && parsed.content !== undefined) { + try { + const { getFileHandler } = await import('../utils/files/factory.js'); + const handler = getFileHandler(parsed.file_path); + + // Parse content if it's a JSON string (AI often sends arrays as JSON strings) + let content = parsed.content; + if (typeof content === 'string') { + try { + content = JSON.parse(content); + } catch { + // Leave as-is if not valid JSON - let handler decide + } + } + + // Check if handler supports range editing + if ('editRange' in handler && typeof handler.editRange === 'function') { + await handler.editRange(parsed.file_path, parsed.range, content, parsed.options); + return { + content: [{ + type: "text", + text: `Successfully updated range ${parsed.range} in ${parsed.file_path}` + }], + }; + } else { + return { + content: [{ + type: "text", + text: `Error: Range-based editing not supported for ${parsed.file_path}` + }], + isError: true + }; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { + content: [{ + type: "text", + text: `Error: ${errorMessage}` + }], + isError: true + }; + } + } + + // Text files: String replacement const searchReplace = { - search: parsed.old_string, - replace: parsed.new_string + search: parsed.old_string!, + replace: parsed.new_string! }; return performSearchReplace(parsed.file_path, searchReplace, parsed.expected_replacements); diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts index d9e634ed..21346589 100644 --- a/src/tools/filesystem.ts +++ b/src/tools/filesystem.ts @@ -8,6 +8,8 @@ import { isBinaryFile } from 'isbinaryfile'; import {capture} from '../utils/capture.js'; import {withTimeout} from '../utils/withTimeout.js'; import {configManager} from '../config-manager.js'; +import { getFileHandler } from '../utils/files/factory.js'; +import type { ReadOptions, FileResult } from '../utils/files/base.js'; // CONSTANTS SECTION - Consolidate all timeouts and thresholds const FILE_OPERATION_TIMEOUTS = { @@ -280,12 +282,8 @@ export async function validatePath(requestedPath: string): Promise { return result; } -// File operation tools -export interface FileResult { - content: string; - mimeType: string; - isImage: boolean; -} +// Re-export FileResult from base for consumers +export type { FileResult } from '../utils/files/base.js'; /** @@ -322,12 +320,12 @@ export async function readFileFromUrl(url: string): Promise { const buffer = await response.arrayBuffer(); const content = Buffer.from(buffer).toString('base64'); - return { content, mimeType: contentType, isImage }; + return { content, mimeType: contentType, metadata: { isImage } }; } else { // For text content const content = await response.text(); - return { content, mimeType: contentType, isImage }; + return { content, mimeType: contentType, metadata: { isImage } }; } } catch (error) { // Clear the timeout to prevent memory leaks @@ -484,7 +482,7 @@ async function readLastNLinesReverse(filePath: string, n: number, mimeType: stri ? `${generateEnhancedStatusMessage(result.length, -n, fileTotalLines, true)}\n\n${result.join('\n')}` : result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } finally { await fd.close(); } @@ -525,7 +523,7 @@ async function readFromEndWithReadline(filePath: string, requestedLines: number, const content = includeStatusMessage ? `${generateEnhancedStatusMessage(result.length, -requestedLines, fileTotalLines, true)}\n\n${result.join('\n')}` : result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } /** @@ -553,10 +551,10 @@ async function readFromStartWithReadline(filePath: string, offset: number, lengt if (includeStatusMessage) { const statusMessage = generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false); const content = `${statusMessage}\n\n${result.join('\n')}`; - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } else { const content = result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } } @@ -628,7 +626,7 @@ async function readFromEstimatedPosition(filePath: string, offset: number, lengt const content = includeStatusMessage ? `${generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false)}\n\n${result.join('\n')}` : result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } finally { await fd.close(); } @@ -637,16 +635,21 @@ async function readFromEstimatedPosition(filePath: string, offset: number, lengt /** * Read file content from the local filesystem * @param filePath Path to the file - * @param offset Starting line number to read from (default: 0) - * @param length Maximum number of lines to read (default: from config or 1000) + * @param options Read options (offset, length, sheet, range) * @returns File content or file result with metadata */ -export async function readFileFromDisk(filePath: string, offset: number = 0, length?: number): Promise { +export async function readFileFromDisk( + filePath: string, + options?: ReadOptions +): Promise { + const { offset = 0, sheet, range } = options ?? {}; + let { length } = options ?? {}; + // Add validation for required parameters if (!filePath || typeof filePath !== 'string') { throw new Error('Invalid file path provided'); } - + // Get default length from config if not provided if (length === undefined) { length = await getDefaultReadLength(); @@ -654,7 +657,7 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len const validPath = await validatePath(filePath); - // Get file extension for telemetry using path module consistently + // Get file extension for telemetry const fileExtension = getFileExtension(validPath); // Check file size before attempting to read @@ -675,40 +678,28 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len // If we can't stat the file, continue anyway and let the read operation handle errors } - // Detect the MIME type based on file extension - const { mimeType, isImage } = await getMimeTypeInfo(validPath); - // Use withTimeout to handle potential hangs const readOperation = async () => { - if (isImage) { - // For image files, read as Buffer and convert to base64 - // Images are always read in full, ignoring offset and length - const buffer = await fs.readFile(validPath); - const content = buffer.toString('base64'); + // Get appropriate handler for this file type + const handler = getFileHandler(validPath); + + // Use handler to read the file + const result = await handler.read(validPath, { + offset, + length, + sheet, + range, + includeStatusMessage: true + }); - return { content, mimeType, isImage }; - } else { - // For all other files, use smart positioning approach - try { - return await readFileWithSmartPositioning(validPath, offset, length, mimeType, true); - } catch (error) { - // If it's our binary file instruction error, return it as content - if (error instanceof Error && error.message.includes('Cannot read binary file as text:')) { - return { content: error.message, mimeType: 'text/plain', isImage: false }; - } - - // If UTF-8 reading fails for other reasons, also check if it's binary - const isBinary = await isBinaryFile(validPath); - if (isBinary) { - const instructions = getBinaryFileInstructions(validPath, mimeType); - return { content: instructions, mimeType: 'text/plain', isImage: false }; - } - - // Only if it's truly not binary, then we have a real UTF-8 reading error - throw error; - } - } + // Return with content as string + return { + content: result.content.toString(), + mimeType: result.mimeType, + metadata: result.metadata + }; }; + // Execute with timeout const result = await withTimeout( readOperation(), @@ -716,6 +707,7 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len `Read file operation for ${filePath}`, null ); + if (result == null) { // Handles the impossible case where withTimeout resolves to null instead of throwing throw new Error('Failed to read the file'); @@ -727,15 +719,17 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len /** * Read a file from either the local filesystem or a URL * @param filePath Path to the file or URL - * @param isUrl Whether the path is a URL - * @param offset Starting line number to read from (default: 0) - * @param length Maximum number of lines to read (default: from config or 1000) + * @param options Read options (isUrl, offset, length, sheet, range) * @returns File content or file result with metadata */ -export async function readFile(filePath: string, isUrl?: boolean, offset?: number, length?: number): Promise { +export async function readFile( + filePath: string, + options?: ReadOptions +): Promise { + const { isUrl, offset, length, sheet, range } = options ?? {}; return isUrl ? readFileFromUrl(filePath) - : readFileFromDisk(filePath, offset, length); + : readFileFromDisk(filePath, { offset, length, sheet, range }); } /** @@ -845,12 +839,11 @@ export async function writeFile(filePath: string, content: string, mode: 'rewrit lineCount: lineCount }); - // Use different fs methods based on mode - if (mode === 'append') { - await fs.appendFile(validPath, content); - } else { - await fs.writeFile(validPath, content); - } + // Get appropriate handler for this file type + const handler = getFileHandler(validPath); + + // Use handler to write the file + await handler.write(validPath, content, mode); } export interface MultiFileResult { @@ -870,9 +863,9 @@ export async function readMultipleFiles(paths: string[]): Promise> { const validPath = await validatePath(filePath); - const stats = await fs.stat(validPath); - // Basic file info + // Get appropriate handler for this file type + const handler = getFileHandler(validPath); + + // Use handler to get file info + const fileInfo = await handler.getInfo(validPath); + + // Convert to legacy format (for backward compatibility) const info: Record = { - size: stats.size, - created: stats.birthtime, - modified: stats.mtime, - accessed: stats.atime, - isDirectory: stats.isDirectory(), - isFile: stats.isFile(), - permissions: stats.mode.toString(8).slice(-3), + size: fileInfo.size, + created: fileInfo.created, + modified: fileInfo.modified, + accessed: fileInfo.accessed, + isDirectory: fileInfo.isDirectory, + isFile: fileInfo.isFile, + permissions: fileInfo.permissions, + fileType: fileInfo.fileType, }; - // For text files that aren't too large, also count lines - if (stats.isFile() && stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { - try { - // Get MIME type information - const { mimeType, isImage } = await getMimeTypeInfo(validPath); - - // Only count lines for non-image, likely text files - if (!isImage) { - const content = await fs.readFile(validPath, 'utf8'); - const lineCount = countLines(content); - info.lineCount = lineCount; - info.lastLine = lineCount - 1; // Zero-indexed last line - info.appendPosition = lineCount; // Position to append at end - } - } catch (error) { - // If reading fails, just skip the line count - // This could happen for binary files or very large files + // Add type-specific metadata + if (fileInfo.metadata) { + // For text files + if (fileInfo.metadata.lineCount !== undefined) { + info.lineCount = fileInfo.metadata.lineCount; + info.lastLine = fileInfo.metadata.lineCount - 1; + info.appendPosition = fileInfo.metadata.lineCount; + } + + // For Excel files + if (fileInfo.metadata.sheets) { + info.sheets = fileInfo.metadata.sheets; + info.isExcelFile = true; + } + + // For images + if (fileInfo.metadata.isImage) { + info.isImage = true; + } + + // For binary files + if (fileInfo.metadata.isBinary) { + info.isBinary = true; } } diff --git a/src/tools/schemas.ts b/src/tools/schemas.ts index c521d713..bf019162 100644 --- a/src/tools/schemas.ts +++ b/src/tools/schemas.ts @@ -47,6 +47,9 @@ export const ReadFileArgsSchema = z.object({ isUrl: z.boolean().optional().default(false), offset: z.number().optional().default(0), length: z.number().optional().default(1000), + sheet: z.union([z.string(), z.number()]).optional(), + range: z.string().optional(), + options: z.record(z.any()).optional() }); export const ReadMultipleFilesArgsSchema = z.object({ @@ -77,13 +80,26 @@ export const GetFileInfoArgsSchema = z.object({ path: z.string(), }); -// Edit tools schema +// Edit tools schema - SIMPLIFIED from three modes to two +// Previously supported: text replacement, location-based edits (edits array), and range rewrites +// Now supports only: text replacement and range rewrites +// Removed 'edits' array parameter - location-based surgical edits were complex and unnecessary +// Range rewrites are more powerful and cover all structured file editing needs export const EditBlockArgsSchema = z.object({ file_path: z.string(), - old_string: z.string(), - new_string: z.string(), + // Text file string replacement + old_string: z.string().optional(), + new_string: z.string().optional(), expected_replacements: z.number().optional().default(1), -}); + // Structured file range rewrite (Excel, etc.) + range: z.string().optional(), + content: z.any().optional(), + options: z.record(z.any()).optional() +}).refine( + data => (data.old_string !== undefined && data.new_string !== undefined) || + (data.range !== undefined && data.content !== undefined), + { message: "Must provide either (old_string + new_string) or (range + content)" } +); // Send input to process schema export const InteractWithProcessArgsSchema = z.object({ @@ -146,4 +162,10 @@ export const GetRecentToolCallsArgsSchema = z.object({ maxResults: z.number().min(1).max(1000).optional().default(50), toolName: z.string().optional(), since: z.string().datetime().optional(), +}); + +// Execute Node.js code schema +export const ExecuteNodeArgsSchema = z.object({ + code: z.string(), + timeout_ms: z.number().optional().default(30000), }); \ No newline at end of file diff --git a/src/utils/files/base.ts b/src/utils/files/base.ts new file mode 100644 index 00000000..a36d8b2e --- /dev/null +++ b/src/utils/files/base.ts @@ -0,0 +1,200 @@ +/** + * Base interfaces and types for file handling system + * All file handlers implement the FileHandler interface + */ + +// ============================================================================ +// Core Interfaces +// ============================================================================ + +/** + * Base interface that all file handlers must implement + */ +export interface FileHandler { + /** + * Read file content + * @param path Validated file path + * @param options Read options (offset, length, sheet, etc.) + * @returns File result with content and metadata + */ + read(path: string, options?: ReadOptions): Promise; + + /** + * Write file (complete rewrite or append) + * @param path Validated file path + * @param content Content to write + * @param mode Write mode: 'rewrite' (default) or 'append' + */ + write(path: string, content: any, mode?: 'rewrite' | 'append'): Promise; + + /** + * Edit a specific range (bulk rewrite) + * PRIMARY METHOD for structured file editing (Excel, etc.) + * Simpler and more powerful than location-based edits + * Supports: + * - Cell ranges: "Sheet1!A1:C10" with 2D array content + * - Whole sheets: "Sheet1" to replace entire sheet + * - Chunking: Update 1000 rows at a time for large files + * + * Currently implemented by: ExcelFileHandler + * TECHNICAL DEBT: TextFileHandler should also implement this for search/replace + * (logic currently in src/tools/edit.ts - see comments there) + * + * @param path Validated file path + * @param range Range identifier (e.g., "Sheet1!A1:C10" or "Sheet1") + * @param content New content for the range (2D array for Excel) + * @param options Additional format-specific options + * @returns Result with success status + */ + editRange?(path: string, range: string, content: any, options?: Record): Promise; + + /** + * Get file metadata + * @param path Validated file path + * @returns File information including type-specific metadata + */ + getInfo(path: string): Promise; + + /** + * Check if this handler can handle the given file + * @param path File path + * @returns true if this handler supports this file type + */ + canHandle(path: string): boolean; +} + +// ============================================================================ +// Read Operations +// ============================================================================ + +/** + * Options for reading files + */ +export interface ReadOptions { + /** Whether the path is a URL */ + isUrl?: boolean; + + /** Starting line/row number (for text/excel) */ + offset?: number; + + /** Maximum number of lines/rows to read */ + length?: number; + + /** Excel-specific: Sheet name or index */ + sheet?: string | number; + + /** Excel-specific: Cell range (e.g., "A1:C10") */ + range?: string; + + /** Whether to include status messages (default: true) */ + includeStatusMessage?: boolean; +} + +/** + * Result from reading a file + */ +export interface FileResult { + /** File content (string for text/csv, Buffer for binary, base64 string for images) */ + content: string | Buffer; + + /** MIME type of the content */ + mimeType: string; + + /** Type-specific metadata */ + metadata?: FileMetadata; +} + +/** + * File-type specific metadata + */ +export interface FileMetadata { + /** For images */ + isImage?: boolean; + + /** For binary files */ + isBinary?: boolean; + + /** For Excel files */ + isExcelFile?: boolean; + sheets?: ExcelSheet[]; + fileSize?: number; + isLargeFile?: boolean; + + /** For text files */ + lineCount?: number; + + /** Error information if operation failed */ + error?: boolean; + errorMessage?: string; +} + +/** + * Excel sheet metadata + */ +export interface ExcelSheet { + /** Sheet name */ + name: string; + + /** Number of rows in sheet */ + rowCount: number; + + /** Number of columns in sheet */ + colCount: number; +} + +// ============================================================================ +// Edit Operations +// ============================================================================ + +/** + * Result from edit operation (used by editRange) + */ +export interface EditResult { + /** Whether all edits succeeded */ + success: boolean; + + /** Number of edits successfully applied */ + editsApplied: number; + + /** Errors that occurred during editing */ + errors?: Array<{ + location: string; + error: string; + }>; +} + +// ============================================================================ +// File Information +// ============================================================================ + +/** + * File information and metadata + */ +export interface FileInfo { + /** File size in bytes */ + size: number; + + /** Creation time */ + created: Date; + + /** Last modification time */ + modified: Date; + + /** Last access time */ + accessed: Date; + + /** Is this a directory */ + isDirectory: boolean; + + /** Is this a regular file */ + isFile: boolean; + + /** File permissions (octal string) */ + permissions: string; + + /** File type classification */ + fileType: 'text' | 'excel' | 'image' | 'binary'; + + /** Type-specific metadata */ + metadata?: FileMetadata; +} diff --git a/src/utils/files/binary.ts b/src/utils/files/binary.ts new file mode 100644 index 00000000..a934e833 --- /dev/null +++ b/src/utils/files/binary.ts @@ -0,0 +1,173 @@ +/** + * Binary file handler + * Catch-all handler for unsupported binary files + * Returns instructions to use start_process with appropriate tools + */ + +import fs from "fs/promises"; +import path from "path"; +import { + FileHandler, + ReadOptions, + FileResult, + FileInfo +} from './base.js'; + +/** + * Binary file handler implementation + * This is a catch-all handler for binary files that aren't supported by other handlers + */ +export class BinaryFileHandler implements FileHandler { + canHandle(path: string): boolean { + // Binary handler is the catch-all - handles everything not handled by other handlers + return true; + } + + async read(filePath: string, options?: ReadOptions): Promise { + const instructions = this.getBinaryInstructions(filePath); + + return { + content: instructions, + mimeType: 'text/plain', + metadata: { + isBinary: true + } + }; + } + + async write(path: string, content: any): Promise { + throw new Error('Cannot write binary files directly. Use start_process with appropriate tools (Python, Node.js libraries, command-line utilities).'); + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'binary', + metadata: { + isBinary: true + } + }; + } + + /** + * Generate instructions for handling binary files + */ + private getBinaryInstructions(filePath: string): string { + const fileName = path.basename(filePath); + const ext = path.extname(filePath).toLowerCase(); + + // Get MIME type suggestion based on extension + const mimeType = this.guessMimeType(ext); + + let specificGuidance = ''; + + // Provide specific guidance based on file type + switch (ext) { + case '.pdf': + specificGuidance = ` +PDF FILES: +- Python: PyPDF2, pdfplumber + start_process("python -i") + interact_with_process(pid, "import pdfplumber") + interact_with_process(pid, "pdf = pdfplumber.open('${filePath}')") + interact_with_process(pid, "print(pdf.pages[0].extract_text())") + +- Node.js: pdf-parse + start_process("node -i") + interact_with_process(pid, "const pdf = require('pdf-parse')")`; + break; + + case '.doc': + case '.docx': + specificGuidance = ` +WORD DOCUMENTS: +- Python: python-docx + start_process("python -i") + interact_with_process(pid, "import docx") + interact_with_process(pid, "doc = docx.Document('${filePath}')") + interact_with_process(pid, "for para in doc.paragraphs: print(para.text)") + +- Node.js: mammoth + start_process("node -i") + interact_with_process(pid, "const mammoth = require('mammoth')")`; + break; + + case '.zip': + case '.tar': + case '.gz': + specificGuidance = ` +ARCHIVE FILES: +- Python: zipfile, tarfile + start_process("python -i") + interact_with_process(pid, "import zipfile") + interact_with_process(pid, "with zipfile.ZipFile('${filePath}') as z: print(z.namelist())") + +- Command-line: + start_process("unzip -l ${filePath}") # For ZIP files + start_process("tar -tzf ${filePath}") # For TAR files`; + break; + + case '.db': + case '.sqlite': + case '.sqlite3': + specificGuidance = ` +SQLITE DATABASES: +- Python: sqlite3 + start_process("python -i") + interact_with_process(pid, "import sqlite3") + interact_with_process(pid, "conn = sqlite3.connect('${filePath}')") + interact_with_process(pid, "cursor = conn.cursor()") + interact_with_process(pid, "cursor.execute('SELECT * FROM sqlite_master')") + +- Command-line: + start_process("sqlite3 ${filePath} '.tables'")`; + break; + + default: + specificGuidance = ` +GENERIC BINARY FILES: +- Use appropriate libraries based on file type +- Python libraries: Check PyPI for ${ext} support +- Node.js libraries: Check npm for ${ext} support +- Command-line tools: Use file-specific utilities`; + } + + return `Cannot read binary file as text: ${fileName} (${mimeType}) + +Use start_process + interact_with_process to analyze binary files with appropriate tools. +${specificGuidance} + +The read_file tool only handles text files, images, and Excel files.`; + } + + /** + * Guess MIME type based on file extension + */ + private guessMimeType(ext: string): string { + const mimeTypes: { [key: string]: string } = { + '.pdf': 'application/pdf', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.zip': 'application/zip', + '.tar': 'application/x-tar', + '.gz': 'application/gzip', + '.db': 'application/x-sqlite3', + '.sqlite': 'application/x-sqlite3', + '.sqlite3': 'application/x-sqlite3', + '.mp3': 'audio/mpeg', + '.mp4': 'video/mp4', + '.avi': 'video/x-msvideo', + '.mkv': 'video/x-matroska', + }; + + return mimeTypes[ext] || 'application/octet-stream'; + } +} diff --git a/src/utils/files/excel.ts b/src/utils/files/excel.ts new file mode 100644 index 00000000..5c0a9df1 --- /dev/null +++ b/src/utils/files/excel.ts @@ -0,0 +1,486 @@ +/** + * Excel file handler using ExcelJS + * Handles reading, writing, and editing Excel files (.xlsx, .xls, .xlsm) + */ + +import ExcelJS from 'exceljs'; +import fs from 'fs/promises'; +import { + FileHandler, + ReadOptions, + FileResult, + EditResult, + FileInfo, + ExcelSheet +} from './base.js'; + +// File size limit: 10MB +const FILE_SIZE_LIMIT = 10 * 1024 * 1024; + +/** + * Excel file metadata (internal use only) + */ +interface ExcelMetadata { + sheets: ExcelSheet[]; + fileSize: number; + isLargeFile: boolean; +} + +/** + * Excel file handler implementation using ExcelJS + * Supports: .xlsx, .xls, .xlsm files + */ +export class ExcelFileHandler implements FileHandler { + + canHandle(path: string): boolean { + const ext = path.toLowerCase(); + return ext.endsWith('.xlsx') || ext.endsWith('.xls') || ext.endsWith('.xlsm'); + } + + async read(path: string, options?: ReadOptions): Promise { + await this.checkFileSize(path); + + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + + const metadata = await this.extractMetadata(workbook, path); + const { sheetName, data, totalRows, returnedRows } = this.worksheetToArray( + workbook, + options?.sheet, + options?.range, + options?.offset, + options?.length + ); + + // Format output with sheet info header, usage hint, and JSON data + const paginationInfo = totalRows > returnedRows + ? `\n[Showing rows ${(options?.offset || 0) + 1}-${(options?.offset || 0) + returnedRows} of ${totalRows} total. Use offset/length to paginate.]` + : ''; + + const content = `[Sheet: '${sheetName}' from ${path}]${paginationInfo} +[To MODIFY cells: use edit_block with range param, e.g., edit_block(path, {range: "Sheet1!E5", content: [[newValue]]})] + +${JSON.stringify(data)}`; + + return { + content, + mimeType: 'application/json', + metadata: { + isExcelFile: true, + sheets: metadata.sheets, + fileSize: metadata.fileSize, + isLargeFile: metadata.isLargeFile + } + }; + } + + async write(path: string, content: any, mode?: 'rewrite' | 'append'): Promise { + // Check existing file size if it exists + try { + await this.checkFileSize(path); + } catch (error) { + // File doesn't exist - that's fine for write + if ((error as any).code !== 'ENOENT' && + !(error instanceof Error && error.message.includes('ENOENT'))) { + throw error; + } + } + + // Parse content + let parsedContent = content; + if (typeof content === 'string') { + try { + parsedContent = JSON.parse(content); + } catch { + throw new Error('Invalid content format. Expected JSON string with 2D array or object with sheet names.'); + } + } + + // Handle append mode by finding last row and writing after it + if (mode === 'append') { + try { + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + + if (Array.isArray(parsedContent)) { + // Append to Sheet1 + let worksheet = workbook.getWorksheet('Sheet1'); + if (!worksheet) { + worksheet = workbook.addWorksheet('Sheet1'); + } + const startRow = (worksheet.actualRowCount || 0) + 1; + this.writeRowsStartingAt(worksheet, startRow, parsedContent); + } else if (typeof parsedContent === 'object' && parsedContent !== null) { + // Append to each named sheet + for (const [sheetName, data] of Object.entries(parsedContent)) { + if (Array.isArray(data)) { + let worksheet = workbook.getWorksheet(sheetName); + if (!worksheet) { + worksheet = workbook.addWorksheet(sheetName); + } + const startRow = (worksheet.actualRowCount || 0) + 1; + this.writeRowsStartingAt(worksheet, startRow, data as any[][]); + } + } + } + + await workbook.xlsx.writeFile(path); + return; + } catch (error) { + // File doesn't exist - fall through to create new file + if ((error as any).code !== 'ENOENT' && + !(error instanceof Error && error.message.includes('ENOENT'))) { + throw error; + } + } + } + + // Rewrite mode (or append to non-existent file): create new workbook + const workbook = new ExcelJS.Workbook(); + + if (Array.isArray(parsedContent)) { + // Single sheet from 2D array + this.writeDataToSheet(workbook, 'Sheet1', parsedContent); + } else if (typeof parsedContent === 'object' && parsedContent !== null) { + // Object with sheet names as keys + for (const [sheetName, data] of Object.entries(parsedContent)) { + if (Array.isArray(data)) { + this.writeDataToSheet(workbook, sheetName, data as any[][]); + } + } + } else { + throw new Error('Invalid content format. Expected 2D array or object with sheet names.'); + } + + await workbook.xlsx.writeFile(path); + } + + async editRange(path: string, range: string, content: any, options?: Record): Promise { + // Verify file exists and check size + try { + await this.checkFileSize(path); + } catch (error) { + if ((error as any).code === 'ENOENT' || + (error instanceof Error && error.message.includes('ENOENT'))) { + throw new Error(`File not found: ${path}`); + } + throw error; + } + + // Validate content + if (!Array.isArray(content)) { + throw new Error('Content must be a 2D array for range editing'); + } + + // Parse range: "Sheet1!A1:C10" or "Sheet1" + const [sheetName, cellRange] = this.parseRange(range); + + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + + // Get or create sheet + let worksheet = workbook.getWorksheet(sheetName); + if (!worksheet) { + worksheet = workbook.addWorksheet(sheetName); + } + + if (cellRange) { + // Write to specific range + const { startRow, startCol } = this.parseCellRange(cellRange); + + for (let r = 0; r < content.length; r++) { + const rowData = content[r]; + if (!Array.isArray(rowData)) continue; + + for (let c = 0; c < rowData.length; c++) { + const cell = worksheet.getCell(startRow + r, startCol + c); + const value = rowData[c]; + + if (typeof value === 'string' && value.startsWith('=')) { + cell.value = { formula: value.substring(1) }; + } else { + cell.value = value; + } + } + } + } else { + // Replace entire sheet content + // Clear existing data + worksheet.eachRow((row, rowNumber) => { + row.eachCell((cell) => { + cell.value = null; + }); + }); + + // Write new data + for (let r = 0; r < content.length; r++) { + const rowData = content[r]; + if (!Array.isArray(rowData)) continue; + + const row = worksheet.getRow(r + 1); + for (let c = 0; c < rowData.length; c++) { + const value = rowData[c]; + if (typeof value === 'string' && value.startsWith('=')) { + row.getCell(c + 1).value = { formula: value.substring(1) }; + } else { + row.getCell(c + 1).value = value; + } + } + row.commit(); + } + } + + await workbook.xlsx.writeFile(path); + + return { success: true, editsApplied: 1 }; + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + try { + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + const metadata = await this.extractMetadata(workbook, path); + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'excel', + metadata: { + isExcelFile: true, + sheets: metadata.sheets, + fileSize: metadata.fileSize, + isLargeFile: metadata.isLargeFile + } + }; + } catch (error) { + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'excel', + metadata: { + isExcelFile: true, + fileSize: stats.size, + error: true, + errorMessage: error instanceof Error ? error.message : String(error) + } + }; + } + } + + // ========== Private Helpers ========== + + private async checkFileSize(path: string): Promise { + const stats = await fs.stat(path); + if (stats.size > FILE_SIZE_LIMIT) { + const sizeMB = (stats.size / 1024 / 1024).toFixed(1); + throw new Error( + `Excel file size (${sizeMB}MB) exceeds 10MB limit. ` + + `Consider using specialized tools for large file processing.` + ); + } + } + + private async extractMetadata(workbook: ExcelJS.Workbook, path: string): Promise { + const stats = await fs.stat(path); + + const sheets: ExcelSheet[] = workbook.worksheets.map(ws => ({ + name: ws.name, + rowCount: ws.actualRowCount || 0, + colCount: ws.actualColumnCount || 0 + })); + + return { + sheets, + fileSize: stats.size, + isLargeFile: stats.size > FILE_SIZE_LIMIT + }; + } + + private worksheetToArray( + workbook: ExcelJS.Workbook, + sheetRef?: string | number, + range?: string, + offset?: number, + length?: number + ): { sheetName: string; data: any[][]; totalRows: number; returnedRows: number } { + if (workbook.worksheets.length === 0) { + return { sheetName: '', data: [], totalRows: 0, returnedRows: 0 }; + } + + // Find target worksheet + let worksheet: ExcelJS.Worksheet | undefined; + let sheetName: string; + + if (sheetRef === undefined) { + worksheet = workbook.worksheets[0]; + sheetName = worksheet.name; + } else if (typeof sheetRef === 'number') { + if (sheetRef < 0 || sheetRef >= workbook.worksheets.length) { + throw new Error(`Sheet index ${sheetRef} out of range (0-${workbook.worksheets.length - 1})`); + } + worksheet = workbook.worksheets[sheetRef]; + sheetName = worksheet.name; + } else { + worksheet = workbook.getWorksheet(sheetRef); + if (!worksheet) { + const available = workbook.worksheets.map(ws => ws.name).join(', '); + throw new Error(`Sheet "${sheetRef}" not found. Available sheets: ${available}`); + } + sheetName = sheetRef; + } + + // Determine range to read + let startRow = 1; + let endRow = worksheet.actualRowCount || 1; + let startCol = 1; + let endCol = worksheet.actualColumnCount || 1; + + if (range) { + const parsed = this.parseCellRange(range); + startRow = parsed.startRow; + startCol = parsed.startCol; + if (parsed.endRow) endRow = parsed.endRow; + if (parsed.endCol) endCol = parsed.endCol; + } + + // Calculate total rows before pagination + const totalRows = endRow - startRow + 1; + + // Apply offset/length pagination (row-based, matching text file behavior) + if (offset !== undefined) { + if (offset < 0) { + // Negative offset: last N rows (like text files) + // offset: -10 means "last 10 rows" + const lastNRows = Math.abs(offset); + startRow = Math.max(startRow, endRow - lastNRows + 1); + } else if (offset > 0) { + // Positive offset: skip first N rows + startRow = startRow + offset; + } + } + + // Apply length limit (only for positive offset or no offset) + if (length !== undefined && length > 0 && (offset === undefined || offset >= 0)) { + endRow = Math.min(endRow, startRow + length - 1); + } + + // Ensure valid range + if (startRow > endRow) { + return { sheetName, data: [], totalRows, returnedRows: 0 }; + } + + // Build 2D array (preserving types) + const data: any[][] = []; + for (let r = startRow; r <= endRow; r++) { + const row = worksheet.getRow(r); + const rowData: any[] = []; + + for (let c = startCol; c <= endCol; c++) { + const cell = row.getCell(c); + let value: any = null; + + if (cell.value !== null && cell.value !== undefined) { + if (typeof cell.value === 'object') { + // Handle formula results, rich text, etc. + if ('result' in cell.value) { + value = cell.value.result ?? null; + } else if ('richText' in cell.value) { + value = (cell.value as any).richText.map((rt: any) => rt.text).join(''); + } else if ('text' in cell.value) { + value = (cell.value as any).text; + } else if (cell.value instanceof Date) { + value = cell.value.toISOString(); + } else { + value = String(cell.value); + } + } else { + // Preserve native types (string, number, boolean) + value = cell.value; + } + } + + rowData.push(value); + } + data.push(rowData); + } + + return { sheetName, data, totalRows, returnedRows: data.length }; + } + + private writeDataToSheet(workbook: ExcelJS.Workbook, sheetName: string, data: any[][]): void { + // Remove existing sheet if it exists + const existing = workbook.getWorksheet(sheetName); + if (existing) { + workbook.removeWorksheet(existing.id); + } + + const worksheet = workbook.addWorksheet(sheetName); + this.writeRowsStartingAt(worksheet, 1, data); + } + + private writeRowsStartingAt(worksheet: ExcelJS.Worksheet, startRow: number, data: any[][]): void { + for (let r = 0; r < data.length; r++) { + const rowData = data[r]; + if (!Array.isArray(rowData)) continue; + + const row = worksheet.getRow(startRow + r); + for (let c = 0; c < rowData.length; c++) { + const value = rowData[c]; + if (typeof value === 'string' && value.startsWith('=')) { + row.getCell(c + 1).value = { formula: value.substring(1) }; + } else { + row.getCell(c + 1).value = value; + } + } + row.commit(); + } + } + + private parseRange(range: string): [string, string | null] { + if (range.includes('!')) { + const [sheetName, cellRange] = range.split('!'); + return [sheetName, cellRange]; + } + return [range, null]; + } + + private parseCellRange(range: string): { startRow: number; startCol: number; endRow?: number; endCol?: number } { + // Parse A1 or A1:C10 format + const match = range.match(/^([A-Z]+)(\d+)(?::([A-Z]+)(\d+))?$/i); + if (!match) { + throw new Error(`Invalid cell range: ${range}`); + } + + const startCol = this.columnToNumber(match[1]); + const startRow = parseInt(match[2], 10); + + if (match[3] && match[4]) { + const endCol = this.columnToNumber(match[3]); + const endRow = parseInt(match[4], 10); + return { startRow, startCol, endRow, endCol }; + } + + return { startRow, startCol }; + } + + private columnToNumber(col: string): number { + let result = 0; + for (let i = 0; i < col.length; i++) { + result = result * 26 + col.charCodeAt(i) - 64; + } + return result; + } +} + diff --git a/src/utils/files/factory.ts b/src/utils/files/factory.ts new file mode 100644 index 00000000..bf84ab9a --- /dev/null +++ b/src/utils/files/factory.ts @@ -0,0 +1,89 @@ +/** + * Factory pattern for creating appropriate file handlers + * Routes file operations to the correct handler based on file type + */ + +import { FileHandler } from './base.js'; +import { TextFileHandler } from './text.js'; +import { ImageFileHandler } from './image.js'; +import { BinaryFileHandler } from './binary.js'; +import { ExcelFileHandler } from './excel.js'; + +// Singleton instances of each handler +let handlers: FileHandler[] | null = null; + +/** + * Initialize handlers (lazy initialization) + */ +function initializeHandlers(): FileHandler[] { + if (handlers) { + return handlers; + } + + handlers = [ + // Order matters! More specific handlers first + new ExcelFileHandler(), // Check Excel first (before binary) + new ImageFileHandler(), // Then images + new TextFileHandler(), // Then text (handles most files) + new BinaryFileHandler(), // Finally binary (catch-all) + ]; + + return handlers; +} + +/** + * Get the appropriate file handler for a given file path + * + * This function checks each handler in priority order and returns the first + * handler that can handle the file type. + * + * Priority order: + * 1. Excel files (xlsx, xls, xlsm) + * 2. Image files (png, jpg, gif, webp) + * 3. Text files (most other files) + * 4. Binary files (catch-all for unsupported formats) + * + * @param path File path (can be before or after validation) + * @returns FileHandler instance that can handle this file + */ +export function getFileHandler(path: string): FileHandler { + const allHandlers = initializeHandlers(); + + // Try each handler in order + for (const handler of allHandlers) { + if (handler.canHandle(path)) { + return handler; + } + } + + // Fallback to binary handler (should never reach here due to binary catch-all) + return allHandlers[allHandlers.length - 1]; +} + +/** + * Check if a file path is an Excel file + * @param path File path + * @returns true if file is Excel format + */ +export function isExcelFile(path: string): boolean { + const ext = path.toLowerCase(); + return ext.endsWith('.xlsx') || ext.endsWith('.xls') || ext.endsWith('.xlsm'); +} + +/** + * Check if a file path is an image file + * @param path File path + * @returns true if file is an image format + */ +export function isImageFile(path: string): boolean { + // This will be implemented by checking MIME type + // For now, use extension-based check + const ext = path.toLowerCase(); + return ext.endsWith('.png') || + ext.endsWith('.jpg') || + ext.endsWith('.jpeg') || + ext.endsWith('.gif') || + ext.endsWith('.webp') || + ext.endsWith('.bmp') || + ext.endsWith('.svg'); +} diff --git a/src/utils/files/image.ts b/src/utils/files/image.ts new file mode 100644 index 00000000..4e97ce9b --- /dev/null +++ b/src/utils/files/image.ts @@ -0,0 +1,93 @@ +/** + * Image file handler + * Handles reading image files and converting to base64 + */ + +import fs from "fs/promises"; +import { + FileHandler, + ReadOptions, + FileResult, + FileInfo +} from './base.js'; + +/** + * Image file handler implementation + * Supports: PNG, JPEG, GIF, WebP, BMP, SVG + */ +export class ImageFileHandler implements FileHandler { + private static readonly IMAGE_EXTENSIONS = [ + '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg' + ]; + + private static readonly IMAGE_MIME_TYPES: { [key: string]: string } = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.bmp': 'image/bmp', + '.svg': 'image/svg+xml' + }; + + canHandle(path: string): boolean { + const lowerPath = path.toLowerCase(); + return ImageFileHandler.IMAGE_EXTENSIONS.some(ext => lowerPath.endsWith(ext)); + } + + async read(path: string, options?: ReadOptions): Promise { + // Images are always read in full, ignoring offset and length + const buffer = await fs.readFile(path); + const content = buffer.toString('base64'); + const mimeType = this.getMimeType(path); + + return { + content, + mimeType, + metadata: { + isImage: true + } + }; + } + + async write(path: string, content: Buffer | string): Promise { + // If content is base64 string, convert to buffer + if (typeof content === 'string') { + const buffer = Buffer.from(content, 'base64'); + await fs.writeFile(path, buffer); + } else { + await fs.writeFile(path, content); + } + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'image', + metadata: { + isImage: true + } + }; + } + + /** + * Get MIME type for image based on file extension + */ + private getMimeType(path: string): string { + const lowerPath = path.toLowerCase(); + for (const [ext, mimeType] of Object.entries(ImageFileHandler.IMAGE_MIME_TYPES)) { + if (lowerPath.endsWith(ext)) { + return mimeType; + } + } + return 'application/octet-stream'; // Fallback + } +} diff --git a/src/utils/files/index.ts b/src/utils/files/index.ts new file mode 100644 index 00000000..c35e0642 --- /dev/null +++ b/src/utils/files/index.ts @@ -0,0 +1,16 @@ +/** + * File handling system + * Exports all file handlers, interfaces, and utilities + */ + +// Base interfaces and types +export * from './base.js'; + +// Factory function +export { getFileHandler, isExcelFile, isImageFile } from './factory.js'; + +// File handlers +export { TextFileHandler } from './text.js'; +export { ImageFileHandler } from './image.js'; +export { BinaryFileHandler } from './binary.js'; +export { ExcelFileHandler } from './excel.js'; diff --git a/src/utils/files/text.ts b/src/utils/files/text.ts new file mode 100644 index 00000000..14b5b844 --- /dev/null +++ b/src/utils/files/text.ts @@ -0,0 +1,441 @@ +/** + * Text file handler + * Handles reading, writing, and editing text files + * + * TECHNICAL DEBT: + * This handler is missing editRange() - text search/replace logic currently lives in + * src/tools/edit.ts (performSearchReplace function) instead of here. + * + * For architectural consistency with ExcelFileHandler.editRange(), the fuzzy + * search/replace logic should be moved here. See comment in src/tools/edit.ts. + */ + +import fs from "fs/promises"; +import { createReadStream } from 'fs'; +import { createInterface } from 'readline'; +import { isBinaryFile } from 'isbinaryfile'; +import { + FileHandler, + ReadOptions, + FileResult, + FileInfo +} from './base.js'; + +// Import constants from filesystem.ts +// These will be imported after we organize the code +const FILE_SIZE_LIMITS = { + LARGE_FILE_THRESHOLD: 10 * 1024 * 1024, // 10MB + LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting +} as const; + +const READ_PERFORMANCE_THRESHOLDS = { + SMALL_READ_THRESHOLD: 100, // For very small reads + DEEP_OFFSET_THRESHOLD: 1000, // For byte estimation + SAMPLE_SIZE: 10000, // Sample size for estimation + CHUNK_SIZE: 8192, // 8KB chunks for reverse reading +} as const; + +/** + * Text file handler implementation + */ +export class TextFileHandler implements FileHandler { + canHandle(path: string): boolean { + // Text handler is the default - handles most files + // Only returns false for known non-text formats (checked by other handlers) + return true; + } + + async read(path: string, options?: ReadOptions): Promise { + const offset = options?.offset ?? 0; + const length = options?.length ?? 1000; // Default from config + const includeStatusMessage = options?.includeStatusMessage ?? true; + + // Check if file is binary + const isBinary = await isBinaryFile(path); + if (isBinary) { + throw new Error('Cannot read binary file as text. Use appropriate handler.'); + } + + // Read with smart positioning + return this.readFileWithSmartPositioning(path, offset, length, 'text/plain', includeStatusMessage); + } + + async write(path: string, content: string, mode: 'rewrite' | 'append' = 'rewrite'): Promise { + if (mode === 'append') { + await fs.appendFile(path, content); + } else { + await fs.writeFile(path, content); + } + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + const info: FileInfo = { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'text', + metadata: {} + }; + + // For text files that aren't too large, count lines + if (stats.isFile() && stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { + try { + const content = await fs.readFile(path, 'utf8'); + const lineCount = this.countLines(content); + info.metadata!.lineCount = lineCount; + } catch (error) { + // If reading fails, skip line count + } + } + + return info; + } + + // ======================================================================== + // Private Helper Methods (extracted from filesystem.ts) + // ======================================================================== + + /** + * Count lines in text content + */ + private countLines(content: string): number { + return content.split('\n').length; + } + + /** + * Get file line count (for files under size limit) + */ + private async getFileLineCount(filePath: string): Promise { + try { + const stats = await fs.stat(filePath); + if (stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { + const content = await fs.readFile(filePath, 'utf8'); + return this.countLines(content); + } + } catch (error) { + // If we can't read the file, return undefined + } + return undefined; + } + + /** + * Generate enhanced status message + */ + private generateEnhancedStatusMessage( + readLines: number, + offset: number, + totalLines?: number, + isNegativeOffset: boolean = false + ): string { + if (isNegativeOffset) { + if (totalLines !== undefined) { + return `[Reading last ${readLines} lines (total: ${totalLines} lines)]`; + } else { + return `[Reading last ${readLines} lines]`; + } + } else { + if (totalLines !== undefined) { + const endLine = offset + readLines; + const remainingLines = Math.max(0, totalLines - endLine); + + if (offset === 0) { + return `[Reading ${readLines} lines from start (total: ${totalLines} lines, ${remainingLines} remaining)]`; + } else { + return `[Reading ${readLines} lines from line ${offset} (total: ${totalLines} lines, ${remainingLines} remaining)]`; + } + } else { + if (offset === 0) { + return `[Reading ${readLines} lines from start]`; + } else { + return `[Reading ${readLines} lines from line ${offset}]`; + } + } + } + } + + /** + * Split text into lines while preserving line endings + */ + private splitLinesPreservingEndings(content: string): string[] { + if (!content) return ['']; + + const lines: string[] = []; + let currentLine = ''; + + for (let i = 0; i < content.length; i++) { + const char = content[i]; + currentLine += char; + + if (char === '\n') { + lines.push(currentLine); + currentLine = ''; + } else if (char === '\r') { + if (i + 1 < content.length && content[i + 1] === '\n') { + currentLine += content[i + 1]; + i++; + } + lines.push(currentLine); + currentLine = ''; + } + } + + if (currentLine) { + lines.push(currentLine); + } + + return lines; + } + + /** + * Read file with smart positioning for optimal performance + */ + private async readFileWithSmartPositioning( + filePath: string, + offset: number, + length: number, + mimeType: string, + includeStatusMessage: boolean = true + ): Promise { + const stats = await fs.stat(filePath); + const fileSize = stats.size; + + const totalLines = await this.getFileLineCount(filePath); + + // For negative offsets (tail behavior), use reverse reading + if (offset < 0) { + const requestedLines = Math.abs(offset); + + if (fileSize > FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD && + requestedLines <= READ_PERFORMANCE_THRESHOLDS.SMALL_READ_THRESHOLD) { + return await this.readLastNLinesReverse(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); + } else { + return await this.readFromEndWithReadline(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); + } + } + // For positive offsets + else { + if (fileSize < FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD || offset === 0) { + return await this.readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); + } else { + if (offset > READ_PERFORMANCE_THRESHOLDS.DEEP_OFFSET_THRESHOLD) { + return await this.readFromEstimatedPosition(filePath, offset, length, mimeType, includeStatusMessage, totalLines); + } else { + return await this.readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); + } + } + } + } + + /** + * Read last N lines efficiently by reading file backwards + */ + private async readLastNLinesReverse( + filePath: string, + n: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + const fd = await fs.open(filePath, 'r'); + try { + const stats = await fd.stat(); + const fileSize = stats.size; + + let position = fileSize; + let lines: string[] = []; + let partialLine = ''; + + while (position > 0 && lines.length < n) { + const readSize = Math.min(READ_PERFORMANCE_THRESHOLDS.CHUNK_SIZE, position); + position -= readSize; + + const buffer = Buffer.alloc(readSize); + await fd.read(buffer, 0, readSize, position); + + const chunk = buffer.toString('utf-8'); + const text = chunk + partialLine; + const chunkLines = text.split('\n'); + + partialLine = chunkLines.shift() || ''; + lines = chunkLines.concat(lines); + } + + if (position === 0 && partialLine) { + lines.unshift(partialLine); + } + + const result = lines.slice(-n); + const content = includeStatusMessage + ? `${this.generateEnhancedStatusMessage(result.length, -n, fileTotalLines, true)}\n\n${result.join('\n')}` + : result.join('\n'); + + return { content, mimeType, metadata: {} }; + } finally { + await fd.close(); + } + } + + /** + * Read from end using readline with circular buffer + */ + private async readFromEndWithReadline( + filePath: string, + requestedLines: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity + }); + + const buffer: string[] = new Array(requestedLines); + let bufferIndex = 0; + let totalLines = 0; + + for await (const line of rl) { + buffer[bufferIndex] = line; + bufferIndex = (bufferIndex + 1) % requestedLines; + totalLines++; + } + + rl.close(); + + let result: string[]; + if (totalLines >= requestedLines) { + result = [ + ...buffer.slice(bufferIndex), + ...buffer.slice(0, bufferIndex) + ].filter(line => line !== undefined); + } else { + result = buffer.slice(0, totalLines); + } + + const content = includeStatusMessage + ? `${this.generateEnhancedStatusMessage(result.length, -requestedLines, fileTotalLines, true)}\n\n${result.join('\n')}` + : result.join('\n'); + + return { content, mimeType, metadata: {} }; + } + + /** + * Read from start/middle using readline + */ + private async readFromStartWithReadline( + filePath: string, + offset: number, + length: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity + }); + + const result: string[] = []; + let lineNumber = 0; + + for await (const line of rl) { + if (lineNumber >= offset && result.length < length) { + result.push(line); + } + if (result.length >= length) break; + lineNumber++; + } + + rl.close(); + + if (includeStatusMessage) { + const statusMessage = this.generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false); + const content = `${statusMessage}\n\n${result.join('\n')}`; + return { content, mimeType, metadata: {} }; + } else { + const content = result.join('\n'); + return { content, mimeType, metadata: {} }; + } + } + + /** + * Read from estimated byte position for very large files + */ + private async readFromEstimatedPosition( + filePath: string, + offset: number, + length: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + // First, do a quick scan to estimate lines per byte + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity + }); + + let sampleLines = 0; + let bytesRead = 0; + + for await (const line of rl) { + bytesRead += Buffer.byteLength(line, 'utf-8') + 1; + sampleLines++; + if (bytesRead >= READ_PERFORMANCE_THRESHOLDS.SAMPLE_SIZE) break; + } + + rl.close(); + + if (sampleLines === 0) { + return await this.readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, fileTotalLines); + } + + // Estimate position + const avgLineLength = bytesRead / sampleLines; + const estimatedBytePosition = Math.floor(offset * avgLineLength); + + const fd = await fs.open(filePath, 'r'); + try { + const stats = await fd.stat(); + const startPosition = Math.min(estimatedBytePosition, stats.size); + + const stream = createReadStream(filePath, { start: startPosition }); + const rl2 = createInterface({ + input: stream, + crlfDelay: Infinity + }); + + const result: string[] = []; + let firstLineSkipped = false; + + for await (const line of rl2) { + if (!firstLineSkipped && startPosition > 0) { + firstLineSkipped = true; + continue; + } + + if (result.length < length) { + result.push(line); + } else { + break; + } + } + + rl2.close(); + + const content = includeStatusMessage + ? `${this.generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false)}\n\n${result.join('\n')}` + : result.join('\n'); + + return { content, mimeType, metadata: {} }; + } finally { + await fd.close(); + } + } +} diff --git a/test/test-excel-files.js b/test/test-excel-files.js new file mode 100644 index 00000000..8913d552 --- /dev/null +++ b/test/test-excel-files.js @@ -0,0 +1,369 @@ +/** + * Test script for Excel file handling functionality + * + * This script tests the ExcelFileHandler implementation: + * 1. Reading Excel files (basic, sheet selection, range, offset/length) + * 2. Writing Excel files (single sheet, multiple sheets, append mode) + * 3. Editing Excel files (range updates) + * 4. Getting Excel file info (sheet metadata) + * 5. File handler factory (correct handler selection) + */ + +import { configManager } from '../dist/config-manager.js'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import assert from 'assert'; +import { readFile, writeFile, getFileInfo } from '../dist/tools/filesystem.js'; +import { handleEditBlock } from '../dist/handlers/edit-search-handlers.js'; +import { getFileHandler } from '../dist/utils/files/factory.js'; + +// Get directory name +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Define test directory and files +const TEST_DIR = path.join(__dirname, 'test_excel_files'); +const BASIC_EXCEL = path.join(TEST_DIR, 'basic.xlsx'); +const MULTI_SHEET_EXCEL = path.join(TEST_DIR, 'multi_sheet.xlsx'); +const EDIT_EXCEL = path.join(TEST_DIR, 'edit_test.xlsx'); + +/** + * Helper function to clean up test directories + */ +async function cleanupTestDirectories() { + try { + await fs.rm(TEST_DIR, { recursive: true, force: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('Error during cleanup:', error); + } + } +} + +/** + * Setup function to prepare the test environment + */ +async function setup() { + // Clean up before tests (in case previous run left files) + await cleanupTestDirectories(); + + // Create test directory + await fs.mkdir(TEST_DIR, { recursive: true }); + console.log(`✓ Setup: created test directory: ${TEST_DIR}`); + + // Save original config to restore later + const originalConfig = await configManager.getConfig(); + + // Set allowed directories to include our test directory + await configManager.setValue('allowedDirectories', [TEST_DIR]); + console.log(`✓ Setup: set allowed directories`); + + return originalConfig; +} + +/** + * Teardown function to clean up after tests + */ +async function teardown(originalConfig) { + // Reset configuration to original + if (originalConfig) { + await configManager.updateConfig(originalConfig); + } + + await cleanupTestDirectories(); + console.log('✓ Teardown: test directory cleaned up and config restored'); +} + +/** + * Test 1: File handler factory selects ExcelFileHandler for .xlsx files + */ +async function testFileHandlerFactory() { + console.log('\n--- Test 1: File Handler Factory ---'); + + const handler = getFileHandler('test.xlsx'); + assert.ok(handler, 'Handler should be returned for .xlsx file'); + assert.ok(handler.constructor.name === 'ExcelFileHandler', + `Expected ExcelFileHandler but got ${handler.constructor.name}`); + + const txtHandler = getFileHandler('test.txt'); + assert.ok(txtHandler.constructor.name === 'TextFileHandler', + `Expected TextFileHandler for .txt but got ${txtHandler.constructor.name}`); + + console.log('✓ File handler factory correctly selects handlers'); +} + +/** + * Test 2: Write and read basic Excel file + */ +async function testBasicWriteRead() { + console.log('\n--- Test 2: Basic Write and Read ---'); + + // Write a simple Excel file + const data = JSON.stringify([ + ['Name', 'Age', 'City'], + ['Alice', 30, 'New York'], + ['Bob', 25, 'Los Angeles'], + ['Charlie', 35, 'Chicago'] + ]); + + await writeFile(BASIC_EXCEL, data); + console.log('✓ Wrote basic Excel file'); + + // Read it back + const result = await readFile(BASIC_EXCEL); + assert.ok(result.content, 'Should have content'); + // Excel handler returns application/json because content is JSON-formatted for LLM consumption + assert.ok(result.mimeType === 'application/json', + `Expected application/json mime type but got ${result.mimeType}`); + + // Verify content contains our data + const content = result.content.toString(); + assert.ok(content.includes('Name'), 'Content should include Name header'); + assert.ok(content.includes('Alice'), 'Content should include Alice'); + assert.ok(content.includes('Chicago'), 'Content should include Chicago'); + + console.log('✓ Read back Excel file with correct content'); +} + +/** + * Test 3: Write and read multi-sheet Excel file + */ +async function testMultiSheetWriteRead() { + console.log('\n--- Test 3: Multi-Sheet Write and Read ---'); + + // Write multi-sheet Excel file + const data = JSON.stringify({ + 'Employees': [ + ['Name', 'Department'], + ['Alice', 'Engineering'], + ['Bob', 'Sales'] + ], + 'Departments': [ + ['Name', 'Budget'], + ['Engineering', 100000], + ['Sales', 50000] + ] + }); + + await writeFile(MULTI_SHEET_EXCEL, data); + console.log('✓ Wrote multi-sheet Excel file'); + + // Read specific sheet by name + const result1 = await readFile(MULTI_SHEET_EXCEL, { sheet: 'Employees' }); + const content1 = result1.content.toString(); + assert.ok(content1.includes('Alice'), 'Employees sheet should contain Alice'); + assert.ok(content1.includes('Engineering'), 'Employees sheet should contain Engineering'); + console.log('✓ Read Employees sheet by name'); + + // Read specific sheet by index + const result2 = await readFile(MULTI_SHEET_EXCEL, { sheet: 1 }); + const content2 = result2.content.toString(); + assert.ok(content2.includes('Budget'), 'Departments sheet should contain Budget'); + assert.ok(content2.includes('100000'), 'Departments sheet should contain 100000'); + console.log('✓ Read Departments sheet by index'); +} + +/** + * Test 4: Read with range parameter + */ +async function testRangeRead() { + console.log('\n--- Test 4: Range Read ---'); + + // Use the basic file we created + const result = await readFile(BASIC_EXCEL, { sheet: 'Sheet1', range: 'A1:B2' }); + const content = result.content.toString(); + + // Should only have first 2 rows and 2 columns + assert.ok(content.includes('Name'), 'Range should include Name'); + assert.ok(content.includes('Age'), 'Range should include Age'); + assert.ok(content.includes('Alice'), 'Range should include Alice'); + // City is column C, should NOT be included + assert.ok(!content.includes('City') || content.split('City').length === 1, + 'Range A1:B2 should not include City column'); + + console.log('✓ Range read returns correct subset of data'); +} + +/** + * Test 5: Read with offset and length + */ +async function testOffsetLengthRead() { + console.log('\n--- Test 5: Offset and Length Read ---'); + + // Read with offset (skip header) + const result = await readFile(BASIC_EXCEL, { offset: 1, length: 2 }); + const content = result.content.toString(); + + // Should have rows 2-3 (Alice, Bob) but not header or Charlie + assert.ok(content.includes('Alice'), 'Should include Alice (row 2)'); + assert.ok(content.includes('Bob'), 'Should include Bob (row 3)'); + + console.log('✓ Offset and length read works correctly'); +} + +/** + * Test 6: Edit Excel range + */ +async function testEditRange() { + console.log('\n--- Test 6: Edit Excel Range ---'); + + // Create a file to edit + const data = JSON.stringify([ + ['Product', 'Price'], + ['Apple', 1.00], + ['Banana', 0.50], + ['Cherry', 2.00] + ]); + await writeFile(EDIT_EXCEL, data); + console.log('✓ Created file for editing'); + + // Edit a cell using edit_block with range + const editResult = await handleEditBlock({ + file_path: EDIT_EXCEL, + range: 'Sheet1!B2', + content: [[1.50]] // Update Apple price + }); + + assert.ok(!editResult.isError, `Edit should succeed: ${editResult.content?.[0]?.text}`); + console.log('✓ Edit range succeeded'); + + // Verify the edit + const readResult = await readFile(EDIT_EXCEL); + const content = readResult.content.toString(); + assert.ok(content.includes('1.5'), 'Price should be updated to 1.50'); + + console.log('✓ Edit was persisted correctly'); +} + +/** + * Test 7: Get Excel file info + */ +async function testGetFileInfo() { + console.log('\n--- Test 7: Get File Info ---'); + + const info = await getFileInfo(MULTI_SHEET_EXCEL); + + assert.ok(info.isExcelFile, 'Should be marked as Excel file'); + assert.ok(info.sheets, 'Should have sheets info'); + assert.ok(Array.isArray(info.sheets), 'Sheets should be an array'); + assert.strictEqual(info.sheets.length, 2, 'Should have 2 sheets'); + + // Check sheet details + const sheetNames = info.sheets.map(s => s.name); + assert.ok(sheetNames.includes('Employees'), 'Should have Employees sheet'); + assert.ok(sheetNames.includes('Departments'), 'Should have Departments sheet'); + + // Check row/column counts + const employeesSheet = info.sheets.find(s => s.name === 'Employees'); + assert.ok(employeesSheet.rowCount >= 3, 'Employees sheet should have at least 3 rows'); + assert.ok(employeesSheet.colCount >= 2, 'Employees sheet should have at least 2 columns'); + + console.log('✓ File info returns correct sheet metadata'); +} + +/** + * Test 8: Append mode + */ +async function testAppendMode() { + console.log('\n--- Test 8: Append Mode ---'); + + // Create initial file + const initialData = JSON.stringify([ + ['Name', 'Score'], + ['Alice', 100] + ]); + await writeFile(BASIC_EXCEL, initialData); + + // Append more data + const appendData = JSON.stringify([ + ['Bob', 95], + ['Charlie', 88] + ]); + await writeFile(BASIC_EXCEL, appendData, 'append'); + console.log('✓ Appended data to Excel file'); + + // Read and verify + const result = await readFile(BASIC_EXCEL); + const content = result.content.toString(); + + assert.ok(content.includes('Alice'), 'Should still have Alice'); + assert.ok(content.includes('Bob'), 'Should have appended Bob'); + assert.ok(content.includes('Charlie'), 'Should have appended Charlie'); + + console.log('✓ Append mode works correctly'); +} + +/** + * Test 9: Negative offset (read from end) + */ +async function testNegativeOffset() { + console.log('\n--- Test 9: Negative Offset (Tail) ---'); + + // Create file with multiple rows + const data = JSON.stringify([ + ['Row', 'Value'], + ['1', 'First'], + ['2', 'Second'], + ['3', 'Third'], + ['4', 'Fourth'], + ['5', 'Fifth'] + ]); + await writeFile(BASIC_EXCEL, data); + + // Read last 2 rows + const result = await readFile(BASIC_EXCEL, { offset: -2 }); + const content = result.content.toString(); + + assert.ok(content.includes('Fourth') || content.includes('Fifth'), + 'Should include data from last rows'); + + console.log('✓ Negative offset reads from end'); +} + +/** + * Run all tests + */ +async function runAllTests() { + console.log('=== Excel File Handling Tests ===\n'); + + await testFileHandlerFactory(); + await testBasicWriteRead(); + await testMultiSheetWriteRead(); + await testRangeRead(); + await testOffsetLengthRead(); + await testEditRange(); + await testGetFileInfo(); + await testAppendMode(); + await testNegativeOffset(); + + console.log('\n✅ All Excel tests passed!'); +} + +// Export the main test function +export default async function runTests() { + let originalConfig; + try { + originalConfig = await setup(); + await runAllTests(); + } catch (error) { + console.error('❌ Test failed:', error.message); + console.error(error.stack); + return false; + } finally { + if (originalConfig) { + await teardown(originalConfig); + } + } + return true; +} + +// If this file is run directly, execute the test +if (import.meta.url === `file://${process.argv[1]}`) { + runTests().then(success => { + process.exit(success ? 0 : 1); + }).catch(error => { + console.error('❌ Unhandled error:', error); + process.exit(1); + }); +} diff --git a/test/test-file-handlers.js b/test/test-file-handlers.js new file mode 100644 index 00000000..6f83f6e0 --- /dev/null +++ b/test/test-file-handlers.js @@ -0,0 +1,313 @@ +/** + * Test script for file handler system + * + * This script tests the file handler architecture: + * 1. File handler factory returns correct handler types + * 2. FileResult interface consistency + * 3. ReadOptions interface usage + * 4. Handler canHandle() method + * 5. Text file handler basic operations + * 6. Image file handler detection + * 7. Binary file handler fallback + */ + +import { configManager } from '../dist/config-manager.js'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import assert from 'assert'; +import { readFile, writeFile, getFileInfo } from '../dist/tools/filesystem.js'; +import { getFileHandler } from '../dist/utils/files/factory.js'; + +// Get directory name +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Define test directory and files +const TEST_DIR = path.join(__dirname, 'test_file_handlers'); +const TEXT_FILE = path.join(TEST_DIR, 'test.txt'); +const JSON_FILE = path.join(TEST_DIR, 'test.json'); +const MD_FILE = path.join(TEST_DIR, 'test.md'); + +/** + * Helper function to clean up test directories + */ +async function cleanupTestDirectories() { + try { + await fs.rm(TEST_DIR, { recursive: true, force: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('Error during cleanup:', error); + } + } +} + +/** + * Setup function + */ +async function setup() { + // Clean up before tests (in case previous run left files) + await cleanupTestDirectories(); + + await fs.mkdir(TEST_DIR, { recursive: true }); + console.log(`✓ Setup: created test directory: ${TEST_DIR}`); + + const originalConfig = await configManager.getConfig(); + await configManager.setValue('allowedDirectories', [TEST_DIR]); + + return originalConfig; +} + +/** + * Teardown function + */ +async function teardown(originalConfig) { + if (originalConfig) { + await configManager.updateConfig(originalConfig); + } + + await cleanupTestDirectories(); + console.log('✓ Teardown: cleaned up'); +} + +/** + * Test 1: Handler factory returns correct types + */ +async function testHandlerFactory() { + console.log('\n--- Test 1: Handler Factory Types ---'); + + // Note: TextFileHandler.canHandle() returns true for all files, + // so it catches most files before BinaryFileHandler. + // BinaryFileHandler only handles files that fail binary detection at read time. + const testCases = [ + { file: 'test.xlsx', expected: 'ExcelFileHandler' }, + { file: 'test.xls', expected: 'ExcelFileHandler' }, + { file: 'test.xlsm', expected: 'ExcelFileHandler' }, + { file: 'test.txt', expected: 'TextFileHandler' }, + { file: 'test.js', expected: 'TextFileHandler' }, + { file: 'test.json', expected: 'TextFileHandler' }, + { file: 'test.md', expected: 'TextFileHandler' }, + { file: 'test.png', expected: 'ImageFileHandler' }, + { file: 'test.jpg', expected: 'ImageFileHandler' }, + { file: 'test.jpeg', expected: 'ImageFileHandler' }, + { file: 'test.gif', expected: 'ImageFileHandler' }, + { file: 'test.webp', expected: 'ImageFileHandler' }, + ]; + + for (const { file, expected } of testCases) { + const handler = getFileHandler(file); + assert.strictEqual(handler.constructor.name, expected, + `${file} should use ${expected} but got ${handler.constructor.name}`); + } + + console.log('✓ All file types map to correct handlers'); +} + +/** + * Test 2: FileResult interface consistency + */ +async function testFileResultInterface() { + console.log('\n--- Test 2: FileResult Interface ---'); + + // Create a text file + await fs.writeFile(TEXT_FILE, 'Hello, World!\nLine 2\nLine 3'); + + const result = await readFile(TEXT_FILE); + + // Check FileResult structure + assert.ok('content' in result, 'FileResult should have content'); + assert.ok('mimeType' in result, 'FileResult should have mimeType'); + assert.ok(result.content !== undefined, 'Content should not be undefined'); + assert.ok(typeof result.mimeType === 'string', 'mimeType should be a string'); + + // metadata is optional but should be an object if present + if (result.metadata) { + assert.ok(typeof result.metadata === 'object', 'metadata should be an object'); + } + + console.log('✓ FileResult interface is consistent'); +} + +/** + * Test 3: ReadOptions interface + */ +async function testReadOptionsInterface() { + console.log('\n--- Test 3: ReadOptions Interface ---'); + + await fs.writeFile(TEXT_FILE, 'Line 1\nLine 2\nLine 3\nLine 4\nLine 5'); + + // Test offset option + const result1 = await readFile(TEXT_FILE, { offset: 2 }); + const content1 = result1.content.toString(); + assert.ok(content1.includes('Line 3'), 'Offset should skip to line 3'); + + // Test length option + const result2 = await readFile(TEXT_FILE, { offset: 0, length: 2 }); + const content2 = result2.content.toString(); + assert.ok(content2.includes('Line 1'), 'Should include Line 1'); + assert.ok(content2.includes('Line 2'), 'Should include Line 2'); + + console.log('✓ ReadOptions work correctly'); +} + +/** + * Test 4: Handler canHandle method + */ +async function testCanHandle() { + console.log('\n--- Test 4: canHandle Method ---'); + + const excelHandler = getFileHandler('test.xlsx'); + const textHandler = getFileHandler('test.txt'); + const imageHandler = getFileHandler('test.png'); + + // Excel handler should handle xlsx + assert.ok(excelHandler.canHandle('anything.xlsx'), 'Excel handler should handle .xlsx'); + assert.ok(excelHandler.canHandle('file.xls'), 'Excel handler should handle .xls'); + + // Image handler should handle images + assert.ok(imageHandler.canHandle('photo.png'), 'Image handler should handle .png'); + assert.ok(imageHandler.canHandle('photo.jpg'), 'Image handler should handle .jpg'); + assert.ok(imageHandler.canHandle('photo.jpeg'), 'Image handler should handle .jpeg'); + + // Text handler handles most things (fallback) + assert.ok(textHandler.canHandle('file.txt'), 'Text handler should handle .txt'); + + console.log('✓ canHandle methods work correctly'); +} + +/** + * Test 5: Text handler read/write + */ +async function testTextHandler() { + console.log('\n--- Test 5: Text Handler Operations ---'); + + const content = 'Test content\nWith multiple lines\nAnd special chars: äöü'; + + // Write + await writeFile(TEXT_FILE, content); + console.log('✓ Text write succeeded'); + + // Read + const result = await readFile(TEXT_FILE); + const readContent = result.content.toString(); + assert.ok(readContent.includes('Test content'), 'Should read back content'); + assert.ok(readContent.includes('äöü'), 'Should preserve special characters'); + + console.log('✓ Text handler read/write works'); +} + +/** + * Test 6: Text handler with JSON file + */ +async function testJsonFile() { + console.log('\n--- Test 6: JSON File Handling ---'); + + const data = { name: 'Test', values: [1, 2, 3] }; + const content = JSON.stringify(data, null, 2); + + await writeFile(JSON_FILE, content); + + const result = await readFile(JSON_FILE); + const readContent = result.content.toString(); + const parsed = JSON.parse(readContent.replace(/^\[.*?\]\n\n/, '')); // Remove status message + + assert.strictEqual(parsed.name, 'Test', 'JSON should be preserved'); + assert.deepStrictEqual(parsed.values, [1, 2, 3], 'Array should be preserved'); + + console.log('✓ JSON file handling works'); +} + +/** + * Test 7: File info returns correct structure + */ +async function testFileInfo() { + console.log('\n--- Test 7: File Info Structure ---'); + + await fs.writeFile(TEXT_FILE, 'Some content'); + + const info = await getFileInfo(TEXT_FILE); + + // Check required fields + assert.ok('size' in info, 'Should have size'); + assert.ok('created' in info || 'birthtime' in info, 'Should have creation time'); + assert.ok('modified' in info || 'mtime' in info, 'Should have modification time'); + assert.ok('isFile' in info, 'Should have isFile'); + assert.ok('isDirectory' in info, 'Should have isDirectory'); + + assert.ok(info.size > 0, 'Size should be > 0'); + assert.ok(info.isFile === true || info.isFile === 'true', 'Should be a file'); + + console.log('✓ File info structure is correct'); +} + +/** + * Test 8: Write mode (rewrite vs append) + */ +async function testWriteModes() { + console.log('\n--- Test 8: Write Modes ---'); + + // Initial write (rewrite mode - default) + await writeFile(TEXT_FILE, 'Initial content'); + + // Overwrite + await writeFile(TEXT_FILE, 'New content', 'rewrite'); + let result = await readFile(TEXT_FILE); + let content = result.content.toString(); + assert.ok(!content.includes('Initial'), 'Rewrite should replace content'); + assert.ok(content.includes('New content'), 'Should have new content'); + console.log('✓ Rewrite mode works'); + + // Append + await writeFile(TEXT_FILE, '\nAppended content', 'append'); + result = await readFile(TEXT_FILE); + content = result.content.toString(); + assert.ok(content.includes('New content'), 'Should keep original'); + assert.ok(content.includes('Appended content'), 'Should have appended'); + console.log('✓ Append mode works'); +} + +/** + * Run all tests + */ +async function runAllTests() { + console.log('=== File Handler System Tests ===\n'); + + await testHandlerFactory(); + await testFileResultInterface(); + await testReadOptionsInterface(); + await testCanHandle(); + await testTextHandler(); + await testJsonFile(); + await testFileInfo(); + await testWriteModes(); + + console.log('\n✅ All file handler tests passed!'); +} + +// Export the main test function +export default async function runTests() { + let originalConfig; + try { + originalConfig = await setup(); + await runAllTests(); + } catch (error) { + console.error('❌ Test failed:', error.message); + console.error(error.stack); + return false; + } finally { + if (originalConfig) { + await teardown(originalConfig); + } + } + return true; +} + +// If this file is run directly, execute the test +if (import.meta.url === `file://${process.argv[1]}`) { + runTests().then(success => { + process.exit(success ? 0 : 1); + }).catch(error => { + console.error('❌ Unhandled error:', error); + process.exit(1); + }); +} diff --git a/test/test_output/node_repl_debug.txt b/test/test_output/node_repl_debug.txt index 0db4e1d4..097abd84 100644 --- a/test/test_output/node_repl_debug.txt +++ b/test/test_output/node_repl_debug.txt @@ -1,16 +1,16 @@ Starting Node.js REPL... Waiting for Node.js startup... -[STDOUT] Welcome to Node.js v22.18.0. +[STDOUT] Welcome to Node.js v21.7.3. Type ".help" for more information. [STDOUT] > -Initial output buffer: Welcome to Node.js v22.18.0. +Initial output buffer: Welcome to Node.js v21.7.3. Type ".help" for more information. >  Sending simple command... [STDOUT] Hello from Node.js! -[STDOUT] undefined -> -Output after first command: Welcome to Node.js v22.18.0. +[STDOUT] undefined +[STDOUT] > +Output after first command: Welcome to Node.js v21.7.3. Type ".help" for more information. > Hello from Node.js! undefined @@ -29,18 +29,18 @@ for (let i = 0; i < 3; i++) { [STDOUT] > [STDOUT] ... [STDOUT] ... -[STDOUT] undefined -> +[STDOUT] undefined +[STDOUT] > [STDOUT] > [STDOUT] ... [STDOUT] ... [STDOUT] Hello, User 0! -[STDOUT] Hello, User 1! -[STDOUT] Hello, User 2! -[STDOUT] undefined -[STDOUT] > +[STDOUT] Hello, User 1! +Hello, User 2! +[STDOUT] undefined +> [STDOUT] > -Final output buffer: Welcome to Node.js v22.18.0. +Final output buffer: Welcome to Node.js v21.7.3. Type ".help" for more information. > Hello from Node.js! undefined From 704a84e06982a2e6cefa8a79ff0c0d9c3b929bed Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Wed, 26 Nov 2025 19:09:01 +0200 Subject: [PATCH 02/17] debug file remove --- test/test_output/node_repl_debug.txt | 56 ---------------------------- 1 file changed, 56 deletions(-) delete mode 100644 test/test_output/node_repl_debug.txt diff --git a/test/test_output/node_repl_debug.txt b/test/test_output/node_repl_debug.txt deleted file mode 100644 index 097abd84..00000000 --- a/test/test_output/node_repl_debug.txt +++ /dev/null @@ -1,56 +0,0 @@ -Starting Node.js REPL... -Waiting for Node.js startup... -[STDOUT] Welcome to Node.js v21.7.3. -Type ".help" for more information. -[STDOUT] > -Initial output buffer: Welcome to Node.js v21.7.3. -Type ".help" for more information. ->  -Sending simple command... -[STDOUT] Hello from Node.js! -[STDOUT] undefined -[STDOUT] > -Output after first command: Welcome to Node.js v21.7.3. -Type ".help" for more information. -> Hello from Node.js! -undefined ->  -Sending multi-line command directly... -Sending code: - -function greet(name) { - return `Hello, ${name}!`; -} - -for (let i = 0; i < 3; i++) { - console.log(greet(`User ${i}`)); -} - -[STDOUT] > -[STDOUT] ... -[STDOUT] ... -[STDOUT] undefined -[STDOUT] > -[STDOUT] > -[STDOUT] ... -[STDOUT] ... -[STDOUT] Hello, User 0! -[STDOUT] Hello, User 1! -Hello, User 2! -[STDOUT] undefined -> -[STDOUT] > -Final output buffer: Welcome to Node.js v21.7.3. -Type ".help" for more information. -> Hello from Node.js! -undefined -> > ... ... undefined -> > ... ... Hello, User 0! -Hello, User 1! -Hello, User 2! -undefined -> >  -Found "Hello from Node.js!": true -Found greetings: true -Terminating Node.js process... -Node.js process exited with code 0 From 4354153053d40cbb7f9bc0dbdc4d783bc5b6aeaf Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Wed, 26 Nov 2025 19:31:02 +0200 Subject: [PATCH 03/17] more emphasis on node for excel processing and analysis --- src/server.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/server.ts b/src/server.ts index 008e4cd1..07875a2b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -994,13 +994,17 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { description: ` Execute Node.js code directly using the MCP server's Node runtime. + PRIMARY TOOL FOR EXCEL FILES AND COMPLEX CALCULATIONS + Use this tool for ANY Excel file (.xlsx, .xls) operations and complex data calculations. + ExcelJS library is built-in and ready to use. + Code runs as ES module (.mjs) with top-level await support. Uses the same Node.js environment that runs the MCP server. - Available libraries: exceljs (for Excel file manipulation), and all Node.js built-ins. + Available libraries: ExcelJS (for Excel file manipulation), and all Node.js built-ins. - Use cases: Data transformations, bulk file operations, complex calculations, - Excel manipulation, JSON processing, or any task better suited to code than tools. + Use cases: Excel file reading/writing/analysis, data transformations, bulk file operations, + complex calculations, JSON processing, or any task better suited to code than tools. Output: Use console.log() to return results. Stdout is captured and returned. From 3afa03c9e345b81611e25cc7c5d3bd9e7fe6d282 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Wed, 26 Nov 2025 22:42:35 +0200 Subject: [PATCH 04/17] fixes from review --- src/handlers/filesystem-handlers.ts | 11 ++++- src/search-manager.ts | 59 ++++++++++++++++++++----- src/tools/filesystem.ts | 67 +++++++++++++++++++++++------ test/test-file-handlers.js | 26 ++++++++--- 4 files changed, 131 insertions(+), 32 deletions(-) diff --git a/src/handlers/filesystem-handlers.ts b/src/handlers/filesystem-handlers.ts index fea83a04..09289dcf 100644 --- a/src/handlers/filesystem-handlers.ts +++ b/src/handlers/filesystem-handlers.ts @@ -71,6 +71,10 @@ export async function handleReadFile(args: unknown): Promise { if (fileResult.metadata?.isImage) { // For image files, return as an image content type + // Content should already be base64-encoded string from handler + const imageData = typeof fileResult.content === 'string' + ? fileResult.content + : fileResult.content.toString('base64'); return { content: [ { @@ -79,15 +83,18 @@ export async function handleReadFile(args: unknown): Promise { }, { type: "image", - data: fileResult.content.toString(), + data: imageData, mimeType: fileResult.mimeType } ], }; } else { // For all other files, return as text + const textContent = typeof fileResult.content === 'string' + ? fileResult.content + : fileResult.content.toString('utf8'); return { - content: [{ type: "text", text: fileResult.content.toString() }], + content: [{ type: "text", text: textContent }], }; } }; diff --git a/src/search-manager.ts b/src/search-manager.ts index d2af0558..8ebacb96 100644 --- a/src/search-manager.ts +++ b/src/search-manager.ts @@ -146,17 +146,21 @@ export interface SearchSessionOptions { validatedPath: validPath }); - // For content searches, also search Excel files in parallel - let excelSearchPromise: Promise | null = null; - if (options.searchType === 'content') { - excelSearchPromise = this.searchExcelFiles( + // For content searches, only search Excel files when contextually relevant: + // - filePattern explicitly targets Excel files (*.xlsx, *.xls, etc.) + // - or rootPath is an Excel file itself + const shouldSearchExcel = options.searchType === 'content' && + this.shouldIncludeExcelSearch(options.filePattern, validPath); + + if (shouldSearchExcel) { + this.searchExcelFiles( validPath, options.pattern, options.ignoreCase !== false, options.maxResults, options.filePattern // Pass filePattern to filter Excel files too ).then(excelResults => { - // Add Excel results to session + // Add Excel results to session (merged after initial response) for (const result of excelResults) { session.results.push(result); session.totalMatches++; @@ -168,6 +172,7 @@ export interface SearchSessionOptions { } // Wait for first chunk of data or early completion instead of fixed delay + // Excel search runs in background and results are merged via readSearchResults const firstChunk = new Promise(resolve => { const onData = () => { session.process.stdout?.off('data', onData); @@ -177,8 +182,8 @@ export interface SearchSessionOptions { setTimeout(resolve, 40); // cap at 40ms instead of 50-100ms }); - // Wait for both ripgrep first chunk and Excel search - await Promise.all([firstChunk, excelSearchPromise].filter(Boolean)); + // Only wait for ripgrep first chunk - Excel results merge asynchronously + await firstChunk; return { sessionId, @@ -507,7 +512,7 @@ export interface SearchSessionOptions { * (has file extension and no glob wildcards) */ private isExactFilename(pattern: string): boolean { - return /\.[a-zA-Z0-9]+$/.test(pattern) && + return /\.[a-zA-Z0-9]+$/.test(pattern) && !this.isGlobPattern(pattern); } @@ -515,14 +520,46 @@ export interface SearchSessionOptions { * Detect if pattern contains glob wildcards */ private isGlobPattern(pattern: string): boolean { - return pattern.includes('*') || - pattern.includes('?') || - pattern.includes('[') || + return pattern.includes('*') || + pattern.includes('?') || + pattern.includes('[') || pattern.includes('{') || pattern.includes(']') || pattern.includes('}'); } + /** + * Determine if Excel search should be included based on context + * Only searches Excel files when: + * - filePattern explicitly targets Excel files (*.xlsx, *.xls, *.xlsm, *.xlsb) + * - or the rootPath itself is an Excel file + */ + private shouldIncludeExcelSearch(filePattern?: string, rootPath?: string): boolean { + const excelExtensions = ['.xlsx', '.xls', '.xlsm', '.xlsb']; + + // Check if rootPath is an Excel file + if (rootPath) { + const lowerPath = rootPath.toLowerCase(); + if (excelExtensions.some(ext => lowerPath.endsWith(ext))) { + return true; + } + } + + // Check if filePattern targets Excel files + if (filePattern) { + const lowerPattern = filePattern.toLowerCase(); + // Check for patterns like *.xlsx, *.xls, or explicit Excel extensions + if (excelExtensions.some(ext => + lowerPattern.includes(`*${ext}`) || + lowerPattern.endsWith(ext) + )) { + return true; + } + } + + return false; + } + private buildRipgrepArgs(options: SearchSessionOptions): string[] { const args: string[] = []; diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts index 21346589..88d814e5 100644 --- a/src/tools/filesystem.ts +++ b/src/tools/filesystem.ts @@ -693,8 +693,20 @@ export async function readFileFromDisk( }); // Return with content as string + // For images: content is already base64-encoded string from handler + // For text: content may be string or Buffer, convert to UTF-8 string + let content: string; + if (typeof result.content === 'string') { + content = result.content; + } else if (result.metadata?.isImage) { + // Image buffer should be base64 encoded, not UTF-8 converted + content = result.content.toString('base64'); + } else { + content = result.content.toString('utf8'); + } + return { - content: result.content.toString(), + content, mimeType: result.mimeType, metadata: result.metadata }; @@ -861,9 +873,19 @@ export async function readMultipleFiles(paths: string[]): Promise> { const validPath = await validatePath(filePath); + // Get fs.stat as a fallback for any missing fields + const stats = await fs.stat(validPath); + const fallbackInfo = { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'text' as const, + metadata: undefined as Record | undefined, + }; + // Get appropriate handler for this file type const handler = getFileHandler(validPath); - // Use handler to get file info - const fileInfo = await handler.getInfo(validPath); + // Use handler to get file info, with fallback + let fileInfo; + try { + fileInfo = await handler.getInfo(validPath); + } catch (error) { + // If handler fails, use fallback stats + fileInfo = fallbackInfo; + } // Convert to legacy format (for backward compatibility) + // Use handler values with fallback to fs.stat values for any missing fields const info: Record = { - size: fileInfo.size, - created: fileInfo.created, - modified: fileInfo.modified, - accessed: fileInfo.accessed, - isDirectory: fileInfo.isDirectory, - isFile: fileInfo.isFile, - permissions: fileInfo.permissions, - fileType: fileInfo.fileType, + size: fileInfo.size ?? fallbackInfo.size, + created: fileInfo.created ?? fallbackInfo.created, + modified: fileInfo.modified ?? fallbackInfo.modified, + accessed: fileInfo.accessed ?? fallbackInfo.accessed, + isDirectory: fileInfo.isDirectory ?? fallbackInfo.isDirectory, + isFile: fileInfo.isFile ?? fallbackInfo.isFile, + permissions: fileInfo.permissions ?? fallbackInfo.permissions, + fileType: fileInfo.fileType ?? fallbackInfo.fileType, }; // Add type-specific metadata diff --git a/test/test-file-handlers.js b/test/test-file-handlers.js index 6f83f6e0..1adcf7ec 100644 --- a/test/test-file-handlers.js +++ b/test/test-file-handlers.js @@ -60,14 +60,26 @@ async function setup() { /** * Teardown function + * Always runs cleanup, restores config only if provided */ async function teardown(originalConfig) { - if (originalConfig) { - await configManager.updateConfig(originalConfig); + // Always clean up test directories, even if setup failed + try { + await cleanupTestDirectories(); + console.log('✓ Teardown: cleaned up test directories'); + } catch (error) { + console.error('Warning: Failed to clean up test directories:', error.message); } - await cleanupTestDirectories(); - console.log('✓ Teardown: cleaned up'); + // Restore config only if we have the original + if (originalConfig) { + try { + await configManager.updateConfig(originalConfig); + console.log('✓ Teardown: restored config'); + } catch (error) { + console.error('Warning: Failed to restore config:', error.message); + } + } } /** @@ -295,9 +307,9 @@ export default async function runTests() { console.error(error.stack); return false; } finally { - if (originalConfig) { - await teardown(originalConfig); - } + // Always run teardown to clean up test directories and restore config + // teardown handles the case where originalConfig is undefined + await teardown(originalConfig); } return true; } From 974be409e594c00d30871365e43630f5cb8d8c5a Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Fri, 28 Nov 2025 20:35:10 +0200 Subject: [PATCH 05/17] removing dead code and prompts left from testing --- src/tools/edit.ts | 2 +- src/tools/filesystem.ts | 406 +------------------------------------ src/utils/files/base.ts | 4 +- src/utils/files/binary.ts | 122 ++--------- src/utils/files/factory.ts | 75 ++++--- src/utils/files/text.ts | 34 ++-- 6 files changed, 90 insertions(+), 553 deletions(-) diff --git a/src/tools/edit.ts b/src/tools/edit.ts index 05453cf0..590c1fb7 100644 --- a/src/tools/edit.ts +++ b/src/tools/edit.ts @@ -374,7 +374,7 @@ export async function handleEditBlock(args: unknown): Promise { if (parsed.range && parsed.content !== undefined) { try { const { getFileHandler } = await import('../utils/files/factory.js'); - const handler = getFileHandler(parsed.file_path); + const handler = await getFileHandler(parsed.file_path); // Parse content if it's a JSON string (AI often sends arrays as JSON strings) let content = parsed.content; diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts index 88d814e5..bc669a17 100644 --- a/src/tools/filesystem.ts +++ b/src/tools/filesystem.ts @@ -2,64 +2,25 @@ import fs from "fs/promises"; import path from "path"; import os from 'os'; import fetch from 'cross-fetch'; -import { createReadStream } from 'fs'; -import { createInterface } from 'readline'; -import { isBinaryFile } from 'isbinaryfile'; import {capture} from '../utils/capture.js'; import {withTimeout} from '../utils/withTimeout.js'; import {configManager} from '../config-manager.js'; -import { getFileHandler } from '../utils/files/factory.js'; +import { getFileHandler, TextFileHandler } from '../utils/files/index.js'; import type { ReadOptions, FileResult } from '../utils/files/base.js'; // CONSTANTS SECTION - Consolidate all timeouts and thresholds const FILE_OPERATION_TIMEOUTS = { PATH_VALIDATION: 10000, // 10 seconds - URL_FETCH: 30000, // 30 seconds + URL_FETCH: 30000, // 30 seconds FILE_READ: 30000, // 30 seconds } as const; const FILE_SIZE_LIMITS = { - LARGE_FILE_THRESHOLD: 10 * 1024 * 1024, // 10MB LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting } as const; -const READ_PERFORMANCE_THRESHOLDS = { - SMALL_READ_THRESHOLD: 100, // For very small reads - DEEP_OFFSET_THRESHOLD: 1000, // For byte estimation - SAMPLE_SIZE: 10000, // Sample size for estimation - CHUNK_SIZE: 8192, // 8KB chunks for reverse reading -} as const; - // UTILITY FUNCTIONS - Eliminate duplication -/** - * Count lines in text content efficiently - * @param content Text content to count lines in - * @returns Number of lines - */ -function countLines(content: string): number { - return content.split('\n').length; -} - -/** - * Count lines in a file efficiently (for files under size limit) - * @param filePath Path to the file - * @returns Line count or undefined if file too large/can't read - */ -async function getFileLineCount(filePath: string): Promise { - try { - const stats = await fs.stat(filePath); - // Only count lines for reasonably sized files to avoid performance issues - if (stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { - const content = await fs.readFile(filePath, 'utf8'); - return countLines(content); - } - } catch (error) { - // If we can't read the file, just return undefined - } - return undefined; -} - /** * Get MIME type information for a file * @param filePath Path to the file @@ -90,22 +51,6 @@ async function getDefaultReadLength(): Promise { return config.fileReadLineLimit ?? 1000; // Default to 1000 lines if not set } -/** - * Generate instructions for handling binary files - * @param filePath Path to the binary file - * @param mimeType MIME type of the file - * @returns Instruction message for the LLM - */ -function getBinaryFileInstructions(filePath: string, mimeType: string): string { - const fileName = path.basename(filePath); - - return `Cannot read binary file as text: ${fileName} (${mimeType}) - -Use start_process + interact_with_process to analyze binary files with appropriate tools (Node.js or Python libraries, command-line utilities, etc.). - -The read_file tool only handles text files and images.`; -} - // Initialize allowed directories from configuration async function getAllowedDirs(): Promise { try { @@ -342,296 +287,6 @@ export async function readFileFromUrl(url: string): Promise { -/** - * Generate enhanced status message with total and remaining line information - * @param readLines Number of lines actually read - * @param offset Starting offset (line number) - * @param totalLines Total lines in the file (if available) - * @param isNegativeOffset Whether this is a tail operation - * @returns Enhanced status message string - */ -function generateEnhancedStatusMessage( - readLines: number, - offset: number, - totalLines?: number, - isNegativeOffset: boolean = false -): string { - if (isNegativeOffset) { - // For tail operations (negative offset) - if (totalLines !== undefined) { - return `[Reading last ${readLines} lines (total: ${totalLines} lines)]`; - } else { - return `[Reading last ${readLines} lines]`; - } - } else { - // For normal reads (positive offset) - if (totalLines !== undefined) { - const endLine = offset + readLines; - const remainingLines = Math.max(0, totalLines - endLine); - - if (offset === 0) { - return `[Reading ${readLines} lines from start (total: ${totalLines} lines, ${remainingLines} remaining)]`; - } else { - return `[Reading ${readLines} lines from line ${offset} (total: ${totalLines} lines, ${remainingLines} remaining)]`; - } - } else { - // Fallback when total lines unknown - if (offset === 0) { - return `[Reading ${readLines} lines from start]`; - } else { - return `[Reading ${readLines} lines from line ${offset}]`; - } - } - } -} - -/** - * Read file content using smart positioning for optimal performance - * @param filePath Path to the file (already validated) - * @param offset Starting line number (negative for tail behavior) - * @param length Maximum number of lines to read - * @param mimeType MIME type of the file - * @param includeStatusMessage Whether to include status headers (default: true) - * @returns File result with content - */ -async function readFileWithSmartPositioning(filePath: string, offset: number, length: number, mimeType: string, includeStatusMessage: boolean = true): Promise { - const stats = await fs.stat(filePath); - const fileSize = stats.size; - - // Check if the file is binary (but allow images to pass through) - const { isImage } = await getMimeTypeInfo(filePath); - if (!isImage) { - const isBinary = await isBinaryFile(filePath); - if (isBinary) { - // Return instructions instead of trying to read binary content - const instructions = getBinaryFileInstructions(filePath, mimeType); - throw new Error(instructions); - } - } - - // Get total line count for enhanced status messages (only for smaller files) - const totalLines = await getFileLineCount(filePath); - - // For negative offsets (tail behavior), use reverse reading - if (offset < 0) { - const requestedLines = Math.abs(offset); - - if (fileSize > FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD && requestedLines <= READ_PERFORMANCE_THRESHOLDS.SMALL_READ_THRESHOLD) { - // Use efficient reverse reading for large files with small tail requests - return await readLastNLinesReverse(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); - } else { - // Use readline circular buffer for other cases - return await readFromEndWithReadline(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); - } - } - - // For positive offsets - else { - // For small files or reading from start, use simple readline - if (fileSize < FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD || offset === 0) { - return await readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); - } - - // For large files with middle/end reads, try to estimate position - else { - // If seeking deep into file, try byte estimation - if (offset > READ_PERFORMANCE_THRESHOLDS.DEEP_OFFSET_THRESHOLD) { - return await readFromEstimatedPosition(filePath, offset, length, mimeType, includeStatusMessage, totalLines); - } else { - return await readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); - } - } - } -} - -/** - * Read last N lines efficiently by reading file backwards in chunks - */ -async function readLastNLinesReverse(filePath: string, n: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - const fd = await fs.open(filePath, 'r'); - try { - const stats = await fd.stat(); - const fileSize = stats.size; - - let position = fileSize; - let lines: string[] = []; - let partialLine = ''; - - while (position > 0 && lines.length < n) { - const readSize = Math.min(READ_PERFORMANCE_THRESHOLDS.CHUNK_SIZE, position); - position -= readSize; - - const buffer = Buffer.alloc(readSize); - await fd.read(buffer, 0, readSize, position); - - const chunk = buffer.toString('utf-8'); - const text = chunk + partialLine; - const chunkLines = text.split('\n'); - - partialLine = chunkLines.shift() || ''; - lines = chunkLines.concat(lines); - } - - // Add the remaining partial line if we reached the beginning - if (position === 0 && partialLine) { - lines.unshift(partialLine); - } - - const result = lines.slice(-n); // Get exactly n lines - const content = includeStatusMessage - ? `${generateEnhancedStatusMessage(result.length, -n, fileTotalLines, true)}\n\n${result.join('\n')}` - : result.join('\n'); - - return { content, mimeType, metadata: { isImage: false } }; - } finally { - await fd.close(); - } -} - -/** - * Read from end using readline with circular buffer - */ -async function readFromEndWithReadline(filePath: string, requestedLines: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - const rl = createInterface({ - input: createReadStream(filePath), - crlfDelay: Infinity - }); - - const buffer: string[] = new Array(requestedLines); - let bufferIndex = 0; - let totalLines = 0; - - for await (const line of rl) { - buffer[bufferIndex] = line; - bufferIndex = (bufferIndex + 1) % requestedLines; - totalLines++; - } - - rl.close(); - - // Extract lines in correct order - let result: string[]; - if (totalLines >= requestedLines) { - result = [ - ...buffer.slice(bufferIndex), - ...buffer.slice(0, bufferIndex) - ].filter(line => line !== undefined); - } else { - result = buffer.slice(0, totalLines); - } - - const content = includeStatusMessage - ? `${generateEnhancedStatusMessage(result.length, -requestedLines, fileTotalLines, true)}\n\n${result.join('\n')}` - : result.join('\n'); - return { content, mimeType, metadata: { isImage: false } }; -} - -/** - * Read from start/middle using readline - */ -async function readFromStartWithReadline(filePath: string, offset: number, length: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - const rl = createInterface({ - input: createReadStream(filePath), - crlfDelay: Infinity - }); - - const result: string[] = []; - let lineNumber = 0; - - for await (const line of rl) { - if (lineNumber >= offset && result.length < length) { - result.push(line); - } - if (result.length >= length) break; // Early exit optimization - lineNumber++; - } - - rl.close(); - - if (includeStatusMessage) { - const statusMessage = generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false); - const content = `${statusMessage}\n\n${result.join('\n')}`; - return { content, mimeType, metadata: { isImage: false } }; - } else { - const content = result.join('\n'); - return { content, mimeType, metadata: { isImage: false } }; - } -} - -/** - * Read from estimated byte position for very large files - */ -async function readFromEstimatedPosition(filePath: string, offset: number, length: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - // First, do a quick scan to estimate lines per byte - const rl = createInterface({ - input: createReadStream(filePath), - crlfDelay: Infinity - }); - - let sampleLines = 0; - let bytesRead = 0; - - - - for await (const line of rl) { - bytesRead += Buffer.byteLength(line, 'utf-8') + 1; // +1 for newline - sampleLines++; - if (bytesRead >= READ_PERFORMANCE_THRESHOLDS.SAMPLE_SIZE) break; - } - - rl.close(); - - if (sampleLines === 0) { - // Fallback to simple read - return await readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, fileTotalLines); - } - - // Estimate average line length and seek position - const avgLineLength = bytesRead / sampleLines; - const estimatedBytePosition = Math.floor(offset * avgLineLength); - - // Create a new stream starting from estimated position - const fd = await fs.open(filePath, 'r'); - try { - const stats = await fd.stat(); - const startPosition = Math.min(estimatedBytePosition, stats.size); - - const stream = createReadStream(filePath, { start: startPosition }); - const rl2 = createInterface({ - input: stream, - crlfDelay: Infinity - }); - - const result: string[] = []; - let lineCount = 0; - let firstLineSkipped = false; - - for await (const line of rl2) { - // Skip first potentially partial line if we didn't start at beginning - if (!firstLineSkipped && startPosition > 0) { - firstLineSkipped = true; - continue; - } - - if (result.length < length) { - result.push(line); - } else { - break; - } - lineCount++; - } - - rl2.close(); - - const content = includeStatusMessage - ? `${generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false)}\n\n${result.join('\n')}` - : result.join('\n'); - return { content, mimeType, metadata: { isImage: false } }; - } finally { - await fd.close(); - } -} - /** * Read file content from the local filesystem * @param filePath Path to the file @@ -680,8 +335,8 @@ export async function readFileFromDisk( // Use withTimeout to handle potential hangs const readOperation = async () => { - // Get appropriate handler for this file type - const handler = getFileHandler(validPath); + // Get appropriate handler for this file type (async - includes binary detection) + const handler = await getFileHandler(validPath); // Use handler to read the file const result = await handler.read(validPath, { @@ -783,7 +438,7 @@ export async function readFileInternal(filePath: string, offset: number = 0, len } // Handle offset/length by splitting on line boundaries while preserving line endings - const lines = splitLinesPreservingEndings(content); + const lines = TextFileHandler.splitLinesPreservingEndings(content); // Apply offset and length const selectedLines = lines.slice(offset, offset + length); @@ -792,47 +447,6 @@ export async function readFileInternal(filePath: string, offset: number = 0, len return selectedLines.join(''); } -/** - * Split text into lines while preserving original line endings with each line - * @param content The text content to split - * @returns Array of lines, each including its original line ending - */ -function splitLinesPreservingEndings(content: string): string[] { - if (!content) return ['']; - - const lines: string[] = []; - let currentLine = ''; - - for (let i = 0; i < content.length; i++) { - const char = content[i]; - currentLine += char; - - // Check for line ending patterns - if (char === '\n') { - // LF or end of CRLF - lines.push(currentLine); - currentLine = ''; - } else if (char === '\r') { - // Could be CR or start of CRLF - if (i + 1 < content.length && content[i + 1] === '\n') { - // It's CRLF, include the \n as well - currentLine += content[i + 1]; - i++; // Skip the \n in next iteration - } - // Either way, we have a complete line - lines.push(currentLine); - currentLine = ''; - } - } - - // Handle any remaining content (file not ending with line ending) - if (currentLine) { - lines.push(currentLine); - } - - return lines; -} - export async function writeFile(filePath: string, content: string, mode: 'rewrite' | 'append' = 'rewrite'): Promise { const validPath = await validatePath(filePath); @@ -841,7 +455,7 @@ export async function writeFile(filePath: string, content: string, mode: 'rewrit // Calculate content metrics const contentBytes = Buffer.from(content).length; - const lineCount = countLines(content); + const lineCount = TextFileHandler.countLines(content); // Capture file extension and operation details in telemetry without capturing the file path capture('server_write_file', { @@ -851,8 +465,8 @@ export async function writeFile(filePath: string, content: string, mode: 'rewrit lineCount: lineCount }); - // Get appropriate handler for this file type - const handler = getFileHandler(validPath); + // Get appropriate handler for this file type (async - includes binary detection) + const handler = await getFileHandler(validPath); // Use handler to write the file await handler.write(validPath, content, mode); @@ -1113,8 +727,8 @@ export async function getFileInfo(filePath: string): Promise metadata: undefined as Record | undefined, }; - // Get appropriate handler for this file type - const handler = getFileHandler(validPath); + // Get appropriate handler for this file type (async - includes binary detection) + const handler = await getFileHandler(validPath); // Use handler to get file info, with fallback let fileInfo; diff --git a/src/utils/files/base.ts b/src/utils/files/base.ts index a36d8b2e..0a750bed 100644 --- a/src/utils/files/base.ts +++ b/src/utils/files/base.ts @@ -58,9 +58,9 @@ export interface FileHandler { /** * Check if this handler can handle the given file * @param path File path - * @returns true if this handler supports this file type + * @returns true if this handler supports this file type (can be async for content-based checks) */ - canHandle(path: string): boolean; + canHandle(path: string): boolean | Promise; } // ============================================================================ diff --git a/src/utils/files/binary.ts b/src/utils/files/binary.ts index a934e833..245bb01f 100644 --- a/src/utils/files/binary.ts +++ b/src/utils/files/binary.ts @@ -1,11 +1,13 @@ /** * Binary file handler - * Catch-all handler for unsupported binary files + * Handles binary files that aren't supported by other handlers (Excel, Image) + * Uses isBinaryFile for content-based detection * Returns instructions to use start_process with appropriate tools */ import fs from "fs/promises"; import path from "path"; +import { isBinaryFile } from 'isbinaryfile'; import { FileHandler, ReadOptions, @@ -15,12 +17,17 @@ import { /** * Binary file handler implementation - * This is a catch-all handler for binary files that aren't supported by other handlers + * Uses content-based detection via isBinaryFile */ export class BinaryFileHandler implements FileHandler { - canHandle(path: string): boolean { - // Binary handler is the catch-all - handles everything not handled by other handlers - return true; + async canHandle(filePath: string): Promise { + // Content-based binary detection using isBinaryFile + try { + return await isBinaryFile(filePath); + } catch (error) { + // If we can't check (file doesn't exist, etc.), don't handle it + return false; + } } async read(filePath: string, options?: ReadOptions): Promise { @@ -62,112 +69,11 @@ export class BinaryFileHandler implements FileHandler { */ private getBinaryInstructions(filePath: string): string { const fileName = path.basename(filePath); - const ext = path.extname(filePath).toLowerCase(); - - // Get MIME type suggestion based on extension - const mimeType = this.guessMimeType(ext); - - let specificGuidance = ''; - - // Provide specific guidance based on file type - switch (ext) { - case '.pdf': - specificGuidance = ` -PDF FILES: -- Python: PyPDF2, pdfplumber - start_process("python -i") - interact_with_process(pid, "import pdfplumber") - interact_with_process(pid, "pdf = pdfplumber.open('${filePath}')") - interact_with_process(pid, "print(pdf.pages[0].extract_text())") - -- Node.js: pdf-parse - start_process("node -i") - interact_with_process(pid, "const pdf = require('pdf-parse')")`; - break; - case '.doc': - case '.docx': - specificGuidance = ` -WORD DOCUMENTS: -- Python: python-docx - start_process("python -i") - interact_with_process(pid, "import docx") - interact_with_process(pid, "doc = docx.Document('${filePath}')") - interact_with_process(pid, "for para in doc.paragraphs: print(para.text)") + return `Cannot read binary file as text: ${fileName} -- Node.js: mammoth - start_process("node -i") - interact_with_process(pid, "const mammoth = require('mammoth')")`; - break; - - case '.zip': - case '.tar': - case '.gz': - specificGuidance = ` -ARCHIVE FILES: -- Python: zipfile, tarfile - start_process("python -i") - interact_with_process(pid, "import zipfile") - interact_with_process(pid, "with zipfile.ZipFile('${filePath}') as z: print(z.namelist())") - -- Command-line: - start_process("unzip -l ${filePath}") # For ZIP files - start_process("tar -tzf ${filePath}") # For TAR files`; - break; - - case '.db': - case '.sqlite': - case '.sqlite3': - specificGuidance = ` -SQLITE DATABASES: -- Python: sqlite3 - start_process("python -i") - interact_with_process(pid, "import sqlite3") - interact_with_process(pid, "conn = sqlite3.connect('${filePath}')") - interact_with_process(pid, "cursor = conn.cursor()") - interact_with_process(pid, "cursor.execute('SELECT * FROM sqlite_master')") - -- Command-line: - start_process("sqlite3 ${filePath} '.tables'")`; - break; - - default: - specificGuidance = ` -GENERIC BINARY FILES: -- Use appropriate libraries based on file type -- Python libraries: Check PyPI for ${ext} support -- Node.js libraries: Check npm for ${ext} support -- Command-line tools: Use file-specific utilities`; - } - - return `Cannot read binary file as text: ${fileName} (${mimeType}) - -Use start_process + interact_with_process to analyze binary files with appropriate tools. -${specificGuidance} +Use start_process + interact_with_process to analyze binary files with appropriate tools (Node.js or Python libraries, command-line utilities, etc.). The read_file tool only handles text files, images, and Excel files.`; } - - /** - * Guess MIME type based on file extension - */ - private guessMimeType(ext: string): string { - const mimeTypes: { [key: string]: string } = { - '.pdf': 'application/pdf', - '.doc': 'application/msword', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.zip': 'application/zip', - '.tar': 'application/x-tar', - '.gz': 'application/gzip', - '.db': 'application/x-sqlite3', - '.sqlite': 'application/x-sqlite3', - '.sqlite3': 'application/x-sqlite3', - '.mp3': 'audio/mpeg', - '.mp4': 'video/mp4', - '.avi': 'video/x-msvideo', - '.mkv': 'video/x-matroska', - }; - - return mimeTypes[ext] || 'application/octet-stream'; - } } diff --git a/src/utils/files/factory.ts b/src/utils/files/factory.ts index bf84ab9a..336bc1f1 100644 --- a/src/utils/files/factory.ts +++ b/src/utils/files/factory.ts @@ -1,6 +1,9 @@ /** * Factory pattern for creating appropriate file handlers * Routes file operations to the correct handler based on file type + * + * Each handler implements canHandle() which can be sync (extension-based) + * or async (content-based like BinaryFileHandler using isBinaryFile) */ import { FileHandler } from './base.js'; @@ -10,54 +13,68 @@ import { BinaryFileHandler } from './binary.js'; import { ExcelFileHandler } from './excel.js'; // Singleton instances of each handler -let handlers: FileHandler[] | null = null; +let excelHandler: ExcelFileHandler | null = null; +let imageHandler: ImageFileHandler | null = null; +let textHandler: TextFileHandler | null = null; +let binaryHandler: BinaryFileHandler | null = null; /** * Initialize handlers (lazy initialization) */ -function initializeHandlers(): FileHandler[] { - if (handlers) { - return handlers; - } +function getExcelHandler(): ExcelFileHandler { + if (!excelHandler) excelHandler = new ExcelFileHandler(); + return excelHandler; +} - handlers = [ - // Order matters! More specific handlers first - new ExcelFileHandler(), // Check Excel first (before binary) - new ImageFileHandler(), // Then images - new TextFileHandler(), // Then text (handles most files) - new BinaryFileHandler(), // Finally binary (catch-all) - ]; +function getImageHandler(): ImageFileHandler { + if (!imageHandler) imageHandler = new ImageFileHandler(); + return imageHandler; +} - return handlers; +function getTextHandler(): TextFileHandler { + if (!textHandler) textHandler = new TextFileHandler(); + return textHandler; +} + +function getBinaryHandler(): BinaryFileHandler { + if (!binaryHandler) binaryHandler = new BinaryFileHandler(); + return binaryHandler; } /** * Get the appropriate file handler for a given file path * - * This function checks each handler in priority order and returns the first - * handler that can handle the file type. + * Each handler's canHandle() determines if it can process the file. + * Extension-based handlers (Excel, Image) return sync boolean. + * BinaryFileHandler uses async isBinaryFile for content-based detection. * * Priority order: - * 1. Excel files (xlsx, xls, xlsm) - * 2. Image files (png, jpg, gif, webp) - * 3. Text files (most other files) - * 4. Binary files (catch-all for unsupported formats) + * 1. Excel files (xlsx, xls, xlsm) - extension based + * 2. Image files (png, jpg, gif, webp) - extension based + * 3. Binary files - content-based detection via isBinaryFile + * 4. Text files (default) * - * @param path File path (can be before or after validation) + * @param filePath File path to get handler for * @returns FileHandler instance that can handle this file */ -export function getFileHandler(path: string): FileHandler { - const allHandlers = initializeHandlers(); +export async function getFileHandler(filePath: string): Promise { + // Check Excel first (extension-based, sync) + if (getExcelHandler().canHandle(filePath)) { + return getExcelHandler(); + } + + // Check Image (extension-based, sync - images are binary but handled specially) + if (getImageHandler().canHandle(filePath)) { + return getImageHandler(); + } - // Try each handler in order - for (const handler of allHandlers) { - if (handler.canHandle(path)) { - return handler; - } + // Check Binary (content-based, async via isBinaryFile) + if (await getBinaryHandler().canHandle(filePath)) { + return getBinaryHandler(); } - // Fallback to binary handler (should never reach here due to binary catch-all) - return allHandlers[allHandlers.length - 1]; + // Default to text handler + return getTextHandler(); } /** diff --git a/src/utils/files/text.ts b/src/utils/files/text.ts index 14b5b844..3e00b11b 100644 --- a/src/utils/files/text.ts +++ b/src/utils/files/text.ts @@ -2,6 +2,9 @@ * Text file handler * Handles reading, writing, and editing text files * + * Binary detection is handled at the factory level (factory.ts) using isBinaryFile. + * This handler only receives files that have been confirmed as text. + * * TECHNICAL DEBT: * This handler is missing editRange() - text search/replace logic currently lives in * src/tools/edit.ts (performSearchReplace function) instead of here. @@ -11,9 +14,9 @@ */ import fs from "fs/promises"; +import path from "path"; import { createReadStream } from 'fs'; import { createInterface } from 'readline'; -import { isBinaryFile } from 'isbinaryfile'; import { FileHandler, ReadOptions, @@ -37,27 +40,22 @@ const READ_PERFORMANCE_THRESHOLDS = { /** * Text file handler implementation + * Binary detection is done at the factory level - this handler assumes file is text */ export class TextFileHandler implements FileHandler { - canHandle(path: string): boolean { - // Text handler is the default - handles most files - // Only returns false for known non-text formats (checked by other handlers) + canHandle(_path: string): boolean { + // Text handler accepts all files that pass the factory's binary check + // The factory routes binary files to BinaryFileHandler before reaching here return true; } - async read(path: string, options?: ReadOptions): Promise { + async read(filePath: string, options?: ReadOptions): Promise { const offset = options?.offset ?? 0; const length = options?.length ?? 1000; // Default from config const includeStatusMessage = options?.includeStatusMessage ?? true; - // Check if file is binary - const isBinary = await isBinaryFile(path); - if (isBinary) { - throw new Error('Cannot read binary file as text. Use appropriate handler.'); - } - - // Read with smart positioning - return this.readFileWithSmartPositioning(path, offset, length, 'text/plain', includeStatusMessage); + // Binary detection is done at factory level - just read as text + return this.readFileWithSmartPositioning(filePath, offset, length, 'text/plain', includeStatusMessage); } async write(path: string, content: string, mode: 'rewrite' | 'append' = 'rewrite'): Promise { @@ -87,7 +85,7 @@ export class TextFileHandler implements FileHandler { if (stats.isFile() && stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { try { const content = await fs.readFile(path, 'utf8'); - const lineCount = this.countLines(content); + const lineCount = TextFileHandler.countLines(content); info.metadata!.lineCount = lineCount; } catch (error) { // If reading fails, skip line count @@ -103,8 +101,9 @@ export class TextFileHandler implements FileHandler { /** * Count lines in text content + * Made static and public for use by other modules (e.g., writeFile telemetry in filesystem.ts) */ - private countLines(content: string): number { + static countLines(content: string): number { return content.split('\n').length; } @@ -116,7 +115,7 @@ export class TextFileHandler implements FileHandler { const stats = await fs.stat(filePath); if (stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { const content = await fs.readFile(filePath, 'utf8'); - return this.countLines(content); + return TextFileHandler.countLines(content); } } catch (error) { // If we can't read the file, return undefined @@ -161,8 +160,9 @@ export class TextFileHandler implements FileHandler { /** * Split text into lines while preserving line endings + * Made static and public for use by other modules (e.g., readFileInternal in filesystem.ts) */ - private splitLinesPreservingEndings(content: string): string[] { + static splitLinesPreservingEndings(content: string): string[] { if (!content) return ['']; const lines: string[] = []; From cbf76dd23447906402e62a00c1f7ef9073a87495 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Sat, 29 Nov 2025 19:24:51 +0200 Subject: [PATCH 06/17] Replace execute_node tool with node:local command and add Python detection - Remove execute_node tool in favor of start_process("node:local") - Add pythonInfo to system info for LLM to check Python availability - Add node:local virtual session handling in improved-process-tools.ts - Add validatePath to edit.ts range+content branch - Fix parameter guard (range !== undefined instead of truthy) - Clean up: Remove node-handlers.ts and ExecuteNodeArgsSchema - Simplify tool descriptions, document node:local fallback The node:local approach is cleaner - LLM uses existing start_process/ interact_with_process flow instead of a separate tool. This is the final fallback when users dont have node and dont have python, we run the code on server, that's the idea. Python detection lets LLM decide when to fall back to Node.js execution. --- src/handlers/index.ts | 1 - src/handlers/node-handlers.ts | 90 -------------------- src/server.ts | 61 +++----------- src/tools/edit.ts | 24 ++++-- src/tools/improved-process-tools.ts | 123 +++++++++++++++++++++++++++- src/tools/schemas.ts | 5 -- src/utils/files/factory.ts | 16 +--- src/utils/files/text.ts | 5 +- src/utils/system-info.ts | 51 +++++++++++- 9 files changed, 207 insertions(+), 169 deletions(-) delete mode 100644 src/handlers/node-handlers.ts diff --git a/src/handlers/index.ts b/src/handlers/index.ts index 10f6eebd..1ac19090 100644 --- a/src/handlers/index.ts +++ b/src/handlers/index.ts @@ -5,4 +5,3 @@ export * from './process-handlers.js'; export * from './edit-search-handlers.js'; export * from './search-handlers.js'; export * from './history-handlers.js'; -export * from './node-handlers.js'; diff --git a/src/handlers/node-handlers.ts b/src/handlers/node-handlers.ts deleted file mode 100644 index cbb76c7d..00000000 --- a/src/handlers/node-handlers.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { spawn } from 'child_process'; -import fs from 'fs/promises'; -import path from 'path'; -import { fileURLToPath } from 'url'; - -import { ExecuteNodeArgsSchema } from '../tools/schemas.js'; -import { ServerResult } from '../types.js'; - -// Get the directory where the MCP is installed (for requiring packages like exceljs) -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const mcpRoot = path.resolve(__dirname, '..', '..'); - -/** - * Handle execute_node command - * Executes Node.js code using the same Node runtime as the MCP - */ -export async function handleExecuteNode(args: unknown): Promise { - const parsed = ExecuteNodeArgsSchema.parse(args); - const { code, timeout_ms } = parsed; - - // Create temp file IN THE MCP DIRECTORY so ES module imports resolve correctly - // (ES modules resolve packages relative to file location, not NODE_PATH or cwd) - const tempFile = path.join(mcpRoot, `.mcp-exec-${Date.now()}-${Math.random().toString(36).slice(2)}.mjs`); - - // User code runs directly - imports will resolve from mcpRoot/node_modules - const wrappedCode = code; - - try { - await fs.writeFile(tempFile, wrappedCode, 'utf8'); - - const result = await new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { - const proc = spawn(process.execPath, [tempFile], { - cwd: mcpRoot, - timeout: timeout_ms - }); - - let stdout = ''; - let stderr = ''; - - proc.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - proc.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - proc.on('close', (exitCode) => { - resolve({ stdout, stderr, exitCode: exitCode ?? 1 }); - }); - - proc.on('error', (err) => { - resolve({ stdout, stderr: stderr + '\n' + err.message, exitCode: 1 }); - }); - }); - - // Clean up temp file - await fs.unlink(tempFile).catch(() => {}); - - if (result.exitCode !== 0) { - return { - content: [{ - type: "text", - text: `Execution failed (exit code ${result.exitCode}):\n${result.stderr}\n${result.stdout}` - }], - isError: true - }; - } - - return { - content: [{ - type: "text", - text: result.stdout || '(no output)' - }] - }; - - } catch (error) { - // Clean up temp file on error - await fs.unlink(tempFile).catch(() => {}); - - return { - content: [{ - type: "text", - text: `Failed to execute Node.js code: ${error instanceof Error ? error.message : String(error)}` - }], - isError: true - }; - } -} diff --git a/src/server.ts b/src/server.ts index 07875a2b..9a9648c4 100644 --- a/src/server.ts +++ b/src/server.ts @@ -46,7 +46,6 @@ import { ListSearchesArgsSchema, GetPromptsArgsSchema, GetRecentToolCallsArgsSchema, - ExecuteNodeArgsSchema, } from './tools/schemas.js'; import {getConfig, setConfigValue} from './tools/config.js'; import {getUsageStats} from './tools/usage.js'; @@ -266,7 +265,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { - Text: Uses offset/length for line-based pagination - Excel (.xlsx, .xls, .xlsm): Returns JSON 2D array * Use sheet param: name (string) or index (number, 0-based) - * Use range param: A1 notation, e.g., "A1:D100" + * Use range param: ALWAYS use FROM:TO format (e.g., "A1:D100", "C1:C1", "B2:B50") * offset/length work as row pagination (optional fallback) - Images (PNG, JPEG, GIF, WebP): Base64 encoded viewable content @@ -596,7 +595,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { EXCEL FILES (.xlsx, .xls, .xlsm) - Range Update mode: Takes: - file_path: Path to the Excel file - - range: "SheetName!A1:C10" or "SheetName" for whole sheet + - range: ALWAYS use FROM:TO format - "SheetName!A1:C10" or "SheetName!C1:C1" - content: 2D array, e.g., [["H1","H2"],["R1","R2"]] TEXT FILES - Find/Replace mode: @@ -692,8 +691,15 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { - Which detection mechanism triggered early exit Use this to identify missed optimization opportunities and improve detection patterns. - ALWAYS USE FOR: Local file analysis, CSV processing, data exploration, system commands - NEVER USE ANALYSIS TOOL FOR: Local file access (analysis tool is browser-only and WILL FAIL) + NODE.JS FALLBACK (node:local): + When Python is unavailable or fails, use start_process("node:local") instead. + - Runs on MCP server where Node.js is guaranteed + - interact_with_process(pid, "complete self-contained script") + - STATELESS: Each call is fresh - include ALL imports/setup/processing in ONE call + - Use ES module imports: import ExcelJS from 'exceljs' + - ExcelJS available for Excel files (NOT xlsx library) + - All Node.js built-ins available (fs, path, http, crypto, etc.) + - Use console.log() for output ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, @@ -865,9 +871,9 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { name: "kill_process", description: ` Terminate a running process by PID. - + Use with caution as this will forcefully terminate the specified process. - + ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(KillProcessArgsSchema), annotations: { @@ -988,35 +994,6 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(GetPromptsArgsSchema), - }, - { - name: "execute_node", - description: ` - Execute Node.js code directly using the MCP server's Node runtime. - - PRIMARY TOOL FOR EXCEL FILES AND COMPLEX CALCULATIONS - Use this tool for ANY Excel file (.xlsx, .xls) operations and complex data calculations. - ExcelJS library is built-in and ready to use. - - Code runs as ES module (.mjs) with top-level await support. - Uses the same Node.js environment that runs the MCP server. - - Available libraries: ExcelJS (for Excel file manipulation), and all Node.js built-ins. - - Use cases: Excel file reading/writing/analysis, data transformations, bulk file operations, - complex calculations, JSON processing, or any task better suited to code than tools. - - Output: Use console.log() to return results. Stdout is captured and returned. - - ${PATH_GUIDANCE} - ${CMD_PREFIX_DESCRIPTION}`, - inputSchema: zodToJsonSchema(ExecuteNodeArgsSchema), - annotations: { - title: "Execute Node.js Code", - readOnlyHint: false, - destructiveHint: true, - openWorldHint: true, - }, } ]; @@ -1203,18 +1180,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request: CallToolRequest) result = await handlers.handleKillProcess(args); break; - case "execute_node": - try { - result = await handlers.handleExecuteNode(args); - } catch (error) { - capture('server_request_error', {message: `Error in execute_node handler: ${error}`}); - result = { - content: [{type: "text", text: `Error executing Node.js code: ${error instanceof Error ? error.message : String(error)}`}], - isError: true, - }; - } - break; - // Note: REPL functionality removed in favor of using general terminal commands // Filesystem tools diff --git a/src/tools/edit.ts b/src/tools/edit.ts index 590c1fb7..4411f160 100644 --- a/src/tools/edit.ts +++ b/src/tools/edit.ts @@ -371,10 +371,13 @@ export async function handleEditBlock(args: unknown): Promise { const parsed = EditBlockArgsSchema.parse(args); // Structured files: Range rewrite - if (parsed.range && parsed.content !== undefined) { + if (parsed.range !== undefined && parsed.content !== undefined) { try { + // Validate path before any filesystem operations + const validatedPath = await validatePath(parsed.file_path); + const { getFileHandler } = await import('../utils/files/factory.js'); - const handler = await getFileHandler(parsed.file_path); + const handler = await getFileHandler(validatedPath); // Parse content if it's a JSON string (AI often sends arrays as JSON strings) let content = parsed.content; @@ -388,7 +391,7 @@ export async function handleEditBlock(args: unknown): Promise { // Check if handler supports range editing if ('editRange' in handler && typeof handler.editRange === 'function') { - await handler.editRange(parsed.file_path, parsed.range, content, parsed.options); + await handler.editRange(validatedPath, parsed.range, content, parsed.options); return { content: [{ type: "text", @@ -417,9 +420,20 @@ export async function handleEditBlock(args: unknown): Promise { } // Text files: String replacement + // Validate required parameters for text replacement + if (parsed.old_string === undefined || parsed.new_string === undefined) { + return { + content: [{ + type: "text", + text: `Error: Text replacement requires both old_string and new_string parameters` + }], + isError: true + }; + } + const searchReplace = { - search: parsed.old_string!, - replace: parsed.new_string! + search: parsed.old_string, + replace: parsed.new_string }; return performSearchReplace(parsed.file_path, searchReplace, parsed.expected_replacements); diff --git a/src/tools/improved-process-tools.ts b/src/tools/improved-process-tools.ts index 3764c489..344a4fde 100644 --- a/src/tools/improved-process-tools.ts +++ b/src/tools/improved-process-tools.ts @@ -4,9 +4,91 @@ import { StartProcessArgsSchema, ReadProcessOutputArgsSchema, InteractWithProces import { capture } from "../utils/capture.js"; import { ServerResult } from '../types.js'; import { analyzeProcessState, cleanProcessOutput, formatProcessStateMessage, ProcessState } from '../utils/process-detection.js'; -import { getSystemInfo } from '../utils/system-info.js'; import * as os from 'os'; import { configManager } from '../config-manager.js'; +import { spawn } from 'child_process'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +// Get the directory where the MCP is installed (for ES module imports) +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const mcpRoot = path.resolve(__dirname, '..', '..'); + +// Track virtual Node sessions (PIDs that are actually Node fallback sessions) +const virtualNodeSessions = new Map(); +let virtualPidCounter = -1000; // Use negative PIDs for virtual sessions + +/** + * Execute Node.js code via temp file (fallback when Python unavailable) + * Creates temp .mjs file in MCP directory for ES module import access + */ +async function executeNodeCode(code: string, timeout_ms: number = 30000): Promise { + const tempFile = path.join(mcpRoot, `.mcp-exec-${Date.now()}-${Math.random().toString(36).slice(2)}.mjs`); + + try { + await fs.writeFile(tempFile, code, 'utf8'); + + const result = await new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { + const proc = spawn(process.execPath, [tempFile], { + cwd: mcpRoot, + timeout: timeout_ms + }); + + let stdout = ''; + let stderr = ''; + + proc.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + proc.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + proc.on('close', (exitCode) => { + resolve({ stdout, stderr, exitCode: exitCode ?? 1 }); + }); + + proc.on('error', (err) => { + resolve({ stdout, stderr: stderr + '\n' + err.message, exitCode: 1 }); + }); + }); + + // Clean up temp file + await fs.unlink(tempFile).catch(() => {}); + + if (result.exitCode !== 0) { + return { + content: [{ + type: "text", + text: `Execution failed (exit code ${result.exitCode}):\n${result.stderr}\n${result.stdout}` + }], + isError: true + }; + } + + return { + content: [{ + type: "text", + text: result.stdout || '(no output)' + }] + }; + + } catch (error) { + // Clean up temp file on error + await fs.unlink(tempFile).catch(() => {}); + + return { + content: [{ + type: "text", + text: `Failed to execute Node.js code: ${error instanceof Error ? error.message : String(error)}` + }], + isError: true + }; + } +} /** * Start a new process (renamed from execute_command) @@ -42,6 +124,31 @@ export async function startProcess(args: unknown): Promise { }; } + const commandToRun = parsed.data.command; + + // Handle node:local - runs Node.js code directly on MCP server + if (commandToRun.trim() === 'node:local') { + const virtualPid = virtualPidCounter--; + virtualNodeSessions.set(virtualPid, { timeout_ms: parsed.data.timeout_ms || 30000 }); + + return { + content: [{ + type: "text", + text: `Node.js session started with PID ${virtualPid} (MCP server execution) + + IMPORTANT: Each interact_with_process call runs as a FRESH script. + State is NOT preserved between calls. Include ALL code in ONE call: + - imports, file reading, processing, and output together. + + Available libraries: + - ExcelJS for Excel files: import ExcelJS from 'exceljs' + - All Node.js built-ins: fs, path, http, crypto, etc. + +🔄 Ready for code - send complete self-contained script via interact_with_process.` + }], + }; + } + let shellUsed: string | undefined = parsed.data.shell; if (!shellUsed) { @@ -61,7 +168,7 @@ export async function startProcess(args: unknown): Promise { } const result = await terminalManager.executeCommand( - parsed.data.command, + commandToRun, parsed.data.timeout_ms, shellUsed, parsed.data.verbose_timing || false @@ -419,6 +526,18 @@ export async function interactWithProcess(args: unknown): Promise verbose_timing = false } = parsed.data; + // Check if this is a virtual Node session (Python fallback) + if (virtualNodeSessions.has(pid)) { + const session = virtualNodeSessions.get(pid)!; + capture('server_interact_with_process_node_fallback', { + pid: pid, + inputLength: input.length + }); + + // Execute code via temp file approach + return executeNodeCode(input, session.timeout_ms); + } + // Timing telemetry const startTime = Date.now(); let firstOutputTime: number | undefined; diff --git a/src/tools/schemas.ts b/src/tools/schemas.ts index bf019162..d92adce8 100644 --- a/src/tools/schemas.ts +++ b/src/tools/schemas.ts @@ -164,8 +164,3 @@ export const GetRecentToolCallsArgsSchema = z.object({ since: z.string().datetime().optional(), }); -// Execute Node.js code schema -export const ExecuteNodeArgsSchema = z.object({ - code: z.string(), - timeout_ms: z.number().optional().default(30000), -}); \ No newline at end of file diff --git a/src/utils/files/factory.ts b/src/utils/files/factory.ts index 336bc1f1..e5f64a34 100644 --- a/src/utils/files/factory.ts +++ b/src/utils/files/factory.ts @@ -79,28 +79,20 @@ export async function getFileHandler(filePath: string): Promise { /** * Check if a file path is an Excel file + * Delegates to ExcelFileHandler.canHandle to avoid duplicating extension logic * @param path File path * @returns true if file is Excel format */ export function isExcelFile(path: string): boolean { - const ext = path.toLowerCase(); - return ext.endsWith('.xlsx') || ext.endsWith('.xls') || ext.endsWith('.xlsm'); + return getExcelHandler().canHandle(path); } /** * Check if a file path is an image file + * Delegates to ImageFileHandler.canHandle to avoid duplicating extension logic * @param path File path * @returns true if file is an image format */ export function isImageFile(path: string): boolean { - // This will be implemented by checking MIME type - // For now, use extension-based check - const ext = path.toLowerCase(); - return ext.endsWith('.png') || - ext.endsWith('.jpg') || - ext.endsWith('.jpeg') || - ext.endsWith('.gif') || - ext.endsWith('.webp') || - ext.endsWith('.bmp') || - ext.endsWith('.svg'); + return getImageHandler().canHandle(path); } diff --git a/src/utils/files/text.ts b/src/utils/files/text.ts index 3e00b11b..18d408f7 100644 --- a/src/utils/files/text.ts +++ b/src/utils/files/text.ts @@ -24,8 +24,9 @@ import { FileInfo } from './base.js'; -// Import constants from filesystem.ts -// These will be imported after we organize the code +// TODO: Centralize these constants with filesystem.ts to avoid silent drift +// These duplicate concepts from filesystem.ts and should be moved to a shared +// constants module (e.g., src/utils/files/constants.ts) during reorganization const FILE_SIZE_LIMITS = { LARGE_FILE_THRESHOLD: 10 * 1024 * 1024, // 10MB LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting diff --git a/src/utils/system-info.ts b/src/utils/system-info.ts index 862e2b0a..5567f29b 100644 --- a/src/utils/system-info.ts +++ b/src/utils/system-info.ts @@ -1,6 +1,7 @@ import os from 'os'; import fs from 'fs'; import path from 'path'; +import { execSync } from 'child_process'; export interface DockerMount { hostPath: string; @@ -43,6 +44,11 @@ export interface SystemInfo { path: string; npmVersion?: string; }; + pythonInfo?: { + available: boolean; + command: string; + version?: string; + }; processInfo: { pid: number; arch: string; @@ -399,13 +405,13 @@ function detectNodeInfo(): SystemInfo['nodeInfo'] { try { // Get Node.js version from current process const version = process.version.replace('v', ''); // Remove 'v' prefix - + // Get Node.js executable path from current process const path = process.execPath; - + // Get npm version from environment if available const npmVersion = process.env.npm_version; - + return { version, path, @@ -416,6 +422,39 @@ function detectNodeInfo(): SystemInfo['nodeInfo'] { } } +/** + * Detect Python installation and version and put on systeminfo.pythonInfo + */ +function detectPythonInfo(): SystemInfo['pythonInfo'] { + // Try python commands in order of preference + const pythonCommands = process.platform === 'win32' + ? ['python', 'python3', 'py'] // Windows: 'python' is common, 'py' launcher + : ['python3', 'python']; // Unix: prefer python3 + + for (const cmd of pythonCommands) { + try { + const version = execSync(`${cmd} --version`, { + encoding: 'utf8', + timeout: 5000, + stdio: ['pipe', 'pipe', 'pipe'] + }).trim(); + + // Verify it's Python 3.x + if (version.includes('Python 3')) { + return { + available: true, + command: cmd, + version: version.replace('Python ', '') + }; + } + } catch { + // Command not found or failed, try next + } + } + + return { available: false, command: '' }; +} + /** * Get comprehensive system information for tool prompts */ @@ -509,7 +548,10 @@ export function getSystemInfo(): SystemInfo { // Detect Node.js installation from current process const nodeInfo = detectNodeInfo(); - + + // Detect Python installation + const pythonInfo = detectPythonInfo(); + // Get process information const processInfo = { pid: process.pid, @@ -538,6 +580,7 @@ export function getSystemInfo(): SystemInfo { }, isDXT: !!process.env.MCP_DXT, nodeInfo, + pythonInfo, processInfo, examplePaths }; From 640942d442c8fc58d0438b5b37fc403ff2c4fd5f Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Sat, 29 Nov 2025 19:30:15 +0200 Subject: [PATCH 07/17] Replace execute_node tool with node:local command and add Python detection --- src/server.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/server.ts b/src/server.ts index 9a9648c4..dd137dc5 100644 --- a/src/server.ts +++ b/src/server.ts @@ -701,6 +701,9 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { - All Node.js built-ins available (fs, path, http, crypto, etc.) - Use console.log() for output + ALWAYS USE FOR: Local file analysis, CSV processing, data exploration, system commands + NEVER USE ANALYSIS TOOL FOR: Local file access (analysis tool is browser-only and WILL FAIL) + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(StartProcessArgsSchema), From f4f817c991f411b8b99f0c0b32d1308a973cdee4 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Sat, 29 Nov 2025 19:39:01 +0200 Subject: [PATCH 08/17] Fix virtual session handling for node:local --- src/tools/improved-process-tools.ts | 49 +++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/src/tools/improved-process-tools.ts b/src/tools/improved-process-tools.ts index 344a4fde..ace21706 100644 --- a/src/tools/improved-process-tools.ts +++ b/src/tools/improved-process-tools.ts @@ -526,7 +526,7 @@ export async function interactWithProcess(args: unknown): Promise verbose_timing = false } = parsed.data; - // Check if this is a virtual Node session (Python fallback) + // Check if this is a virtual Node session (node:local) if (virtualNodeSessions.has(pid)) { const session = virtualNodeSessions.get(pid)!; capture('server_interact_with_process_node_fallback', { @@ -535,7 +535,9 @@ export async function interactWithProcess(args: unknown): Promise }); // Execute code via temp file approach - return executeNodeCode(input, session.timeout_ms); + // Respect per-call timeout if provided, otherwise use session default + const effectiveTimeout = timeout_ms ?? session.timeout_ms; + return executeNodeCode(input, effectiveTimeout); } // Timing telemetry @@ -758,13 +760,26 @@ export async function forceTerminate(args: unknown): Promise { }; } - const success = terminalManager.forceTerminate(parsed.data.pid); + const pid = parsed.data.pid; + + // Handle virtual Node.js sessions (node:local) + if (virtualNodeSessions.has(pid)) { + virtualNodeSessions.delete(pid); + return { + content: [{ + type: "text", + text: `Cleared virtual Node.js session ${pid}` + }], + }; + } + + const success = terminalManager.forceTerminate(pid); return { content: [{ type: "text", text: success - ? `Successfully initiated termination of session ${parsed.data.pid}` - : `No active session found for PID ${parsed.data.pid}` + ? `Successfully initiated termination of session ${pid}` + : `No active session found for PID ${pid}` }], }; } @@ -774,14 +789,30 @@ export async function forceTerminate(args: unknown): Promise { */ export async function listSessions(): Promise { const sessions = terminalManager.listActiveSessions(); + + // Include virtual Node.js sessions + const virtualSessions = Array.from(virtualNodeSessions.entries()).map(([pid, session]) => ({ + pid, + type: 'node:local', + timeout_ms: session.timeout_ms + })); + + const realSessionsText = sessions.map(s => + `PID: ${s.pid}, Blocked: ${s.isBlocked}, Runtime: ${Math.round(s.runtime / 1000)}s` + ); + + const virtualSessionsText = virtualSessions.map(s => + `PID: ${s.pid} (node:local), Timeout: ${s.timeout_ms}ms` + ); + + const allSessions = [...realSessionsText, ...virtualSessionsText]; + return { content: [{ type: "text", - text: sessions.length === 0 + text: allSessions.length === 0 ? 'No active sessions' - : sessions.map(s => - `PID: ${s.pid}, Blocked: ${s.isBlocked}, Runtime: ${Math.round(s.runtime / 1000)}s` - ).join('\n') + : allSessions.join('\n') }], }; } \ No newline at end of file From 72e0e4e0983fffc8ceaf15baf7699cc057db0ac5 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Tue, 2 Dec 2025 22:57:39 +0200 Subject: [PATCH 09/17] make the REPL process fallback hierarchy a bit stornger --- src/handlers/filesystem-handlers.ts | 8 +++++++- src/server.ts | 30 ++++++++++++----------------- src/tools/schemas.ts | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/handlers/filesystem-handlers.ts b/src/handlers/filesystem-handlers.ts index 09289dcf..c219641e 100644 --- a/src/handlers/filesystem-handlers.ts +++ b/src/handlers/filesystem-handlers.ts @@ -60,11 +60,17 @@ export async function handleReadFile(args: unknown): Promise { const defaultLimit = config.fileReadLineLimit ?? 1000; + // Convert sheet parameter: numeric strings become numbers for Excel index access + let sheetParam: string | number | undefined = parsed.sheet; + if (parsed.sheet !== undefined && /^\d+$/.test(parsed.sheet)) { + sheetParam = parseInt(parsed.sheet, 10); + } + const options: ReadOptions = { isUrl: parsed.isUrl, offset: parsed.offset ?? 0, length: parsed.length ?? defaultLimit, - sheet: parsed.sheet, + sheet: sheetParam, range: parsed.range }; const fileResult = await readFile(parsed.path, options); diff --git a/src/server.ts b/src/server.ts index dd137dc5..a39f9c09 100644 --- a/src/server.ts +++ b/src/server.ts @@ -264,8 +264,8 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { FORMAT HANDLING (by extension): - Text: Uses offset/length for line-based pagination - Excel (.xlsx, .xls, .xlsm): Returns JSON 2D array - * Use sheet param: name (string) or index (number, 0-based) - * Use range param: ALWAYS use FROM:TO format (e.g., "A1:D100", "C1:C1", "B2:B50") + * sheet: "Sheet1" (name) or "0" (index as string, 0-based) + * range: ALWAYS use FROM:TO format (e.g., "A1:D100", "C1:C1", "B2:B50") * offset/length work as row pagination (optional fallback) - Images (PNG, JPEG, GIF, WebP): Base64 encoded viewable content @@ -657,7 +657,8 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { COMMON FILE ANALYSIS PATTERNS: • start_process("python3 -i") → Python REPL for data analysis (RECOMMENDED) - • start_process("node -i") → Node.js for JSON processing + • start_process("node -i") → Node.js REPL for JSON processing + • start_process("node:local") → Node.js on MCP server (stateless, ES imports, all code in one call) • start_process("cut -d',' -f1 file.csv | sort | uniq -c") → Quick CSV analysis • start_process("wc -l /path/file.csv") → Line counting • start_process("head -10 /path/file.csv") → File preview @@ -666,12 +667,15 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { For PDF, Excel, Word, archives, databases, and other binary formats, use process tools with appropriate libraries or command-line utilities. INTERACTIVE PROCESSES FOR DATA ANALYSIS: - 1. start_process("python3 -i") - Start Python REPL for data work - 2. start_process("node -i") - Start Node.js REPL for JSON/JS - 3. start_process("bash") - Start interactive bash shell + For code/calculations, use in this priority order: + 1. start_process("python3 -i") - Python REPL (preferred) + 2. start_process("node -i") - Node.js REPL (when Python unavailable) + 3. start_process("node:local") - Node.js fallback (when node -i fails) 4. Use interact_with_process() to send commands 5. Use read_process_output() to get responses - + When Python is unavailable, prefer Node.js over shell for calculations. + Node.js: Always use ES import syntax (import x from 'y'), not require(). + SMART DETECTION: - Detects REPL prompts (>>>, >, $, etc.) - Identifies when process is waiting for input @@ -691,16 +695,6 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { - Which detection mechanism triggered early exit Use this to identify missed optimization opportunities and improve detection patterns. - NODE.JS FALLBACK (node:local): - When Python is unavailable or fails, use start_process("node:local") instead. - - Runs on MCP server where Node.js is guaranteed - - interact_with_process(pid, "complete self-contained script") - - STATELESS: Each call is fresh - include ALL imports/setup/processing in ONE call - - Use ES module imports: import ExcelJS from 'exceljs' - - ExcelJS available for Excel files (NOT xlsx library) - - All Node.js built-ins available (fs, path, http, crypto, etc.) - - Use console.log() for output - ALWAYS USE FOR: Local file analysis, CSV processing, data exploration, system commands NEVER USE ANALYSIS TOOL FOR: Local file access (analysis tool is browser-only and WILL FAIL) @@ -785,7 +779,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { SUPPORTED REPLs: - Python: python3 -i (RECOMMENDED for data analysis) - - Node.js: node -i + - Node.js: node -i - R: R - Julia: julia - Shell: bash, zsh diff --git a/src/tools/schemas.ts b/src/tools/schemas.ts index d92adce8..73b4bfc7 100644 --- a/src/tools/schemas.ts +++ b/src/tools/schemas.ts @@ -47,7 +47,7 @@ export const ReadFileArgsSchema = z.object({ isUrl: z.boolean().optional().default(false), offset: z.number().optional().default(0), length: z.number().optional().default(1000), - sheet: z.union([z.string(), z.number()]).optional(), + sheet: z.string().optional(), // String only for MCP client compatibility (Cursor doesn't support union types in JSON Schema) range: z.string().optional(), options: z.record(z.any()).optional() }); From 15aa322a0aba040c171968b696c74369099dc44b Mon Sep 17 00:00:00 2001 From: Eduard Ruzga Date: Fri, 5 Dec 2025 13:18:52 +0200 Subject: [PATCH 10/17] Add Excel file handling support with file handler architecture - Implement ExcelFileHandler for .xlsx/.xls/.xlsm files using xlsx library - Add FileHandler interface with read(), write(), getInfo(), editRange() methods - Create file handler factory for automatic handler selection by extension - Support Excel-specific features: sheet selection, range queries, offset/length - Add edit_block range mode for Excel cell/range updates - Refactor handleGetFileInfo to be generic (handles any file type's nested data) - Remove .describe() calls from schemas (WriteFileArgsSchema, EditBlockArgsSchema) - Add technical debt comments documenting text edit architectural inconsistency # Conflicts: # package-lock.json # package.json # src/handlers/filesystem-handlers.ts # src/server.ts # src/tools/filesystem.ts # test/test_output/node_repl_debug.txt --- manifest.template.json | 6 +- package.json | 1 + src/handlers/filesystem-handlers.ts | 78 +++-- src/handlers/index.ts | 1 + src/handlers/node-handlers.ts | 90 ++++++ src/search-manager.ts | 208 +++++++++++- src/server.ts | 96 ++++-- src/tools/edit.ts | 86 ++++- src/tools/filesystem.ts | 230 ++++++------- src/tools/schemas.ts | 30 +- src/utils/files/base.ts | 219 +++++++++++++ src/utils/files/binary.ts | 173 ++++++++++ src/utils/files/excel.ts | 486 ++++++++++++++++++++++++++++ src/utils/files/factory.ts | 91 ++++++ src/utils/files/image.ts | 93 ++++++ src/utils/files/index.ts | 16 + src/utils/files/pdf.ts | 147 +++++++++ src/utils/files/text.ts | 441 +++++++++++++++++++++++++ test/test-excel-files.js | 369 +++++++++++++++++++++ test/test-file-handlers.js | 313 ++++++++++++++++++ 20 files changed, 2994 insertions(+), 180 deletions(-) create mode 100644 src/handlers/node-handlers.ts create mode 100644 src/utils/files/base.ts create mode 100644 src/utils/files/binary.ts create mode 100644 src/utils/files/excel.ts create mode 100644 src/utils/files/factory.ts create mode 100644 src/utils/files/image.ts create mode 100644 src/utils/files/index.ts create mode 100644 src/utils/files/pdf.ts create mode 100644 src/utils/files/text.ts create mode 100644 test/test-excel-files.js create mode 100644 test/test-file-handlers.js diff --git a/manifest.template.json b/manifest.template.json index 9ff08da2..4da5a1de 100644 --- a/manifest.template.json +++ b/manifest.template.json @@ -88,7 +88,7 @@ }, { "name": "edit_block", - "description": "Apply surgical text replacements to files. Make small, focused edits with minimal context for precision." + "description": "Apply surgical edits to files. Supports text replacement (old_string/new_string) with fuzzy matching for text files, and range updates (range/content) for Excel files." }, { "name": "start_process", @@ -133,6 +133,10 @@ { "name": "get_prompts", "description": "Browse and retrieve curated Desktop Commander prompts for various tasks and workflows." + }, + { + "name": "execute_node", + "description": "Execute Node.js code directly using the MCP server's Node runtime. Supports ES modules with top-level await." } ], "keywords": [ diff --git a/package.json b/package.json index 023d57a6..c8c35eb9 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "file-type": "^21.1.1", "glob": "^10.3.10", "isbinaryfile": "^5.0.4", + "exceljs": "^4.4.0", "md-to-pdf": "^5.2.5", "pdf-lib": "^1.17.1", "remark": "^15.0.1", diff --git a/src/handlers/filesystem-handlers.ts b/src/handlers/filesystem-handlers.ts index e4e0093f..ce41adc4 100644 --- a/src/handlers/filesystem-handlers.ts +++ b/src/handlers/filesystem-handlers.ts @@ -10,6 +10,7 @@ import { type FileResult, type MultiFileResult } from '../tools/filesystem.js'; +import type { ReadOptions } from '../utils/files/base.js'; import { ServerResult } from '../types.js'; import { withTimeout } from '../utils/withTimeout.js'; @@ -61,23 +62,23 @@ export async function handleReadFile(args: unknown): Promise { const defaultLimit = config.fileReadLineLimit ?? 1000; - // Use the provided limits or defaults - const offset = parsed.offset ?? 0; - const length = parsed.length ?? defaultLimit; - - const fileResult = await readFile(parsed.path, parsed.isUrl, offset, length); - if (fileResult.isPdf) { - const meta = fileResult.payload?.metadata; + const options: ReadOptions = { + isUrl: parsed.isUrl, + offset: parsed.offset ?? 0, + length: parsed.length ?? defaultLimit, + sheet: parsed.sheet, + range: parsed.range + }; + const fileResult = await readFile(parsed.path, options); + + // Handle PDF files + if (fileResult.metadata?.isPdf) { + const meta = fileResult.metadata; const author = meta?.author ? `, Author: ${meta?.author}` : ""; const title = meta?.title ? `, Title: ${meta?.title}` : ""; - // Use the provided limits or defaults. - // If the caller did not supply an explicit length, fall back to the configured default. - const rawArgs = args as { offset?: number; length?: number } | undefined; - const offset = rawArgs && 'offset' in rawArgs ? parsed.offset : 0; - const length = rawArgs && 'length' in rawArgs ? parsed.length : defaultLimit; - - const content = fileResult.payload?.pages?.flatMap(p => [ - ...(p.images?.map((image, i) => ({ + + const pdfContent = fileResult.metadata?.pages?.flatMap((p: any) => [ + ...(p.images?.map((image: any) => ({ type: "image", data: image.data, mimeType: image.mimeType @@ -94,11 +95,13 @@ export async function handleReadFile(args: unknown): Promise { type: "text", text: `PDF file: ${parsed.path}${author}${title} (${meta?.totalPages} pages) \n` }, - ...content + ...pdfContent ] }; } - if (fileResult.isImage) { + + // Handle image files + if (fileResult.metadata?.isImage) { // For image files, return as an image content type return { content: [ @@ -108,7 +111,7 @@ export async function handleReadFile(args: unknown): Promise { }, { type: "image", - data: fileResult.content, + data: fileResult.content.toString(), mimeType: fileResult.mimeType } ], @@ -116,7 +119,7 @@ export async function handleReadFile(args: unknown): Promise { } else { // For all other files, return as text return { - content: [{ type: "text", text: fileResult.content }], + content: [{ type: "text", text: fileResult.content.toString() }], }; } }; @@ -290,6 +293,33 @@ export async function handleMoveFile(args: unknown): Promise { } } +/** + * Format a value for display, handling objects and arrays + */ +function formatValue(value: unknown, indent: string = ''): string { + if (value === null || value === undefined) { + return String(value); + } + if (Array.isArray(value)) { + if (value.length === 0) return '[]'; + // For arrays of objects (like sheets), format each item + const items = value.map((item, i) => { + if (typeof item === 'object' && item !== null) { + const props = Object.entries(item) + .map(([k, v]) => `${k}: ${v}`) + .join(', '); + return `${indent} [${i}] { ${props} }`; + } + return `${indent} [${i}] ${item}`; + }); + return `\n${items.join('\n')}`; + } + if (typeof value === 'object') { + return JSON.stringify(value); + } + return String(value); +} + /** * Handle get_file_info command */ @@ -297,12 +327,16 @@ export async function handleGetFileInfo(args: unknown): Promise { try { const parsed = GetFileInfoArgsSchema.parse(args); const info = await getFileInfo(parsed.path); + + // Generic formatting for any file type + const formattedText = Object.entries(info) + .map(([key, value]) => `${key}: ${formatValue(value)}`) + .join('\n'); + return { content: [{ type: "text", - text: Object.entries(info) - .map(([key, value]) => `${key}: ${value}`) - .join('\n') + text: formattedText }], }; } catch (error) { diff --git a/src/handlers/index.ts b/src/handlers/index.ts index 1ac19090..10f6eebd 100644 --- a/src/handlers/index.ts +++ b/src/handlers/index.ts @@ -5,3 +5,4 @@ export * from './process-handlers.js'; export * from './edit-search-handlers.js'; export * from './search-handlers.js'; export * from './history-handlers.js'; +export * from './node-handlers.js'; diff --git a/src/handlers/node-handlers.ts b/src/handlers/node-handlers.ts new file mode 100644 index 00000000..cbb76c7d --- /dev/null +++ b/src/handlers/node-handlers.ts @@ -0,0 +1,90 @@ +import { spawn } from 'child_process'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +import { ExecuteNodeArgsSchema } from '../tools/schemas.js'; +import { ServerResult } from '../types.js'; + +// Get the directory where the MCP is installed (for requiring packages like exceljs) +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const mcpRoot = path.resolve(__dirname, '..', '..'); + +/** + * Handle execute_node command + * Executes Node.js code using the same Node runtime as the MCP + */ +export async function handleExecuteNode(args: unknown): Promise { + const parsed = ExecuteNodeArgsSchema.parse(args); + const { code, timeout_ms } = parsed; + + // Create temp file IN THE MCP DIRECTORY so ES module imports resolve correctly + // (ES modules resolve packages relative to file location, not NODE_PATH or cwd) + const tempFile = path.join(mcpRoot, `.mcp-exec-${Date.now()}-${Math.random().toString(36).slice(2)}.mjs`); + + // User code runs directly - imports will resolve from mcpRoot/node_modules + const wrappedCode = code; + + try { + await fs.writeFile(tempFile, wrappedCode, 'utf8'); + + const result = await new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { + const proc = spawn(process.execPath, [tempFile], { + cwd: mcpRoot, + timeout: timeout_ms + }); + + let stdout = ''; + let stderr = ''; + + proc.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + proc.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + proc.on('close', (exitCode) => { + resolve({ stdout, stderr, exitCode: exitCode ?? 1 }); + }); + + proc.on('error', (err) => { + resolve({ stdout, stderr: stderr + '\n' + err.message, exitCode: 1 }); + }); + }); + + // Clean up temp file + await fs.unlink(tempFile).catch(() => {}); + + if (result.exitCode !== 0) { + return { + content: [{ + type: "text", + text: `Execution failed (exit code ${result.exitCode}):\n${result.stderr}\n${result.stdout}` + }], + isError: true + }; + } + + return { + content: [{ + type: "text", + text: result.stdout || '(no output)' + }] + }; + + } catch (error) { + // Clean up temp file on error + await fs.unlink(tempFile).catch(() => {}); + + return { + content: [{ + type: "text", + text: `Failed to execute Node.js code: ${error instanceof Error ? error.message : String(error)}` + }], + isError: true + }; + } +} diff --git a/src/search-manager.ts b/src/search-manager.ts index 980f1d3c..d2af0558 100644 --- a/src/search-manager.ts +++ b/src/search-manager.ts @@ -1,8 +1,10 @@ import { spawn, ChildProcess } from 'child_process'; import path from 'path'; +import fs from 'fs/promises'; import { validatePath } from './tools/filesystem.js'; import { capture } from './utils/capture.js'; import { getRipgrepPath } from './utils/ripgrep-resolver.js'; +import { isExcelFile } from './utils/files/index.js'; export interface SearchResult { file: string; @@ -144,6 +146,27 @@ export interface SearchSessionOptions { validatedPath: validPath }); + // For content searches, also search Excel files in parallel + let excelSearchPromise: Promise | null = null; + if (options.searchType === 'content') { + excelSearchPromise = this.searchExcelFiles( + validPath, + options.pattern, + options.ignoreCase !== false, + options.maxResults, + options.filePattern // Pass filePattern to filter Excel files too + ).then(excelResults => { + // Add Excel results to session + for (const result of excelResults) { + session.results.push(result); + session.totalMatches++; + } + }).catch((err) => { + // Log Excel search errors but don't fail the whole search + capture('excel_search_error', { error: err instanceof Error ? err.message : String(err) }); + }); + } + // Wait for first chunk of data or early completion instead of fixed delay const firstChunk = new Promise(resolve => { const onData = () => { @@ -153,7 +176,9 @@ export interface SearchSessionOptions { session.process.stdout?.once('data', onData); setTimeout(resolve, 40); // cap at 40ms instead of 50-100ms }); - await firstChunk; + + // Wait for both ripgrep first chunk and Excel search + await Promise.all([firstChunk, excelSearchPromise].filter(Boolean)); return { sessionId, @@ -275,6 +300,187 @@ export interface SearchSessionOptions { })); } + /** + * Search Excel files for content matches + * Called during content search to include Excel files alongside text files + * Searches ALL sheets in each Excel file (row-wise for cross-column matching) + * + * TODO: Refactor - Extract Excel search logic to separate module (src/utils/search/excel-search.ts) + * and inject into SearchManager, similar to how file handlers are structured in src/utils/files/ + * This would allow adding other file type searches (PDF, etc.) without bloating search-manager.ts + */ + private async searchExcelFiles( + rootPath: string, + pattern: string, + ignoreCase: boolean, + maxResults?: number, + filePattern?: string + ): Promise { + const results: SearchResult[] = []; + + // Build regex for matching content + const flags = ignoreCase ? 'i' : ''; + let regex: RegExp; + try { + regex = new RegExp(pattern, flags); + } catch { + // If pattern is not valid regex, escape it for literal matching + const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + regex = new RegExp(escaped, flags); + } + + // Find Excel files recursively + let excelFiles = await this.findExcelFiles(rootPath); + + // Filter by filePattern if provided + if (filePattern) { + const patterns = filePattern.split('|').map(p => p.trim()).filter(Boolean); + excelFiles = excelFiles.filter(filePath => { + const fileName = path.basename(filePath); + return patterns.some(pat => { + // Support glob-like patterns + if (pat.includes('*')) { + const regexPat = pat.replace(/\./g, '\\.').replace(/\*/g, '.*'); + return new RegExp(`^${regexPat}$`, 'i').test(fileName); + } + // Exact match (case-insensitive) + return fileName.toLowerCase() === pat.toLowerCase(); + }); + }); + } + + // Dynamically import ExcelJS to search all sheets + const ExcelJS = await import('exceljs'); + + for (const filePath of excelFiles) { + if (maxResults && results.length >= maxResults) break; + + try { + const workbook = new ExcelJS.default.Workbook(); + await workbook.xlsx.readFile(filePath); + + // Search ALL sheets in the workbook (row-wise for speed and cross-column matching) + for (const worksheet of workbook.worksheets) { + if (maxResults && results.length >= maxResults) break; + + const sheetName = worksheet.name; + + // Iterate through rows (faster than cell-by-cell) + worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => { + if (maxResults && results.length >= maxResults) return; + + // Build a concatenated string of all cell values in the row + const rowValues: string[] = []; + row.eachCell({ includeEmpty: false }, (cell) => { + if (cell.value === null || cell.value === undefined) return; + + let cellStr: string; + if (typeof cell.value === 'object') { + if ('result' in cell.value) { + cellStr = String(cell.value.result ?? ''); + } else if ('richText' in cell.value) { + cellStr = (cell.value as any).richText.map((rt: any) => rt.text).join(''); + } else if ('text' in cell.value) { + cellStr = String((cell.value as any).text); + } else { + cellStr = String(cell.value); + } + } else { + cellStr = String(cell.value); + } + + if (cellStr.trim()) { + rowValues.push(cellStr); + } + }); + + // Join all cell values with space for cross-column matching + const rowText = rowValues.join(' '); + + if (regex.test(rowText)) { + // Extract the matching portion for display + const match = rowText.match(regex); + const matchContext = match + ? this.getMatchContext(rowText, match.index || 0, match[0].length) + : rowText.substring(0, 150); + + results.push({ + file: `${filePath}:${sheetName}!Row${rowNumber}`, + line: rowNumber, + match: matchContext, + type: 'content' + }); + } + }); + } + } catch (error) { + // Skip files that can't be read (permission issues, corrupted, etc.) + continue; + } + } + + return results; + } + + /** + * Find all Excel files in a directory recursively + */ + private async findExcelFiles(rootPath: string): Promise { + const excelFiles: string[] = []; + + async function walk(dir: string): Promise { + try { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + // Skip node_modules, .git, etc. + if (!entry.name.startsWith('.') && entry.name !== 'node_modules') { + await walk(fullPath); + } + } else if (entry.isFile() && isExcelFile(entry.name)) { + excelFiles.push(fullPath); + } + } + } catch { + // Skip directories we can't read + } + } + + // Check if rootPath is a file or directory + try { + const stats = await fs.stat(rootPath); + if (stats.isFile() && isExcelFile(rootPath)) { + return [rootPath]; + } else if (stats.isDirectory()) { + await walk(rootPath); + } + } catch { + // Path doesn't exist or can't be accessed + } + + return excelFiles; + } + + /** + * Extract context around a match for display (show surrounding text) + */ + private getMatchContext(text: string, matchStart: number, matchLength: number): string { + const contextChars = 50; // chars before and after match + const start = Math.max(0, matchStart - contextChars); + const end = Math.min(text.length, matchStart + matchLength + contextChars); + + let context = text.substring(start, end); + + // Add ellipsis if truncated + if (start > 0) context = '...' + context; + if (end < text.length) context = context + '...'; + + return context; + } + /** * Clean up completed sessions older than specified time * Called automatically by cleanup interval diff --git a/src/server.ts b/src/server.ts index 4d20156b..62a9f822 100644 --- a/src/server.ts +++ b/src/server.ts @@ -263,17 +263,18 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { Can fetch content from URLs when isUrl parameter is set to true (URLs are always read in full regardless of offset/length). - Handles text files normally and image files are returned as viewable images. - Recognized image types: PNG, JPEG, GIF, WebP. - - PDF Support: - - Automatically extracts text content as markdown - - Preserves basic document structure with paragraph breaks - - Special handling for 'offset' and 'length': - * 'offset': Start page number (0-based, e.g., 0 is page 1) - * 'length': Number of pages to read - * Negative offsets work similarly (e.g., -1 is the last page) - + FORMAT HANDLING (by extension): + - Text: Uses offset/length for line-based pagination + - Excel (.xlsx, .xls, .xlsm): Returns JSON 2D array + * Use sheet param: name (string) or index (number, 0-based) + * Use range param: ALWAYS use FROM:TO format (e.g., "A1:D100", "C1:C1", "B2:B50") + * offset/length work as row pagination (optional fallback) + - Images (PNG, JPEG, GIF, WebP): Base64 encoded viewable content + - PDF: Extracts text content as markdown with page structure + * offset/length work as page pagination (0-based) + * Includes embedded images when available + + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(ReadFileArgsSchema), @@ -306,9 +307,10 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { { name: "write_file", description: ` - Write or append to file contents. + Write or append to file contents. - IMPORTANT: DO NOT use this tool to create PDF files. Use 'write_pdf' for all PDF creation tasks. + IMPORTANT: DO NOT use this tool to create PDF files. Use 'write_pdf' for all PDF creation tasks. + CHUNKING IS STANDARD PRACTICE: Always write files in chunks of 25-30 lines maximum. This is the normal, recommended way to write files - not an emergency measure. @@ -324,16 +326,21 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { 1. Any file expected to be longer than 25-30 lines 2. When writing multiple files in sequence 3. When creating documentation, code files, or configuration files - + HANDLING CONTINUATION ("Continue" prompts): If user asks to "Continue" after an incomplete operation: 1. Read the file to see what was successfully written 2. Continue writing ONLY the remaining content using {mode: 'append'} 3. Keep chunks to 25-30 lines each - + + FORMAT HANDLING (by extension): + - Text files: String content + - Excel (.xlsx, .xls, .xlsm): JSON 2D array or {"SheetName": [[...]]} + Example: '[["Name","Age"],["Alice",30]]' + Files over 50 lines will generate performance notes but are still written successfully. Only works within allowed directories. - + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(WriteFileArgsSchema), @@ -624,13 +631,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { Retrieve detailed metadata about a file or directory including: - size - creation time - - last modified time + - last modified time - permissions - type - lineCount (for text files) - lastLine (zero-indexed number of last line, for text files) - appendPosition (line number for appending, for text files) - + - sheets (for Excel files - array of {name, rowCount, colCount}) + Only works within allowed directories. ${PATH_GUIDANCE} @@ -643,45 +651,55 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { }, // Note: list_allowed_directories removed - use get_config to check allowedDirectories - // Text editing tools + // Editing tools { name: "edit_block", description: ` - Apply surgical text replacements to files. - + Apply surgical edits to files. + + BEST PRACTICE: Make multiple small, focused edits rather than one large edit. - Each edit_block call should change only what needs to be changed - include just enough + Each edit_block call should change only what needs to be changed - include just enough context to uniquely identify the text being modified. - + + FORMAT HANDLING (by extension): + + EXCEL FILES (.xlsx, .xls, .xlsm) - Range Update mode: + Takes: + - file_path: Path to the Excel file + - range: "SheetName!A1:C10" or "SheetName" for whole sheet + - content: 2D array, e.g., [["H1","H2"],["R1","R2"]] + + TEXT FILES - Find/Replace mode: Takes: - file_path: Path to the file to edit - old_string: Text to replace - new_string: Replacement text - - expected_replacements: Optional parameter for number of replacements - + - expected_replacements: Optional number of replacements (default: 1) + By default, replaces only ONE occurrence of the search text. - To replace multiple occurrences, provide the expected_replacements parameter with + To replace multiple occurrences, provide expected_replacements with the exact number of matches expected. - + UNIQUENESS REQUIREMENT: When expected_replacements=1 (default), include the minimal amount of context necessary (typically 1-3 lines) before and after the change point, with exact whitespace and indentation. - + When editing multiple sections, make separate edit_block calls for each distinct change rather than one large replacement. - + When a close but non-exact match is found, a character-level diff is shown in the format: common_prefix{-removed-}{+added+}common_suffix to help you identify what's different. - + Similar to write_file, there is a configurable line limit (fileWriteLineLimit) that warns if the edited file exceeds this limit. If this happens, consider breaking your edits into smaller, more focused changes. - + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(EditBlockArgsSchema), annotations: { - title: "Edit Text Block", + title: "Edit Block", readOnlyHint: false, destructiveHint: true, openWorldHint: false, @@ -1038,7 +1056,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { If unclear from context, use: "exploring tool capabilities" The prompt content will be injected and execution begins immediately. - + ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(GetPromptsArgsSchema), } @@ -1230,6 +1248,18 @@ server.setRequestHandler(CallToolRequestSchema, async (request: CallToolRequest) result = await handlers.handleKillProcess(args); break; + case "execute_node": + try { + result = await handlers.handleExecuteNode(args); + } catch (error) { + capture('server_request_error', {message: `Error in execute_node handler: ${error}`}); + result = { + content: [{type: "text", text: `Error executing Node.js code: ${error instanceof Error ? error.message : String(error)}`}], + isError: true, + }; + } + break; + // Note: REPL functionality removed in favor of using general terminal commands // Filesystem tools diff --git a/src/tools/edit.ts b/src/tools/edit.ts index 5847a16c..05453cf0 100644 --- a/src/tools/edit.ts +++ b/src/tools/edit.ts @@ -1,3 +1,20 @@ +/** + * Text file editing via search/replace with fuzzy matching support. + * + * TECHNICAL DEBT / ARCHITECTURAL NOTE: + * This file contains text editing logic that should ideally live in TextFileHandler.editRange() + * to be consistent with how Excel editing works (ExcelFileHandler.editRange()). + * + * Current inconsistency: + * - Excel: edit_block → ExcelFileHandler.editRange() ✓ uses file handler + * - Text: edit_block → performSearchReplace() here → bypasses TextFileHandler + * + * Future refactor should: + * 1. Move performSearchReplace() + fuzzy logic into TextFileHandler.editRange() + * 2. Make this file a thin dispatch layer that routes to appropriate FileHandler + * 3. Unify the editRange() signature to handle both text search/replace and structured edits + */ + import { readFile, writeFile, readFileInternal, validatePath } from './filesystem.js'; import fs from 'fs/promises'; import { ServerResult } from '../types.js'; @@ -337,17 +354,72 @@ function highlightDifferences(expected: string, actual: string): string { } /** - * Handle edit_block command with enhanced functionality - * - Supports multiple replacements - * - Validates expected replacements count - * - Provides detailed error messages + * Handle edit_block command + * + * 1. Text files: String replacement (old_string/new_string) + * - Uses fuzzy matching for resilience + * - Handles expected_replacements parameter + * + * 2. Structured files (Excel): Range rewrite (range + content) + * - Bulk updates to cell ranges (e.g., "Sheet1!A1:C10") + * - Whole sheet replacement (e.g., "Sheet1") + * - More powerful and simpler than surgical location-based edits + * - Supports chunking for large datasets (e.g., 1000 rows at a time) + */ export async function handleEditBlock(args: unknown): Promise { const parsed = EditBlockArgsSchema.parse(args); - + + // Structured files: Range rewrite + if (parsed.range && parsed.content !== undefined) { + try { + const { getFileHandler } = await import('../utils/files/factory.js'); + const handler = getFileHandler(parsed.file_path); + + // Parse content if it's a JSON string (AI often sends arrays as JSON strings) + let content = parsed.content; + if (typeof content === 'string') { + try { + content = JSON.parse(content); + } catch { + // Leave as-is if not valid JSON - let handler decide + } + } + + // Check if handler supports range editing + if ('editRange' in handler && typeof handler.editRange === 'function') { + await handler.editRange(parsed.file_path, parsed.range, content, parsed.options); + return { + content: [{ + type: "text", + text: `Successfully updated range ${parsed.range} in ${parsed.file_path}` + }], + }; + } else { + return { + content: [{ + type: "text", + text: `Error: Range-based editing not supported for ${parsed.file_path}` + }], + isError: true + }; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { + content: [{ + type: "text", + text: `Error: ${errorMessage}` + }], + isError: true + }; + } + } + + // Text files: String replacement const searchReplace = { - search: parsed.old_string, - replace: parsed.new_string + search: parsed.old_string!, + replace: parsed.new_string! }; return performSearchReplace(parsed.file_path, searchReplace, parsed.expected_replacements); diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts index 77deed5b..4a1d2c2d 100644 --- a/src/tools/filesystem.ts +++ b/src/tools/filesystem.ts @@ -9,8 +9,9 @@ import { capture } from '../utils/capture.js'; import { withTimeout } from '../utils/withTimeout.js'; import { configManager } from '../config-manager.js'; import { isPdfFile } from "./mime-types.js"; -// import { editPdf, PdfEditOperation, createPdfFromMarkdown } from './pdf-v3.js'; -import { editPdf, PdfOperations, PdfMetadata, PdfPageItem, parsePdfToMarkdown, parseMarkdownToPdf } from './pdf/index.js'; +import { editPdf, PdfOperations, PdfMetadata, parsePdfToMarkdown, parseMarkdownToPdf } from './pdf/index.js'; +import { getFileHandler } from '../utils/files/factory.js'; +import type { ReadOptions, FileResult, PdfPageItem } from '../utils/files/base.js'; // CONSTANTS SECTION - Consolidate all timeouts and thresholds const FILE_OPERATION_TIMEOUTS = { @@ -284,6 +285,10 @@ export async function validatePath(requestedPath: string): Promise { return result; } +// Re-export FileResult from base for consumers +export type { FileResult } from '../utils/files/base.js'; + +// Type alias for backward compatibility with PDF handling code type PdfPayload = { metadata: PdfMetadata; pages: PdfPageItem[]; @@ -291,15 +296,6 @@ type PdfPayload = { type FileResultPayloads = PdfPayload; -// File operation tools -export interface FileResult { - content: string; - mimeType: string; - isImage: boolean; - isPdf?: boolean; - payload?: FileResultPayloads; -} - /** * Read file content from a URL * @param url URL to fetch content from @@ -338,10 +334,12 @@ export async function readFileFromUrl(url: string): Promise { return { content: "", mimeType: 'text/plain', - isImage: false, - isPdf: true, - payload: { - metadata: pdfResult.metadata, + metadata: { + isImage: false, + isPdf: true, + author: pdfResult.metadata.author, + title: pdfResult.metadata.title, + totalPages: pdfResult.metadata.totalPages, pages: pdfResult.pages } }; @@ -351,12 +349,12 @@ export async function readFileFromUrl(url: string): Promise { const buffer = await response.arrayBuffer(); const content = Buffer.from(buffer).toString('base64'); - return { content, mimeType: contentType, isImage }; + return { content, mimeType: contentType, metadata: { isImage } }; } else { // For text content const content = await response.text(); - return { content, mimeType: contentType, isImage }; + return { content, mimeType: contentType, metadata: { isImage } }; } } catch (error) { // Clear the timeout to prevent memory leaks @@ -513,7 +511,7 @@ async function readLastNLinesReverse(filePath: string, n: number, mimeType: stri ? `${generateEnhancedStatusMessage(result.length, -n, fileTotalLines, true)}\n\n${result.join('\n')}` : result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } finally { await fd.close(); } @@ -554,7 +552,7 @@ async function readFromEndWithReadline(filePath: string, requestedLines: number, const content = includeStatusMessage ? `${generateEnhancedStatusMessage(result.length, -requestedLines, fileTotalLines, true)}\n\n${result.join('\n')}` : result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } /** @@ -582,10 +580,10 @@ async function readFromStartWithReadline(filePath: string, offset: number, lengt if (includeStatusMessage) { const statusMessage = generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false); const content = `${statusMessage}\n\n${result.join('\n')}`; - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } else { const content = result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } } @@ -657,7 +655,7 @@ async function readFromEstimatedPosition(filePath: string, offset: number, lengt const content = includeStatusMessage ? `${generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false)}\n\n${result.join('\n')}` : result.join('\n'); - return { content, mimeType, isImage: false }; + return { content, mimeType, metadata: { isImage: false } }; } finally { await fd.close(); } @@ -666,11 +664,16 @@ async function readFromEstimatedPosition(filePath: string, offset: number, lengt /** * Read file content from the local filesystem * @param filePath Path to the file - * @param offset Starting line number to read from (default: 0) - * @param length Maximum number of lines to read (default: from config or 1000) + * @param options Read options (offset, length, sheet, range) * @returns File content or file result with metadata */ -export async function readFileFromDisk(filePath: string, offset: number = 0, length?: number): Promise { +export async function readFileFromDisk( + filePath: string, + options?: ReadOptions +): Promise { + const { offset = 0, sheet, range } = options ?? {}; + let { length } = options ?? {}; + // Add validation for required parameters if (!filePath || typeof filePath !== 'string') { throw new Error('Invalid file path provided'); @@ -683,7 +686,7 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len const validPath = await validatePath(filePath); - // Get file extension for telemetry using path module consistently + // Get file extension for telemetry const fileExtension = getFileExtension(validPath); // Check file size before attempting to read @@ -704,54 +707,28 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len // If we can't stat the file, continue anyway and let the read operation handle errors } - // Detect the MIME type based on file extension - const { mimeType, isImage, isPdf } = await getMimeTypeInfo(validPath); - // Use withTimeout to handle potential hangs const readOperation = async () => { - if (isPdf) { - // Pass file path directly to extractPdfText which handles file reading - const pdfResult = await parsePdfToMarkdown(validPath, { offset, length }); - - return { - content: "", - mimeType: 'text/plain', - isImage: false, - isPdf: true, - payload: { - metadata: pdfResult.metadata, - pages: pdfResult.pages - } - }; - } else if (isImage) { - // For image files, read as Buffer and convert to base64 - // Images are always read in full, ignoring offset and length - const buffer = await fs.readFile(validPath); - const content = buffer.toString('base64'); - - return { content, mimeType, isImage }; - } else { - // For all other files, use smart positioning approach - try { - return await readFileWithSmartPositioning(validPath, offset, length, mimeType, true); - } catch (error) { - // If it's our binary file instruction error, return it as content - if (error instanceof Error && error.message.includes('Cannot read binary file as text:')) { - return { content: error.message, mimeType: 'text/plain', isImage: false }; - } - - // If UTF-8 reading fails for other reasons, also check if it's binary - const isBinary = await isBinaryFile(validPath); - if (isBinary) { - const instructions = getBinaryFileInstructions(validPath, mimeType); - return { content: instructions, mimeType: 'text/plain', isImage: false }; - } + // Get appropriate handler for this file type (now includes PDF handler) + const handler = getFileHandler(validPath); + + // Use handler to read the file + const result = await handler.read(validPath, { + offset, + length, + sheet, + range, + includeStatusMessage: true + }); - // Only if it's truly not binary, then we have a real UTF-8 reading error - throw error; - } - } + // Return result - handler provides correct format for each type + return { + content: result.content, + mimeType: result.mimeType, + metadata: result.metadata + }; }; + // Execute with timeout const result = await withTimeout( readOperation(), @@ -759,6 +736,7 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len `Read file operation for ${filePath}`, null ); + if (result == null) { // Handles the impossible case where withTimeout resolves to null instead of throwing throw new Error('Failed to read the file'); @@ -770,15 +748,17 @@ export async function readFileFromDisk(filePath: string, offset: number = 0, len /** * Read a file from either the local filesystem or a URL * @param filePath Path to the file or URL - * @param isUrl Whether the path is a URL - * @param offset Starting line number to read from (default: 0) - * @param length Maximum number of lines to read (default: from config or 1000) + * @param options Read options (isUrl, offset, length, sheet, range) * @returns File content or file result with metadata */ -export async function readFile(filePath: string, isUrl?: boolean, offset?: number, length?: number): Promise { +export async function readFile( + filePath: string, + options?: ReadOptions +): Promise { + const { isUrl, offset, length, sheet, range } = options ?? {}; return isUrl ? readFileFromUrl(filePath) - : readFileFromDisk(filePath, offset, length); + : readFileFromDisk(filePath, { offset, length, sheet, range }); } /** @@ -888,12 +868,11 @@ export async function writeFile(filePath: string, content: string, mode: 'rewrit lineCount: lineCount }); - // Use different fs methods based on mode - if (mode === 'append') { - await fs.appendFile(validPath, content); - } else { - await fs.writeFile(validPath, content); - } + // Get appropriate handler for this file type + const handler = getFileHandler(validPath); + + // Use handler to write the file + await handler.write(validPath, content, mode); } export interface MultiFileResult { @@ -915,11 +894,18 @@ export async function readMultipleFiles(paths: string[]): Promise> { const validPath = await validatePath(filePath); - const stats = await fs.stat(validPath); - // Basic file info + // Get appropriate handler for this file type + const handler = getFileHandler(validPath); + + // Use handler to get file info + const fileInfo = await handler.getInfo(validPath); + + // Convert to legacy format (for backward compatibility) const info: Record = { - size: stats.size, - created: stats.birthtime, - modified: stats.mtime, - accessed: stats.atime, - isDirectory: stats.isDirectory(), - isFile: stats.isFile(), - permissions: stats.mode.toString(8).slice(-3), + size: fileInfo.size, + created: fileInfo.created, + modified: fileInfo.modified, + accessed: fileInfo.accessed, + isDirectory: fileInfo.isDirectory, + isFile: fileInfo.isFile, + permissions: fileInfo.permissions, + fileType: fileInfo.fileType, }; - // For text files that aren't too large, also count lines - if (stats.isFile() && stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { - try { - // Get MIME type information - const { mimeType, isImage, isPdf } = await getMimeTypeInfo(validPath); - - // Only count lines for non-image, likely text files - if (!isImage && !isPdf) { - const content = await fs.readFile(validPath, 'utf8'); - const lineCount = countLines(content); - info.lineCount = lineCount; - info.lastLine = lineCount - 1; // Zero-indexed last line - info.appendPosition = lineCount; // Position to append at end - } - } catch (error) { - // If reading fails, just skip the line count - // This could happen for binary files or very large files + // Add type-specific metadata from file handler + if (fileInfo.metadata) { + // For text files + if (fileInfo.metadata.lineCount !== undefined) { + info.lineCount = fileInfo.metadata.lineCount; + info.lastLine = fileInfo.metadata.lineCount - 1; + info.appendPosition = fileInfo.metadata.lineCount; + } + + // For Excel files + if (fileInfo.metadata.sheets) { + info.sheets = fileInfo.metadata.sheets; + info.isExcelFile = true; + } + + // For images + if (fileInfo.metadata.isImage) { + info.isImage = true; + } + + // For PDF files + if (fileInfo.metadata.isPdf) { + info.isPdf = true; + info.totalPages = fileInfo.metadata.totalPages; + if (fileInfo.metadata.title) info.title = fileInfo.metadata.title; + if (fileInfo.metadata.author) info.author = fileInfo.metadata.author; + } + + // For binary files + if (fileInfo.metadata.isBinary) { + info.isBinary = true; } } diff --git a/src/tools/schemas.ts b/src/tools/schemas.ts index da824b7f..d76748b5 100644 --- a/src/tools/schemas.ts +++ b/src/tools/schemas.ts @@ -47,6 +47,9 @@ export const ReadFileArgsSchema = z.object({ isUrl: z.boolean().optional().default(false), offset: z.number().optional().default(0), length: z.number().optional().default(1000), + sheet: z.union([z.string(), z.number()]).optional(), + range: z.string().optional(), + options: z.record(z.any()).optional() }); export const ReadMultipleFilesArgsSchema = z.object({ @@ -116,13 +119,26 @@ export const GetFileInfoArgsSchema = z.object({ path: z.string(), }); -// Edit tools schema +// Edit tools schema - SIMPLIFIED from three modes to two +// Previously supported: text replacement, location-based edits (edits array), and range rewrites +// Now supports only: text replacement and range rewrites +// Removed 'edits' array parameter - location-based surgical edits were complex and unnecessary +// Range rewrites are more powerful and cover all structured file editing needs export const EditBlockArgsSchema = z.object({ file_path: z.string(), - old_string: z.string(), - new_string: z.string(), + // Text file string replacement + old_string: z.string().optional(), + new_string: z.string().optional(), expected_replacements: z.number().optional().default(1), -}); + // Structured file range rewrite (Excel, etc.) + range: z.string().optional(), + content: z.any().optional(), + options: z.record(z.any()).optional() +}).refine( + data => (data.old_string !== undefined && data.new_string !== undefined) || + (data.range !== undefined && data.content !== undefined), + { message: "Must provide either (old_string + new_string) or (range + content)" } +); // Send input to process schema export const InteractWithProcessArgsSchema = z.object({ @@ -185,4 +201,10 @@ export const GetRecentToolCallsArgsSchema = z.object({ maxResults: z.number().min(1).max(1000).optional().default(50), toolName: z.string().optional(), since: z.string().datetime().optional(), +}); + +// Execute Node.js code schema +export const ExecuteNodeArgsSchema = z.object({ + code: z.string(), + timeout_ms: z.number().optional().default(30000), }); \ No newline at end of file diff --git a/src/utils/files/base.ts b/src/utils/files/base.ts new file mode 100644 index 00000000..a275441d --- /dev/null +++ b/src/utils/files/base.ts @@ -0,0 +1,219 @@ +/** + * Base interfaces and types for file handling system + * All file handlers implement the FileHandler interface + */ + +// ============================================================================ +// Core Interfaces +// ============================================================================ + +/** + * Base interface that all file handlers must implement + */ +export interface FileHandler { + /** + * Read file content + * @param path Validated file path + * @param options Read options (offset, length, sheet, etc.) + * @returns File result with content and metadata + */ + read(path: string, options?: ReadOptions): Promise; + + /** + * Write file (complete rewrite or append) + * @param path Validated file path + * @param content Content to write + * @param mode Write mode: 'rewrite' (default) or 'append' + */ + write(path: string, content: any, mode?: 'rewrite' | 'append'): Promise; + + /** + * Edit a specific range (bulk rewrite) + * PRIMARY METHOD for structured file editing (Excel, etc.) + * Simpler and more powerful than location-based edits + * Supports: + * - Cell ranges: "Sheet1!A1:C10" with 2D array content + * - Whole sheets: "Sheet1" to replace entire sheet + * - Chunking: Update 1000 rows at a time for large files + * + * Currently implemented by: ExcelFileHandler + * TECHNICAL DEBT: TextFileHandler should also implement this for search/replace + * (logic currently in src/tools/edit.ts - see comments there) + * + * @param path Validated file path + * @param range Range identifier (e.g., "Sheet1!A1:C10" or "Sheet1") + * @param content New content for the range (2D array for Excel) + * @param options Additional format-specific options + * @returns Result with success status + */ + editRange?(path: string, range: string, content: any, options?: Record): Promise; + + /** + * Get file metadata + * @param path Validated file path + * @returns File information including type-specific metadata + */ + getInfo(path: string): Promise; + + /** + * Check if this handler can handle the given file + * @param path File path + * @returns true if this handler supports this file type + */ + canHandle(path: string): boolean; +} + +// ============================================================================ +// Read Operations +// ============================================================================ + +/** + * Options for reading files + */ +export interface ReadOptions { + /** Whether the path is a URL */ + isUrl?: boolean; + + /** Starting line/row number (for text/excel) */ + offset?: number; + + /** Maximum number of lines/rows to read */ + length?: number; + + /** Excel-specific: Sheet name or index */ + sheet?: string | number; + + /** Excel-specific: Cell range (e.g., "A1:C10") */ + range?: string; + + /** Whether to include status messages (default: true) */ + includeStatusMessage?: boolean; +} + +/** + * Result from reading a file + */ +export interface FileResult { + /** File content (string for text/csv, Buffer for binary, base64 string for images) */ + content: string | Buffer; + + /** MIME type of the content */ + mimeType: string; + + /** Type-specific metadata */ + metadata?: FileMetadata; +} + +/** + * File-type specific metadata + */ +export interface FileMetadata { + /** For images */ + isImage?: boolean; + + /** For binary files */ + isBinary?: boolean; + + /** For Excel files */ + isExcelFile?: boolean; + sheets?: ExcelSheet[]; + fileSize?: number; + isLargeFile?: boolean; + + /** For text files */ + lineCount?: number; + + /** For PDF files */ + isPdf?: boolean; + author?: string; + title?: string; + totalPages?: number; + pages?: PdfPageItem[]; + + /** Error information if operation failed */ + error?: boolean; + errorMessage?: string; +} + +/** + * PDF page content item + */ +export interface PdfPageItem { + pageNumber: number; + text: string; + images: Array<{ + data: string; + mimeType: string; + }>; +} + +/** + * Excel sheet metadata + */ +export interface ExcelSheet { + /** Sheet name */ + name: string; + + /** Number of rows in sheet */ + rowCount: number; + + /** Number of columns in sheet */ + colCount: number; +} + +// ============================================================================ +// Edit Operations +// ============================================================================ + +/** + * Result from edit operation (used by editRange) + */ +export interface EditResult { + /** Whether all edits succeeded */ + success: boolean; + + /** Number of edits successfully applied */ + editsApplied: number; + + /** Errors that occurred during editing */ + errors?: Array<{ + location: string; + error: string; + }>; +} + +// ============================================================================ +// File Information +// ============================================================================ + +/** + * File information and metadata + */ +export interface FileInfo { + /** File size in bytes */ + size: number; + + /** Creation time */ + created: Date; + + /** Last modification time */ + modified: Date; + + /** Last access time */ + accessed: Date; + + /** Is this a directory */ + isDirectory: boolean; + + /** Is this a regular file */ + isFile: boolean; + + /** File permissions (octal string) */ + permissions: string; + + /** File type classification */ + fileType: 'text' | 'excel' | 'image' | 'binary'; + + /** Type-specific metadata */ + metadata?: FileMetadata; +} diff --git a/src/utils/files/binary.ts b/src/utils/files/binary.ts new file mode 100644 index 00000000..a934e833 --- /dev/null +++ b/src/utils/files/binary.ts @@ -0,0 +1,173 @@ +/** + * Binary file handler + * Catch-all handler for unsupported binary files + * Returns instructions to use start_process with appropriate tools + */ + +import fs from "fs/promises"; +import path from "path"; +import { + FileHandler, + ReadOptions, + FileResult, + FileInfo +} from './base.js'; + +/** + * Binary file handler implementation + * This is a catch-all handler for binary files that aren't supported by other handlers + */ +export class BinaryFileHandler implements FileHandler { + canHandle(path: string): boolean { + // Binary handler is the catch-all - handles everything not handled by other handlers + return true; + } + + async read(filePath: string, options?: ReadOptions): Promise { + const instructions = this.getBinaryInstructions(filePath); + + return { + content: instructions, + mimeType: 'text/plain', + metadata: { + isBinary: true + } + }; + } + + async write(path: string, content: any): Promise { + throw new Error('Cannot write binary files directly. Use start_process with appropriate tools (Python, Node.js libraries, command-line utilities).'); + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'binary', + metadata: { + isBinary: true + } + }; + } + + /** + * Generate instructions for handling binary files + */ + private getBinaryInstructions(filePath: string): string { + const fileName = path.basename(filePath); + const ext = path.extname(filePath).toLowerCase(); + + // Get MIME type suggestion based on extension + const mimeType = this.guessMimeType(ext); + + let specificGuidance = ''; + + // Provide specific guidance based on file type + switch (ext) { + case '.pdf': + specificGuidance = ` +PDF FILES: +- Python: PyPDF2, pdfplumber + start_process("python -i") + interact_with_process(pid, "import pdfplumber") + interact_with_process(pid, "pdf = pdfplumber.open('${filePath}')") + interact_with_process(pid, "print(pdf.pages[0].extract_text())") + +- Node.js: pdf-parse + start_process("node -i") + interact_with_process(pid, "const pdf = require('pdf-parse')")`; + break; + + case '.doc': + case '.docx': + specificGuidance = ` +WORD DOCUMENTS: +- Python: python-docx + start_process("python -i") + interact_with_process(pid, "import docx") + interact_with_process(pid, "doc = docx.Document('${filePath}')") + interact_with_process(pid, "for para in doc.paragraphs: print(para.text)") + +- Node.js: mammoth + start_process("node -i") + interact_with_process(pid, "const mammoth = require('mammoth')")`; + break; + + case '.zip': + case '.tar': + case '.gz': + specificGuidance = ` +ARCHIVE FILES: +- Python: zipfile, tarfile + start_process("python -i") + interact_with_process(pid, "import zipfile") + interact_with_process(pid, "with zipfile.ZipFile('${filePath}') as z: print(z.namelist())") + +- Command-line: + start_process("unzip -l ${filePath}") # For ZIP files + start_process("tar -tzf ${filePath}") # For TAR files`; + break; + + case '.db': + case '.sqlite': + case '.sqlite3': + specificGuidance = ` +SQLITE DATABASES: +- Python: sqlite3 + start_process("python -i") + interact_with_process(pid, "import sqlite3") + interact_with_process(pid, "conn = sqlite3.connect('${filePath}')") + interact_with_process(pid, "cursor = conn.cursor()") + interact_with_process(pid, "cursor.execute('SELECT * FROM sqlite_master')") + +- Command-line: + start_process("sqlite3 ${filePath} '.tables'")`; + break; + + default: + specificGuidance = ` +GENERIC BINARY FILES: +- Use appropriate libraries based on file type +- Python libraries: Check PyPI for ${ext} support +- Node.js libraries: Check npm for ${ext} support +- Command-line tools: Use file-specific utilities`; + } + + return `Cannot read binary file as text: ${fileName} (${mimeType}) + +Use start_process + interact_with_process to analyze binary files with appropriate tools. +${specificGuidance} + +The read_file tool only handles text files, images, and Excel files.`; + } + + /** + * Guess MIME type based on file extension + */ + private guessMimeType(ext: string): string { + const mimeTypes: { [key: string]: string } = { + '.pdf': 'application/pdf', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.zip': 'application/zip', + '.tar': 'application/x-tar', + '.gz': 'application/gzip', + '.db': 'application/x-sqlite3', + '.sqlite': 'application/x-sqlite3', + '.sqlite3': 'application/x-sqlite3', + '.mp3': 'audio/mpeg', + '.mp4': 'video/mp4', + '.avi': 'video/x-msvideo', + '.mkv': 'video/x-matroska', + }; + + return mimeTypes[ext] || 'application/octet-stream'; + } +} diff --git a/src/utils/files/excel.ts b/src/utils/files/excel.ts new file mode 100644 index 00000000..5c0a9df1 --- /dev/null +++ b/src/utils/files/excel.ts @@ -0,0 +1,486 @@ +/** + * Excel file handler using ExcelJS + * Handles reading, writing, and editing Excel files (.xlsx, .xls, .xlsm) + */ + +import ExcelJS from 'exceljs'; +import fs from 'fs/promises'; +import { + FileHandler, + ReadOptions, + FileResult, + EditResult, + FileInfo, + ExcelSheet +} from './base.js'; + +// File size limit: 10MB +const FILE_SIZE_LIMIT = 10 * 1024 * 1024; + +/** + * Excel file metadata (internal use only) + */ +interface ExcelMetadata { + sheets: ExcelSheet[]; + fileSize: number; + isLargeFile: boolean; +} + +/** + * Excel file handler implementation using ExcelJS + * Supports: .xlsx, .xls, .xlsm files + */ +export class ExcelFileHandler implements FileHandler { + + canHandle(path: string): boolean { + const ext = path.toLowerCase(); + return ext.endsWith('.xlsx') || ext.endsWith('.xls') || ext.endsWith('.xlsm'); + } + + async read(path: string, options?: ReadOptions): Promise { + await this.checkFileSize(path); + + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + + const metadata = await this.extractMetadata(workbook, path); + const { sheetName, data, totalRows, returnedRows } = this.worksheetToArray( + workbook, + options?.sheet, + options?.range, + options?.offset, + options?.length + ); + + // Format output with sheet info header, usage hint, and JSON data + const paginationInfo = totalRows > returnedRows + ? `\n[Showing rows ${(options?.offset || 0) + 1}-${(options?.offset || 0) + returnedRows} of ${totalRows} total. Use offset/length to paginate.]` + : ''; + + const content = `[Sheet: '${sheetName}' from ${path}]${paginationInfo} +[To MODIFY cells: use edit_block with range param, e.g., edit_block(path, {range: "Sheet1!E5", content: [[newValue]]})] + +${JSON.stringify(data)}`; + + return { + content, + mimeType: 'application/json', + metadata: { + isExcelFile: true, + sheets: metadata.sheets, + fileSize: metadata.fileSize, + isLargeFile: metadata.isLargeFile + } + }; + } + + async write(path: string, content: any, mode?: 'rewrite' | 'append'): Promise { + // Check existing file size if it exists + try { + await this.checkFileSize(path); + } catch (error) { + // File doesn't exist - that's fine for write + if ((error as any).code !== 'ENOENT' && + !(error instanceof Error && error.message.includes('ENOENT'))) { + throw error; + } + } + + // Parse content + let parsedContent = content; + if (typeof content === 'string') { + try { + parsedContent = JSON.parse(content); + } catch { + throw new Error('Invalid content format. Expected JSON string with 2D array or object with sheet names.'); + } + } + + // Handle append mode by finding last row and writing after it + if (mode === 'append') { + try { + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + + if (Array.isArray(parsedContent)) { + // Append to Sheet1 + let worksheet = workbook.getWorksheet('Sheet1'); + if (!worksheet) { + worksheet = workbook.addWorksheet('Sheet1'); + } + const startRow = (worksheet.actualRowCount || 0) + 1; + this.writeRowsStartingAt(worksheet, startRow, parsedContent); + } else if (typeof parsedContent === 'object' && parsedContent !== null) { + // Append to each named sheet + for (const [sheetName, data] of Object.entries(parsedContent)) { + if (Array.isArray(data)) { + let worksheet = workbook.getWorksheet(sheetName); + if (!worksheet) { + worksheet = workbook.addWorksheet(sheetName); + } + const startRow = (worksheet.actualRowCount || 0) + 1; + this.writeRowsStartingAt(worksheet, startRow, data as any[][]); + } + } + } + + await workbook.xlsx.writeFile(path); + return; + } catch (error) { + // File doesn't exist - fall through to create new file + if ((error as any).code !== 'ENOENT' && + !(error instanceof Error && error.message.includes('ENOENT'))) { + throw error; + } + } + } + + // Rewrite mode (or append to non-existent file): create new workbook + const workbook = new ExcelJS.Workbook(); + + if (Array.isArray(parsedContent)) { + // Single sheet from 2D array + this.writeDataToSheet(workbook, 'Sheet1', parsedContent); + } else if (typeof parsedContent === 'object' && parsedContent !== null) { + // Object with sheet names as keys + for (const [sheetName, data] of Object.entries(parsedContent)) { + if (Array.isArray(data)) { + this.writeDataToSheet(workbook, sheetName, data as any[][]); + } + } + } else { + throw new Error('Invalid content format. Expected 2D array or object with sheet names.'); + } + + await workbook.xlsx.writeFile(path); + } + + async editRange(path: string, range: string, content: any, options?: Record): Promise { + // Verify file exists and check size + try { + await this.checkFileSize(path); + } catch (error) { + if ((error as any).code === 'ENOENT' || + (error instanceof Error && error.message.includes('ENOENT'))) { + throw new Error(`File not found: ${path}`); + } + throw error; + } + + // Validate content + if (!Array.isArray(content)) { + throw new Error('Content must be a 2D array for range editing'); + } + + // Parse range: "Sheet1!A1:C10" or "Sheet1" + const [sheetName, cellRange] = this.parseRange(range); + + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + + // Get or create sheet + let worksheet = workbook.getWorksheet(sheetName); + if (!worksheet) { + worksheet = workbook.addWorksheet(sheetName); + } + + if (cellRange) { + // Write to specific range + const { startRow, startCol } = this.parseCellRange(cellRange); + + for (let r = 0; r < content.length; r++) { + const rowData = content[r]; + if (!Array.isArray(rowData)) continue; + + for (let c = 0; c < rowData.length; c++) { + const cell = worksheet.getCell(startRow + r, startCol + c); + const value = rowData[c]; + + if (typeof value === 'string' && value.startsWith('=')) { + cell.value = { formula: value.substring(1) }; + } else { + cell.value = value; + } + } + } + } else { + // Replace entire sheet content + // Clear existing data + worksheet.eachRow((row, rowNumber) => { + row.eachCell((cell) => { + cell.value = null; + }); + }); + + // Write new data + for (let r = 0; r < content.length; r++) { + const rowData = content[r]; + if (!Array.isArray(rowData)) continue; + + const row = worksheet.getRow(r + 1); + for (let c = 0; c < rowData.length; c++) { + const value = rowData[c]; + if (typeof value === 'string' && value.startsWith('=')) { + row.getCell(c + 1).value = { formula: value.substring(1) }; + } else { + row.getCell(c + 1).value = value; + } + } + row.commit(); + } + } + + await workbook.xlsx.writeFile(path); + + return { success: true, editsApplied: 1 }; + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + try { + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + const metadata = await this.extractMetadata(workbook, path); + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'excel', + metadata: { + isExcelFile: true, + sheets: metadata.sheets, + fileSize: metadata.fileSize, + isLargeFile: metadata.isLargeFile + } + }; + } catch (error) { + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'excel', + metadata: { + isExcelFile: true, + fileSize: stats.size, + error: true, + errorMessage: error instanceof Error ? error.message : String(error) + } + }; + } + } + + // ========== Private Helpers ========== + + private async checkFileSize(path: string): Promise { + const stats = await fs.stat(path); + if (stats.size > FILE_SIZE_LIMIT) { + const sizeMB = (stats.size / 1024 / 1024).toFixed(1); + throw new Error( + `Excel file size (${sizeMB}MB) exceeds 10MB limit. ` + + `Consider using specialized tools for large file processing.` + ); + } + } + + private async extractMetadata(workbook: ExcelJS.Workbook, path: string): Promise { + const stats = await fs.stat(path); + + const sheets: ExcelSheet[] = workbook.worksheets.map(ws => ({ + name: ws.name, + rowCount: ws.actualRowCount || 0, + colCount: ws.actualColumnCount || 0 + })); + + return { + sheets, + fileSize: stats.size, + isLargeFile: stats.size > FILE_SIZE_LIMIT + }; + } + + private worksheetToArray( + workbook: ExcelJS.Workbook, + sheetRef?: string | number, + range?: string, + offset?: number, + length?: number + ): { sheetName: string; data: any[][]; totalRows: number; returnedRows: number } { + if (workbook.worksheets.length === 0) { + return { sheetName: '', data: [], totalRows: 0, returnedRows: 0 }; + } + + // Find target worksheet + let worksheet: ExcelJS.Worksheet | undefined; + let sheetName: string; + + if (sheetRef === undefined) { + worksheet = workbook.worksheets[0]; + sheetName = worksheet.name; + } else if (typeof sheetRef === 'number') { + if (sheetRef < 0 || sheetRef >= workbook.worksheets.length) { + throw new Error(`Sheet index ${sheetRef} out of range (0-${workbook.worksheets.length - 1})`); + } + worksheet = workbook.worksheets[sheetRef]; + sheetName = worksheet.name; + } else { + worksheet = workbook.getWorksheet(sheetRef); + if (!worksheet) { + const available = workbook.worksheets.map(ws => ws.name).join(', '); + throw new Error(`Sheet "${sheetRef}" not found. Available sheets: ${available}`); + } + sheetName = sheetRef; + } + + // Determine range to read + let startRow = 1; + let endRow = worksheet.actualRowCount || 1; + let startCol = 1; + let endCol = worksheet.actualColumnCount || 1; + + if (range) { + const parsed = this.parseCellRange(range); + startRow = parsed.startRow; + startCol = parsed.startCol; + if (parsed.endRow) endRow = parsed.endRow; + if (parsed.endCol) endCol = parsed.endCol; + } + + // Calculate total rows before pagination + const totalRows = endRow - startRow + 1; + + // Apply offset/length pagination (row-based, matching text file behavior) + if (offset !== undefined) { + if (offset < 0) { + // Negative offset: last N rows (like text files) + // offset: -10 means "last 10 rows" + const lastNRows = Math.abs(offset); + startRow = Math.max(startRow, endRow - lastNRows + 1); + } else if (offset > 0) { + // Positive offset: skip first N rows + startRow = startRow + offset; + } + } + + // Apply length limit (only for positive offset or no offset) + if (length !== undefined && length > 0 && (offset === undefined || offset >= 0)) { + endRow = Math.min(endRow, startRow + length - 1); + } + + // Ensure valid range + if (startRow > endRow) { + return { sheetName, data: [], totalRows, returnedRows: 0 }; + } + + // Build 2D array (preserving types) + const data: any[][] = []; + for (let r = startRow; r <= endRow; r++) { + const row = worksheet.getRow(r); + const rowData: any[] = []; + + for (let c = startCol; c <= endCol; c++) { + const cell = row.getCell(c); + let value: any = null; + + if (cell.value !== null && cell.value !== undefined) { + if (typeof cell.value === 'object') { + // Handle formula results, rich text, etc. + if ('result' in cell.value) { + value = cell.value.result ?? null; + } else if ('richText' in cell.value) { + value = (cell.value as any).richText.map((rt: any) => rt.text).join(''); + } else if ('text' in cell.value) { + value = (cell.value as any).text; + } else if (cell.value instanceof Date) { + value = cell.value.toISOString(); + } else { + value = String(cell.value); + } + } else { + // Preserve native types (string, number, boolean) + value = cell.value; + } + } + + rowData.push(value); + } + data.push(rowData); + } + + return { sheetName, data, totalRows, returnedRows: data.length }; + } + + private writeDataToSheet(workbook: ExcelJS.Workbook, sheetName: string, data: any[][]): void { + // Remove existing sheet if it exists + const existing = workbook.getWorksheet(sheetName); + if (existing) { + workbook.removeWorksheet(existing.id); + } + + const worksheet = workbook.addWorksheet(sheetName); + this.writeRowsStartingAt(worksheet, 1, data); + } + + private writeRowsStartingAt(worksheet: ExcelJS.Worksheet, startRow: number, data: any[][]): void { + for (let r = 0; r < data.length; r++) { + const rowData = data[r]; + if (!Array.isArray(rowData)) continue; + + const row = worksheet.getRow(startRow + r); + for (let c = 0; c < rowData.length; c++) { + const value = rowData[c]; + if (typeof value === 'string' && value.startsWith('=')) { + row.getCell(c + 1).value = { formula: value.substring(1) }; + } else { + row.getCell(c + 1).value = value; + } + } + row.commit(); + } + } + + private parseRange(range: string): [string, string | null] { + if (range.includes('!')) { + const [sheetName, cellRange] = range.split('!'); + return [sheetName, cellRange]; + } + return [range, null]; + } + + private parseCellRange(range: string): { startRow: number; startCol: number; endRow?: number; endCol?: number } { + // Parse A1 or A1:C10 format + const match = range.match(/^([A-Z]+)(\d+)(?::([A-Z]+)(\d+))?$/i); + if (!match) { + throw new Error(`Invalid cell range: ${range}`); + } + + const startCol = this.columnToNumber(match[1]); + const startRow = parseInt(match[2], 10); + + if (match[3] && match[4]) { + const endCol = this.columnToNumber(match[3]); + const endRow = parseInt(match[4], 10); + return { startRow, startCol, endRow, endCol }; + } + + return { startRow, startCol }; + } + + private columnToNumber(col: string): number { + let result = 0; + for (let i = 0; i < col.length; i++) { + result = result * 26 + col.charCodeAt(i) - 64; + } + return result; + } +} + diff --git a/src/utils/files/factory.ts b/src/utils/files/factory.ts new file mode 100644 index 00000000..d911e1ae --- /dev/null +++ b/src/utils/files/factory.ts @@ -0,0 +1,91 @@ +/** + * Factory pattern for creating appropriate file handlers + * Routes file operations to the correct handler based on file type + */ + +import { FileHandler } from './base.js'; +import { TextFileHandler } from './text.js'; +import { ImageFileHandler } from './image.js'; +import { BinaryFileHandler } from './binary.js'; +import { ExcelFileHandler } from './excel.js'; +import { PdfFileHandler } from './pdf.js'; + +// Singleton instances of each handler +let handlers: FileHandler[] | null = null; + +/** + * Initialize handlers (lazy initialization) + */ +function initializeHandlers(): FileHandler[] { + if (handlers) { + return handlers; + } + + handlers = [ + // Order matters! More specific handlers first + new PdfFileHandler(), // Check PDF first + new ExcelFileHandler(), // Check Excel (before binary) + new ImageFileHandler(), // Then images + new TextFileHandler(), // Then text (handles most files) + new BinaryFileHandler(), // Finally binary (catch-all) + ]; + + return handlers; +} + +/** + * Get the appropriate file handler for a given file path + * + * This function checks each handler in priority order and returns the first + * handler that can handle the file type. + * + * Priority order: + * 1. Excel files (xlsx, xls, xlsm) + * 2. Image files (png, jpg, gif, webp) + * 3. Text files (most other files) + * 4. Binary files (catch-all for unsupported formats) + * + * @param path File path (can be before or after validation) + * @returns FileHandler instance that can handle this file + */ +export function getFileHandler(path: string): FileHandler { + const allHandlers = initializeHandlers(); + + // Try each handler in order + for (const handler of allHandlers) { + if (handler.canHandle(path)) { + return handler; + } + } + + // Fallback to binary handler (should never reach here due to binary catch-all) + return allHandlers[allHandlers.length - 1]; +} + +/** + * Check if a file path is an Excel file + * @param path File path + * @returns true if file is Excel format + */ +export function isExcelFile(path: string): boolean { + const ext = path.toLowerCase(); + return ext.endsWith('.xlsx') || ext.endsWith('.xls') || ext.endsWith('.xlsm'); +} + +/** + * Check if a file path is an image file + * @param path File path + * @returns true if file is an image format + */ +export function isImageFile(path: string): boolean { + // This will be implemented by checking MIME type + // For now, use extension-based check + const ext = path.toLowerCase(); + return ext.endsWith('.png') || + ext.endsWith('.jpg') || + ext.endsWith('.jpeg') || + ext.endsWith('.gif') || + ext.endsWith('.webp') || + ext.endsWith('.bmp') || + ext.endsWith('.svg'); +} diff --git a/src/utils/files/image.ts b/src/utils/files/image.ts new file mode 100644 index 00000000..4e97ce9b --- /dev/null +++ b/src/utils/files/image.ts @@ -0,0 +1,93 @@ +/** + * Image file handler + * Handles reading image files and converting to base64 + */ + +import fs from "fs/promises"; +import { + FileHandler, + ReadOptions, + FileResult, + FileInfo +} from './base.js'; + +/** + * Image file handler implementation + * Supports: PNG, JPEG, GIF, WebP, BMP, SVG + */ +export class ImageFileHandler implements FileHandler { + private static readonly IMAGE_EXTENSIONS = [ + '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg' + ]; + + private static readonly IMAGE_MIME_TYPES: { [key: string]: string } = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.bmp': 'image/bmp', + '.svg': 'image/svg+xml' + }; + + canHandle(path: string): boolean { + const lowerPath = path.toLowerCase(); + return ImageFileHandler.IMAGE_EXTENSIONS.some(ext => lowerPath.endsWith(ext)); + } + + async read(path: string, options?: ReadOptions): Promise { + // Images are always read in full, ignoring offset and length + const buffer = await fs.readFile(path); + const content = buffer.toString('base64'); + const mimeType = this.getMimeType(path); + + return { + content, + mimeType, + metadata: { + isImage: true + } + }; + } + + async write(path: string, content: Buffer | string): Promise { + // If content is base64 string, convert to buffer + if (typeof content === 'string') { + const buffer = Buffer.from(content, 'base64'); + await fs.writeFile(path, buffer); + } else { + await fs.writeFile(path, content); + } + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'image', + metadata: { + isImage: true + } + }; + } + + /** + * Get MIME type for image based on file extension + */ + private getMimeType(path: string): string { + const lowerPath = path.toLowerCase(); + for (const [ext, mimeType] of Object.entries(ImageFileHandler.IMAGE_MIME_TYPES)) { + if (lowerPath.endsWith(ext)) { + return mimeType; + } + } + return 'application/octet-stream'; // Fallback + } +} diff --git a/src/utils/files/index.ts b/src/utils/files/index.ts new file mode 100644 index 00000000..c35e0642 --- /dev/null +++ b/src/utils/files/index.ts @@ -0,0 +1,16 @@ +/** + * File handling system + * Exports all file handlers, interfaces, and utilities + */ + +// Base interfaces and types +export * from './base.js'; + +// Factory function +export { getFileHandler, isExcelFile, isImageFile } from './factory.js'; + +// File handlers +export { TextFileHandler } from './text.js'; +export { ImageFileHandler } from './image.js'; +export { BinaryFileHandler } from './binary.js'; +export { ExcelFileHandler } from './excel.js'; diff --git a/src/utils/files/pdf.ts b/src/utils/files/pdf.ts new file mode 100644 index 00000000..ac7f4011 --- /dev/null +++ b/src/utils/files/pdf.ts @@ -0,0 +1,147 @@ +/** + * PDF File Handler + * Implements FileHandler interface for PDF documents + */ + +import fs from 'fs/promises'; +import { FileHandler, FileResult, FileInfo, ReadOptions, EditResult } from './base.js'; +import { parsePdfToMarkdown, parseMarkdownToPdf, editPdf } from '../../tools/pdf/index.js'; + +/** + * File handler for PDF documents + * Extracts text and images, supports page-based pagination + */ +export class PdfFileHandler implements FileHandler { + private readonly extensions = ['.pdf']; + + /** + * Check if this handler can handle the given file + */ + canHandle(path: string): boolean { + const ext = path.toLowerCase(); + return this.extensions.some(e => ext.endsWith(e)); + } + + /** + * Read PDF content - extracts text as markdown with images + */ + async read(path: string, options?: ReadOptions): Promise { + const { offset = 0, length } = options ?? {}; + + try { + // Use existing PDF parser + // Ensure we pass a valid PageRange or number array + // If length is undefined, we assume "rest of file" which requires careful handling. + // If length is defined, we pass { offset, length }. + // If neither, we pass empty array (all pages). + // Note: offset defaults to 0 if undefined. + + let range: any; + if (length !== undefined) { + range = { offset, length }; + } else if (offset > 0) { + // If offset provided but no length, try to read reasonable amount or all? + // PageRange requires length. Let's assume 0 means "all" or use a large number? + // Looking at pdf2md implementation, it uses generatePageNumbers(offset, length, total). + // We'll pass 0 for length to imply "rest" if supported, or just undefined length if valid. + // But typescript requires length. + range = { offset, length: 0 }; + } else { + range = []; + } + + const pdfResult = await parsePdfToMarkdown(path, range); + + return { + content: '', // Main content is in metadata.pages + mimeType: 'application/pdf', + metadata: { + isPdf: true, + author: pdfResult.metadata.author, + title: pdfResult.metadata.title, + totalPages: pdfResult.metadata.totalPages, + pages: pdfResult.pages + } + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { + content: `Error reading PDF: ${errorMessage}`, + mimeType: 'text/plain', + metadata: { + error: true, + errorMessage + } + }; + } + } + + /** + * Write PDF - creates from markdown or operations + */ + async write(path: string, content: any, mode?: 'rewrite' | 'append'): Promise { + // If content is string, treat as markdown to convert + if (typeof content === 'string') { + await parseMarkdownToPdf(content, path); + } else if (Array.isArray(content)) { + // Array of operations - use editPdf + const resultBuffer = await editPdf(path, content); + await fs.writeFile(path, resultBuffer); + } else { + throw new Error('PDF write requires markdown string or array of operations'); + } + } + + /** + * Edit PDF by range/operations + */ + async editRange(path: string, range: string, content: any, options?: Record): Promise { + try { + // For PDF, range editing isn't directly supported + // Could interpret range as page numbers in future + const resultBuffer = await editPdf(path, content); + await fs.writeFile(options?.outputPath || path, resultBuffer); + return { success: true, editsApplied: 1 }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { + success: false, + editsApplied: 0, + errors: [{ location: range, error: errorMessage }] + }; + } + } + + /** + * Get PDF file information + */ + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + // Get basic PDF metadata + let metadata: any = { isPdf: true }; + try { + const pdfResult = await parsePdfToMarkdown(path, { offset: 0, length: 0 }); // Just metadata + metadata = { + isPdf: true, + title: pdfResult.metadata.title, + author: pdfResult.metadata.author, + totalPages: pdfResult.metadata.totalPages + }; + } catch { + // If we can't parse, just return basic info + } + + return { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: false, + isFile: true, + permissions: (stats.mode & 0o777).toString(8), + fileType: 'binary', + metadata + }; + } +} diff --git a/src/utils/files/text.ts b/src/utils/files/text.ts new file mode 100644 index 00000000..14b5b844 --- /dev/null +++ b/src/utils/files/text.ts @@ -0,0 +1,441 @@ +/** + * Text file handler + * Handles reading, writing, and editing text files + * + * TECHNICAL DEBT: + * This handler is missing editRange() - text search/replace logic currently lives in + * src/tools/edit.ts (performSearchReplace function) instead of here. + * + * For architectural consistency with ExcelFileHandler.editRange(), the fuzzy + * search/replace logic should be moved here. See comment in src/tools/edit.ts. + */ + +import fs from "fs/promises"; +import { createReadStream } from 'fs'; +import { createInterface } from 'readline'; +import { isBinaryFile } from 'isbinaryfile'; +import { + FileHandler, + ReadOptions, + FileResult, + FileInfo +} from './base.js'; + +// Import constants from filesystem.ts +// These will be imported after we organize the code +const FILE_SIZE_LIMITS = { + LARGE_FILE_THRESHOLD: 10 * 1024 * 1024, // 10MB + LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting +} as const; + +const READ_PERFORMANCE_THRESHOLDS = { + SMALL_READ_THRESHOLD: 100, // For very small reads + DEEP_OFFSET_THRESHOLD: 1000, // For byte estimation + SAMPLE_SIZE: 10000, // Sample size for estimation + CHUNK_SIZE: 8192, // 8KB chunks for reverse reading +} as const; + +/** + * Text file handler implementation + */ +export class TextFileHandler implements FileHandler { + canHandle(path: string): boolean { + // Text handler is the default - handles most files + // Only returns false for known non-text formats (checked by other handlers) + return true; + } + + async read(path: string, options?: ReadOptions): Promise { + const offset = options?.offset ?? 0; + const length = options?.length ?? 1000; // Default from config + const includeStatusMessage = options?.includeStatusMessage ?? true; + + // Check if file is binary + const isBinary = await isBinaryFile(path); + if (isBinary) { + throw new Error('Cannot read binary file as text. Use appropriate handler.'); + } + + // Read with smart positioning + return this.readFileWithSmartPositioning(path, offset, length, 'text/plain', includeStatusMessage); + } + + async write(path: string, content: string, mode: 'rewrite' | 'append' = 'rewrite'): Promise { + if (mode === 'append') { + await fs.appendFile(path, content); + } else { + await fs.writeFile(path, content); + } + } + + async getInfo(path: string): Promise { + const stats = await fs.stat(path); + + const info: FileInfo = { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'text', + metadata: {} + }; + + // For text files that aren't too large, count lines + if (stats.isFile() && stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { + try { + const content = await fs.readFile(path, 'utf8'); + const lineCount = this.countLines(content); + info.metadata!.lineCount = lineCount; + } catch (error) { + // If reading fails, skip line count + } + } + + return info; + } + + // ======================================================================== + // Private Helper Methods (extracted from filesystem.ts) + // ======================================================================== + + /** + * Count lines in text content + */ + private countLines(content: string): number { + return content.split('\n').length; + } + + /** + * Get file line count (for files under size limit) + */ + private async getFileLineCount(filePath: string): Promise { + try { + const stats = await fs.stat(filePath); + if (stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { + const content = await fs.readFile(filePath, 'utf8'); + return this.countLines(content); + } + } catch (error) { + // If we can't read the file, return undefined + } + return undefined; + } + + /** + * Generate enhanced status message + */ + private generateEnhancedStatusMessage( + readLines: number, + offset: number, + totalLines?: number, + isNegativeOffset: boolean = false + ): string { + if (isNegativeOffset) { + if (totalLines !== undefined) { + return `[Reading last ${readLines} lines (total: ${totalLines} lines)]`; + } else { + return `[Reading last ${readLines} lines]`; + } + } else { + if (totalLines !== undefined) { + const endLine = offset + readLines; + const remainingLines = Math.max(0, totalLines - endLine); + + if (offset === 0) { + return `[Reading ${readLines} lines from start (total: ${totalLines} lines, ${remainingLines} remaining)]`; + } else { + return `[Reading ${readLines} lines from line ${offset} (total: ${totalLines} lines, ${remainingLines} remaining)]`; + } + } else { + if (offset === 0) { + return `[Reading ${readLines} lines from start]`; + } else { + return `[Reading ${readLines} lines from line ${offset}]`; + } + } + } + } + + /** + * Split text into lines while preserving line endings + */ + private splitLinesPreservingEndings(content: string): string[] { + if (!content) return ['']; + + const lines: string[] = []; + let currentLine = ''; + + for (let i = 0; i < content.length; i++) { + const char = content[i]; + currentLine += char; + + if (char === '\n') { + lines.push(currentLine); + currentLine = ''; + } else if (char === '\r') { + if (i + 1 < content.length && content[i + 1] === '\n') { + currentLine += content[i + 1]; + i++; + } + lines.push(currentLine); + currentLine = ''; + } + } + + if (currentLine) { + lines.push(currentLine); + } + + return lines; + } + + /** + * Read file with smart positioning for optimal performance + */ + private async readFileWithSmartPositioning( + filePath: string, + offset: number, + length: number, + mimeType: string, + includeStatusMessage: boolean = true + ): Promise { + const stats = await fs.stat(filePath); + const fileSize = stats.size; + + const totalLines = await this.getFileLineCount(filePath); + + // For negative offsets (tail behavior), use reverse reading + if (offset < 0) { + const requestedLines = Math.abs(offset); + + if (fileSize > FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD && + requestedLines <= READ_PERFORMANCE_THRESHOLDS.SMALL_READ_THRESHOLD) { + return await this.readLastNLinesReverse(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); + } else { + return await this.readFromEndWithReadline(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); + } + } + // For positive offsets + else { + if (fileSize < FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD || offset === 0) { + return await this.readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); + } else { + if (offset > READ_PERFORMANCE_THRESHOLDS.DEEP_OFFSET_THRESHOLD) { + return await this.readFromEstimatedPosition(filePath, offset, length, mimeType, includeStatusMessage, totalLines); + } else { + return await this.readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); + } + } + } + } + + /** + * Read last N lines efficiently by reading file backwards + */ + private async readLastNLinesReverse( + filePath: string, + n: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + const fd = await fs.open(filePath, 'r'); + try { + const stats = await fd.stat(); + const fileSize = stats.size; + + let position = fileSize; + let lines: string[] = []; + let partialLine = ''; + + while (position > 0 && lines.length < n) { + const readSize = Math.min(READ_PERFORMANCE_THRESHOLDS.CHUNK_SIZE, position); + position -= readSize; + + const buffer = Buffer.alloc(readSize); + await fd.read(buffer, 0, readSize, position); + + const chunk = buffer.toString('utf-8'); + const text = chunk + partialLine; + const chunkLines = text.split('\n'); + + partialLine = chunkLines.shift() || ''; + lines = chunkLines.concat(lines); + } + + if (position === 0 && partialLine) { + lines.unshift(partialLine); + } + + const result = lines.slice(-n); + const content = includeStatusMessage + ? `${this.generateEnhancedStatusMessage(result.length, -n, fileTotalLines, true)}\n\n${result.join('\n')}` + : result.join('\n'); + + return { content, mimeType, metadata: {} }; + } finally { + await fd.close(); + } + } + + /** + * Read from end using readline with circular buffer + */ + private async readFromEndWithReadline( + filePath: string, + requestedLines: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity + }); + + const buffer: string[] = new Array(requestedLines); + let bufferIndex = 0; + let totalLines = 0; + + for await (const line of rl) { + buffer[bufferIndex] = line; + bufferIndex = (bufferIndex + 1) % requestedLines; + totalLines++; + } + + rl.close(); + + let result: string[]; + if (totalLines >= requestedLines) { + result = [ + ...buffer.slice(bufferIndex), + ...buffer.slice(0, bufferIndex) + ].filter(line => line !== undefined); + } else { + result = buffer.slice(0, totalLines); + } + + const content = includeStatusMessage + ? `${this.generateEnhancedStatusMessage(result.length, -requestedLines, fileTotalLines, true)}\n\n${result.join('\n')}` + : result.join('\n'); + + return { content, mimeType, metadata: {} }; + } + + /** + * Read from start/middle using readline + */ + private async readFromStartWithReadline( + filePath: string, + offset: number, + length: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity + }); + + const result: string[] = []; + let lineNumber = 0; + + for await (const line of rl) { + if (lineNumber >= offset && result.length < length) { + result.push(line); + } + if (result.length >= length) break; + lineNumber++; + } + + rl.close(); + + if (includeStatusMessage) { + const statusMessage = this.generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false); + const content = `${statusMessage}\n\n${result.join('\n')}`; + return { content, mimeType, metadata: {} }; + } else { + const content = result.join('\n'); + return { content, mimeType, metadata: {} }; + } + } + + /** + * Read from estimated byte position for very large files + */ + private async readFromEstimatedPosition( + filePath: string, + offset: number, + length: number, + mimeType: string, + includeStatusMessage: boolean = true, + fileTotalLines?: number + ): Promise { + // First, do a quick scan to estimate lines per byte + const rl = createInterface({ + input: createReadStream(filePath), + crlfDelay: Infinity + }); + + let sampleLines = 0; + let bytesRead = 0; + + for await (const line of rl) { + bytesRead += Buffer.byteLength(line, 'utf-8') + 1; + sampleLines++; + if (bytesRead >= READ_PERFORMANCE_THRESHOLDS.SAMPLE_SIZE) break; + } + + rl.close(); + + if (sampleLines === 0) { + return await this.readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, fileTotalLines); + } + + // Estimate position + const avgLineLength = bytesRead / sampleLines; + const estimatedBytePosition = Math.floor(offset * avgLineLength); + + const fd = await fs.open(filePath, 'r'); + try { + const stats = await fd.stat(); + const startPosition = Math.min(estimatedBytePosition, stats.size); + + const stream = createReadStream(filePath, { start: startPosition }); + const rl2 = createInterface({ + input: stream, + crlfDelay: Infinity + }); + + const result: string[] = []; + let firstLineSkipped = false; + + for await (const line of rl2) { + if (!firstLineSkipped && startPosition > 0) { + firstLineSkipped = true; + continue; + } + + if (result.length < length) { + result.push(line); + } else { + break; + } + } + + rl2.close(); + + const content = includeStatusMessage + ? `${this.generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false)}\n\n${result.join('\n')}` + : result.join('\n'); + + return { content, mimeType, metadata: {} }; + } finally { + await fd.close(); + } + } +} diff --git a/test/test-excel-files.js b/test/test-excel-files.js new file mode 100644 index 00000000..8913d552 --- /dev/null +++ b/test/test-excel-files.js @@ -0,0 +1,369 @@ +/** + * Test script for Excel file handling functionality + * + * This script tests the ExcelFileHandler implementation: + * 1. Reading Excel files (basic, sheet selection, range, offset/length) + * 2. Writing Excel files (single sheet, multiple sheets, append mode) + * 3. Editing Excel files (range updates) + * 4. Getting Excel file info (sheet metadata) + * 5. File handler factory (correct handler selection) + */ + +import { configManager } from '../dist/config-manager.js'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import assert from 'assert'; +import { readFile, writeFile, getFileInfo } from '../dist/tools/filesystem.js'; +import { handleEditBlock } from '../dist/handlers/edit-search-handlers.js'; +import { getFileHandler } from '../dist/utils/files/factory.js'; + +// Get directory name +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Define test directory and files +const TEST_DIR = path.join(__dirname, 'test_excel_files'); +const BASIC_EXCEL = path.join(TEST_DIR, 'basic.xlsx'); +const MULTI_SHEET_EXCEL = path.join(TEST_DIR, 'multi_sheet.xlsx'); +const EDIT_EXCEL = path.join(TEST_DIR, 'edit_test.xlsx'); + +/** + * Helper function to clean up test directories + */ +async function cleanupTestDirectories() { + try { + await fs.rm(TEST_DIR, { recursive: true, force: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('Error during cleanup:', error); + } + } +} + +/** + * Setup function to prepare the test environment + */ +async function setup() { + // Clean up before tests (in case previous run left files) + await cleanupTestDirectories(); + + // Create test directory + await fs.mkdir(TEST_DIR, { recursive: true }); + console.log(`✓ Setup: created test directory: ${TEST_DIR}`); + + // Save original config to restore later + const originalConfig = await configManager.getConfig(); + + // Set allowed directories to include our test directory + await configManager.setValue('allowedDirectories', [TEST_DIR]); + console.log(`✓ Setup: set allowed directories`); + + return originalConfig; +} + +/** + * Teardown function to clean up after tests + */ +async function teardown(originalConfig) { + // Reset configuration to original + if (originalConfig) { + await configManager.updateConfig(originalConfig); + } + + await cleanupTestDirectories(); + console.log('✓ Teardown: test directory cleaned up and config restored'); +} + +/** + * Test 1: File handler factory selects ExcelFileHandler for .xlsx files + */ +async function testFileHandlerFactory() { + console.log('\n--- Test 1: File Handler Factory ---'); + + const handler = getFileHandler('test.xlsx'); + assert.ok(handler, 'Handler should be returned for .xlsx file'); + assert.ok(handler.constructor.name === 'ExcelFileHandler', + `Expected ExcelFileHandler but got ${handler.constructor.name}`); + + const txtHandler = getFileHandler('test.txt'); + assert.ok(txtHandler.constructor.name === 'TextFileHandler', + `Expected TextFileHandler for .txt but got ${txtHandler.constructor.name}`); + + console.log('✓ File handler factory correctly selects handlers'); +} + +/** + * Test 2: Write and read basic Excel file + */ +async function testBasicWriteRead() { + console.log('\n--- Test 2: Basic Write and Read ---'); + + // Write a simple Excel file + const data = JSON.stringify([ + ['Name', 'Age', 'City'], + ['Alice', 30, 'New York'], + ['Bob', 25, 'Los Angeles'], + ['Charlie', 35, 'Chicago'] + ]); + + await writeFile(BASIC_EXCEL, data); + console.log('✓ Wrote basic Excel file'); + + // Read it back + const result = await readFile(BASIC_EXCEL); + assert.ok(result.content, 'Should have content'); + // Excel handler returns application/json because content is JSON-formatted for LLM consumption + assert.ok(result.mimeType === 'application/json', + `Expected application/json mime type but got ${result.mimeType}`); + + // Verify content contains our data + const content = result.content.toString(); + assert.ok(content.includes('Name'), 'Content should include Name header'); + assert.ok(content.includes('Alice'), 'Content should include Alice'); + assert.ok(content.includes('Chicago'), 'Content should include Chicago'); + + console.log('✓ Read back Excel file with correct content'); +} + +/** + * Test 3: Write and read multi-sheet Excel file + */ +async function testMultiSheetWriteRead() { + console.log('\n--- Test 3: Multi-Sheet Write and Read ---'); + + // Write multi-sheet Excel file + const data = JSON.stringify({ + 'Employees': [ + ['Name', 'Department'], + ['Alice', 'Engineering'], + ['Bob', 'Sales'] + ], + 'Departments': [ + ['Name', 'Budget'], + ['Engineering', 100000], + ['Sales', 50000] + ] + }); + + await writeFile(MULTI_SHEET_EXCEL, data); + console.log('✓ Wrote multi-sheet Excel file'); + + // Read specific sheet by name + const result1 = await readFile(MULTI_SHEET_EXCEL, { sheet: 'Employees' }); + const content1 = result1.content.toString(); + assert.ok(content1.includes('Alice'), 'Employees sheet should contain Alice'); + assert.ok(content1.includes('Engineering'), 'Employees sheet should contain Engineering'); + console.log('✓ Read Employees sheet by name'); + + // Read specific sheet by index + const result2 = await readFile(MULTI_SHEET_EXCEL, { sheet: 1 }); + const content2 = result2.content.toString(); + assert.ok(content2.includes('Budget'), 'Departments sheet should contain Budget'); + assert.ok(content2.includes('100000'), 'Departments sheet should contain 100000'); + console.log('✓ Read Departments sheet by index'); +} + +/** + * Test 4: Read with range parameter + */ +async function testRangeRead() { + console.log('\n--- Test 4: Range Read ---'); + + // Use the basic file we created + const result = await readFile(BASIC_EXCEL, { sheet: 'Sheet1', range: 'A1:B2' }); + const content = result.content.toString(); + + // Should only have first 2 rows and 2 columns + assert.ok(content.includes('Name'), 'Range should include Name'); + assert.ok(content.includes('Age'), 'Range should include Age'); + assert.ok(content.includes('Alice'), 'Range should include Alice'); + // City is column C, should NOT be included + assert.ok(!content.includes('City') || content.split('City').length === 1, + 'Range A1:B2 should not include City column'); + + console.log('✓ Range read returns correct subset of data'); +} + +/** + * Test 5: Read with offset and length + */ +async function testOffsetLengthRead() { + console.log('\n--- Test 5: Offset and Length Read ---'); + + // Read with offset (skip header) + const result = await readFile(BASIC_EXCEL, { offset: 1, length: 2 }); + const content = result.content.toString(); + + // Should have rows 2-3 (Alice, Bob) but not header or Charlie + assert.ok(content.includes('Alice'), 'Should include Alice (row 2)'); + assert.ok(content.includes('Bob'), 'Should include Bob (row 3)'); + + console.log('✓ Offset and length read works correctly'); +} + +/** + * Test 6: Edit Excel range + */ +async function testEditRange() { + console.log('\n--- Test 6: Edit Excel Range ---'); + + // Create a file to edit + const data = JSON.stringify([ + ['Product', 'Price'], + ['Apple', 1.00], + ['Banana', 0.50], + ['Cherry', 2.00] + ]); + await writeFile(EDIT_EXCEL, data); + console.log('✓ Created file for editing'); + + // Edit a cell using edit_block with range + const editResult = await handleEditBlock({ + file_path: EDIT_EXCEL, + range: 'Sheet1!B2', + content: [[1.50]] // Update Apple price + }); + + assert.ok(!editResult.isError, `Edit should succeed: ${editResult.content?.[0]?.text}`); + console.log('✓ Edit range succeeded'); + + // Verify the edit + const readResult = await readFile(EDIT_EXCEL); + const content = readResult.content.toString(); + assert.ok(content.includes('1.5'), 'Price should be updated to 1.50'); + + console.log('✓ Edit was persisted correctly'); +} + +/** + * Test 7: Get Excel file info + */ +async function testGetFileInfo() { + console.log('\n--- Test 7: Get File Info ---'); + + const info = await getFileInfo(MULTI_SHEET_EXCEL); + + assert.ok(info.isExcelFile, 'Should be marked as Excel file'); + assert.ok(info.sheets, 'Should have sheets info'); + assert.ok(Array.isArray(info.sheets), 'Sheets should be an array'); + assert.strictEqual(info.sheets.length, 2, 'Should have 2 sheets'); + + // Check sheet details + const sheetNames = info.sheets.map(s => s.name); + assert.ok(sheetNames.includes('Employees'), 'Should have Employees sheet'); + assert.ok(sheetNames.includes('Departments'), 'Should have Departments sheet'); + + // Check row/column counts + const employeesSheet = info.sheets.find(s => s.name === 'Employees'); + assert.ok(employeesSheet.rowCount >= 3, 'Employees sheet should have at least 3 rows'); + assert.ok(employeesSheet.colCount >= 2, 'Employees sheet should have at least 2 columns'); + + console.log('✓ File info returns correct sheet metadata'); +} + +/** + * Test 8: Append mode + */ +async function testAppendMode() { + console.log('\n--- Test 8: Append Mode ---'); + + // Create initial file + const initialData = JSON.stringify([ + ['Name', 'Score'], + ['Alice', 100] + ]); + await writeFile(BASIC_EXCEL, initialData); + + // Append more data + const appendData = JSON.stringify([ + ['Bob', 95], + ['Charlie', 88] + ]); + await writeFile(BASIC_EXCEL, appendData, 'append'); + console.log('✓ Appended data to Excel file'); + + // Read and verify + const result = await readFile(BASIC_EXCEL); + const content = result.content.toString(); + + assert.ok(content.includes('Alice'), 'Should still have Alice'); + assert.ok(content.includes('Bob'), 'Should have appended Bob'); + assert.ok(content.includes('Charlie'), 'Should have appended Charlie'); + + console.log('✓ Append mode works correctly'); +} + +/** + * Test 9: Negative offset (read from end) + */ +async function testNegativeOffset() { + console.log('\n--- Test 9: Negative Offset (Tail) ---'); + + // Create file with multiple rows + const data = JSON.stringify([ + ['Row', 'Value'], + ['1', 'First'], + ['2', 'Second'], + ['3', 'Third'], + ['4', 'Fourth'], + ['5', 'Fifth'] + ]); + await writeFile(BASIC_EXCEL, data); + + // Read last 2 rows + const result = await readFile(BASIC_EXCEL, { offset: -2 }); + const content = result.content.toString(); + + assert.ok(content.includes('Fourth') || content.includes('Fifth'), + 'Should include data from last rows'); + + console.log('✓ Negative offset reads from end'); +} + +/** + * Run all tests + */ +async function runAllTests() { + console.log('=== Excel File Handling Tests ===\n'); + + await testFileHandlerFactory(); + await testBasicWriteRead(); + await testMultiSheetWriteRead(); + await testRangeRead(); + await testOffsetLengthRead(); + await testEditRange(); + await testGetFileInfo(); + await testAppendMode(); + await testNegativeOffset(); + + console.log('\n✅ All Excel tests passed!'); +} + +// Export the main test function +export default async function runTests() { + let originalConfig; + try { + originalConfig = await setup(); + await runAllTests(); + } catch (error) { + console.error('❌ Test failed:', error.message); + console.error(error.stack); + return false; + } finally { + if (originalConfig) { + await teardown(originalConfig); + } + } + return true; +} + +// If this file is run directly, execute the test +if (import.meta.url === `file://${process.argv[1]}`) { + runTests().then(success => { + process.exit(success ? 0 : 1); + }).catch(error => { + console.error('❌ Unhandled error:', error); + process.exit(1); + }); +} diff --git a/test/test-file-handlers.js b/test/test-file-handlers.js new file mode 100644 index 00000000..6f83f6e0 --- /dev/null +++ b/test/test-file-handlers.js @@ -0,0 +1,313 @@ +/** + * Test script for file handler system + * + * This script tests the file handler architecture: + * 1. File handler factory returns correct handler types + * 2. FileResult interface consistency + * 3. ReadOptions interface usage + * 4. Handler canHandle() method + * 5. Text file handler basic operations + * 6. Image file handler detection + * 7. Binary file handler fallback + */ + +import { configManager } from '../dist/config-manager.js'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import assert from 'assert'; +import { readFile, writeFile, getFileInfo } from '../dist/tools/filesystem.js'; +import { getFileHandler } from '../dist/utils/files/factory.js'; + +// Get directory name +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Define test directory and files +const TEST_DIR = path.join(__dirname, 'test_file_handlers'); +const TEXT_FILE = path.join(TEST_DIR, 'test.txt'); +const JSON_FILE = path.join(TEST_DIR, 'test.json'); +const MD_FILE = path.join(TEST_DIR, 'test.md'); + +/** + * Helper function to clean up test directories + */ +async function cleanupTestDirectories() { + try { + await fs.rm(TEST_DIR, { recursive: true, force: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + console.error('Error during cleanup:', error); + } + } +} + +/** + * Setup function + */ +async function setup() { + // Clean up before tests (in case previous run left files) + await cleanupTestDirectories(); + + await fs.mkdir(TEST_DIR, { recursive: true }); + console.log(`✓ Setup: created test directory: ${TEST_DIR}`); + + const originalConfig = await configManager.getConfig(); + await configManager.setValue('allowedDirectories', [TEST_DIR]); + + return originalConfig; +} + +/** + * Teardown function + */ +async function teardown(originalConfig) { + if (originalConfig) { + await configManager.updateConfig(originalConfig); + } + + await cleanupTestDirectories(); + console.log('✓ Teardown: cleaned up'); +} + +/** + * Test 1: Handler factory returns correct types + */ +async function testHandlerFactory() { + console.log('\n--- Test 1: Handler Factory Types ---'); + + // Note: TextFileHandler.canHandle() returns true for all files, + // so it catches most files before BinaryFileHandler. + // BinaryFileHandler only handles files that fail binary detection at read time. + const testCases = [ + { file: 'test.xlsx', expected: 'ExcelFileHandler' }, + { file: 'test.xls', expected: 'ExcelFileHandler' }, + { file: 'test.xlsm', expected: 'ExcelFileHandler' }, + { file: 'test.txt', expected: 'TextFileHandler' }, + { file: 'test.js', expected: 'TextFileHandler' }, + { file: 'test.json', expected: 'TextFileHandler' }, + { file: 'test.md', expected: 'TextFileHandler' }, + { file: 'test.png', expected: 'ImageFileHandler' }, + { file: 'test.jpg', expected: 'ImageFileHandler' }, + { file: 'test.jpeg', expected: 'ImageFileHandler' }, + { file: 'test.gif', expected: 'ImageFileHandler' }, + { file: 'test.webp', expected: 'ImageFileHandler' }, + ]; + + for (const { file, expected } of testCases) { + const handler = getFileHandler(file); + assert.strictEqual(handler.constructor.name, expected, + `${file} should use ${expected} but got ${handler.constructor.name}`); + } + + console.log('✓ All file types map to correct handlers'); +} + +/** + * Test 2: FileResult interface consistency + */ +async function testFileResultInterface() { + console.log('\n--- Test 2: FileResult Interface ---'); + + // Create a text file + await fs.writeFile(TEXT_FILE, 'Hello, World!\nLine 2\nLine 3'); + + const result = await readFile(TEXT_FILE); + + // Check FileResult structure + assert.ok('content' in result, 'FileResult should have content'); + assert.ok('mimeType' in result, 'FileResult should have mimeType'); + assert.ok(result.content !== undefined, 'Content should not be undefined'); + assert.ok(typeof result.mimeType === 'string', 'mimeType should be a string'); + + // metadata is optional but should be an object if present + if (result.metadata) { + assert.ok(typeof result.metadata === 'object', 'metadata should be an object'); + } + + console.log('✓ FileResult interface is consistent'); +} + +/** + * Test 3: ReadOptions interface + */ +async function testReadOptionsInterface() { + console.log('\n--- Test 3: ReadOptions Interface ---'); + + await fs.writeFile(TEXT_FILE, 'Line 1\nLine 2\nLine 3\nLine 4\nLine 5'); + + // Test offset option + const result1 = await readFile(TEXT_FILE, { offset: 2 }); + const content1 = result1.content.toString(); + assert.ok(content1.includes('Line 3'), 'Offset should skip to line 3'); + + // Test length option + const result2 = await readFile(TEXT_FILE, { offset: 0, length: 2 }); + const content2 = result2.content.toString(); + assert.ok(content2.includes('Line 1'), 'Should include Line 1'); + assert.ok(content2.includes('Line 2'), 'Should include Line 2'); + + console.log('✓ ReadOptions work correctly'); +} + +/** + * Test 4: Handler canHandle method + */ +async function testCanHandle() { + console.log('\n--- Test 4: canHandle Method ---'); + + const excelHandler = getFileHandler('test.xlsx'); + const textHandler = getFileHandler('test.txt'); + const imageHandler = getFileHandler('test.png'); + + // Excel handler should handle xlsx + assert.ok(excelHandler.canHandle('anything.xlsx'), 'Excel handler should handle .xlsx'); + assert.ok(excelHandler.canHandle('file.xls'), 'Excel handler should handle .xls'); + + // Image handler should handle images + assert.ok(imageHandler.canHandle('photo.png'), 'Image handler should handle .png'); + assert.ok(imageHandler.canHandle('photo.jpg'), 'Image handler should handle .jpg'); + assert.ok(imageHandler.canHandle('photo.jpeg'), 'Image handler should handle .jpeg'); + + // Text handler handles most things (fallback) + assert.ok(textHandler.canHandle('file.txt'), 'Text handler should handle .txt'); + + console.log('✓ canHandle methods work correctly'); +} + +/** + * Test 5: Text handler read/write + */ +async function testTextHandler() { + console.log('\n--- Test 5: Text Handler Operations ---'); + + const content = 'Test content\nWith multiple lines\nAnd special chars: äöü'; + + // Write + await writeFile(TEXT_FILE, content); + console.log('✓ Text write succeeded'); + + // Read + const result = await readFile(TEXT_FILE); + const readContent = result.content.toString(); + assert.ok(readContent.includes('Test content'), 'Should read back content'); + assert.ok(readContent.includes('äöü'), 'Should preserve special characters'); + + console.log('✓ Text handler read/write works'); +} + +/** + * Test 6: Text handler with JSON file + */ +async function testJsonFile() { + console.log('\n--- Test 6: JSON File Handling ---'); + + const data = { name: 'Test', values: [1, 2, 3] }; + const content = JSON.stringify(data, null, 2); + + await writeFile(JSON_FILE, content); + + const result = await readFile(JSON_FILE); + const readContent = result.content.toString(); + const parsed = JSON.parse(readContent.replace(/^\[.*?\]\n\n/, '')); // Remove status message + + assert.strictEqual(parsed.name, 'Test', 'JSON should be preserved'); + assert.deepStrictEqual(parsed.values, [1, 2, 3], 'Array should be preserved'); + + console.log('✓ JSON file handling works'); +} + +/** + * Test 7: File info returns correct structure + */ +async function testFileInfo() { + console.log('\n--- Test 7: File Info Structure ---'); + + await fs.writeFile(TEXT_FILE, 'Some content'); + + const info = await getFileInfo(TEXT_FILE); + + // Check required fields + assert.ok('size' in info, 'Should have size'); + assert.ok('created' in info || 'birthtime' in info, 'Should have creation time'); + assert.ok('modified' in info || 'mtime' in info, 'Should have modification time'); + assert.ok('isFile' in info, 'Should have isFile'); + assert.ok('isDirectory' in info, 'Should have isDirectory'); + + assert.ok(info.size > 0, 'Size should be > 0'); + assert.ok(info.isFile === true || info.isFile === 'true', 'Should be a file'); + + console.log('✓ File info structure is correct'); +} + +/** + * Test 8: Write mode (rewrite vs append) + */ +async function testWriteModes() { + console.log('\n--- Test 8: Write Modes ---'); + + // Initial write (rewrite mode - default) + await writeFile(TEXT_FILE, 'Initial content'); + + // Overwrite + await writeFile(TEXT_FILE, 'New content', 'rewrite'); + let result = await readFile(TEXT_FILE); + let content = result.content.toString(); + assert.ok(!content.includes('Initial'), 'Rewrite should replace content'); + assert.ok(content.includes('New content'), 'Should have new content'); + console.log('✓ Rewrite mode works'); + + // Append + await writeFile(TEXT_FILE, '\nAppended content', 'append'); + result = await readFile(TEXT_FILE); + content = result.content.toString(); + assert.ok(content.includes('New content'), 'Should keep original'); + assert.ok(content.includes('Appended content'), 'Should have appended'); + console.log('✓ Append mode works'); +} + +/** + * Run all tests + */ +async function runAllTests() { + console.log('=== File Handler System Tests ===\n'); + + await testHandlerFactory(); + await testFileResultInterface(); + await testReadOptionsInterface(); + await testCanHandle(); + await testTextHandler(); + await testJsonFile(); + await testFileInfo(); + await testWriteModes(); + + console.log('\n✅ All file handler tests passed!'); +} + +// Export the main test function +export default async function runTests() { + let originalConfig; + try { + originalConfig = await setup(); + await runAllTests(); + } catch (error) { + console.error('❌ Test failed:', error.message); + console.error(error.stack); + return false; + } finally { + if (originalConfig) { + await teardown(originalConfig); + } + } + return true; +} + +// If this file is run directly, execute the test +if (import.meta.url === `file://${process.argv[1]}`) { + runTests().then(success => { + process.exit(success ? 0 : 1); + }).catch(error => { + console.error('❌ Unhandled error:', error); + process.exit(1); + }); +} From aa6ca3a964c11953e9941a091dc487ae796bedce Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Wed, 26 Nov 2025 19:31:02 +0200 Subject: [PATCH 11/17] more emphasis on node for excel processing and analysis --- src/server.ts | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/server.ts b/src/server.ts index 62a9f822..e1b99d2b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1058,9 +1058,38 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { The prompt content will be injected and execution begins immediately. ${CMD_PREFIX_DESCRIPTION}`, - inputSchema: zodToJsonSchema(GetPromptsArgsSchema), - } - ]; + inputSchema: zodToJsonSchema(GetPromptsArgsSchema), + }, + { + name: "execute_node", + description: ` + Execute Node.js code directly using the MCP server's Node runtime. + + PRIMARY TOOL FOR EXCEL FILES AND COMPLEX CALCULATIONS + Use this tool for ANY Excel file (.xlsx, .xls) operations and complex data calculations. + ExcelJS library is built-in and ready to use. + + Code runs as ES module (.mjs) with top-level await support. + Uses the same Node.js environment that runs the MCP server. + + Available libraries: ExcelJS (for Excel file manipulation), and all Node.js built-ins. + + Use cases: Excel file reading/writing/analysis, data transformations, bulk file operations, + complex calculations, JSON processing, or any task better suited to code than tools. + + Output: Use console.log() to return results. Stdout is captured and returned. + + ${PATH_GUIDANCE} + ${CMD_PREFIX_DESCRIPTION}`, + inputSchema: zodToJsonSchema(ExecuteNodeArgsSchema), + annotations: { + title: "Execute Node.js Code", + readOnlyHint: false, + destructiveHint: true, + openWorldHint: true, + }, + } + ]; // Filter tools based on current client const filteredTools = allTools.filter(tool => shouldIncludeTool(tool.name)); From 509f0556b414df0c35318dc212f19c834840385b Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Wed, 26 Nov 2025 22:42:35 +0200 Subject: [PATCH 12/17] fixes from review  Conflicts:  src/tools/filesystem.ts --- src/handlers/filesystem-handlers.ts | 11 ++++- src/search-manager.ts | 59 +++++++++++++++++++----- src/server.ts | 1 + src/tools/filesystem.ts | 70 +++++++++++++++++++++++------ test/test-file-handlers.js | 26 ++++++++--- 5 files changed, 133 insertions(+), 34 deletions(-) diff --git a/src/handlers/filesystem-handlers.ts b/src/handlers/filesystem-handlers.ts index ce41adc4..d6325204 100644 --- a/src/handlers/filesystem-handlers.ts +++ b/src/handlers/filesystem-handlers.ts @@ -103,6 +103,10 @@ export async function handleReadFile(args: unknown): Promise { // Handle image files if (fileResult.metadata?.isImage) { // For image files, return as an image content type + // Content should already be base64-encoded string from handler + const imageData = typeof fileResult.content === 'string' + ? fileResult.content + : fileResult.content.toString('base64'); return { content: [ { @@ -111,15 +115,18 @@ export async function handleReadFile(args: unknown): Promise { }, { type: "image", - data: fileResult.content.toString(), + data: imageData, mimeType: fileResult.mimeType } ], }; } else { // For all other files, return as text + const textContent = typeof fileResult.content === 'string' + ? fileResult.content + : fileResult.content.toString('utf8'); return { - content: [{ type: "text", text: fileResult.content.toString() }], + content: [{ type: "text", text: textContent }], }; } }; diff --git a/src/search-manager.ts b/src/search-manager.ts index d2af0558..8ebacb96 100644 --- a/src/search-manager.ts +++ b/src/search-manager.ts @@ -146,17 +146,21 @@ export interface SearchSessionOptions { validatedPath: validPath }); - // For content searches, also search Excel files in parallel - let excelSearchPromise: Promise | null = null; - if (options.searchType === 'content') { - excelSearchPromise = this.searchExcelFiles( + // For content searches, only search Excel files when contextually relevant: + // - filePattern explicitly targets Excel files (*.xlsx, *.xls, etc.) + // - or rootPath is an Excel file itself + const shouldSearchExcel = options.searchType === 'content' && + this.shouldIncludeExcelSearch(options.filePattern, validPath); + + if (shouldSearchExcel) { + this.searchExcelFiles( validPath, options.pattern, options.ignoreCase !== false, options.maxResults, options.filePattern // Pass filePattern to filter Excel files too ).then(excelResults => { - // Add Excel results to session + // Add Excel results to session (merged after initial response) for (const result of excelResults) { session.results.push(result); session.totalMatches++; @@ -168,6 +172,7 @@ export interface SearchSessionOptions { } // Wait for first chunk of data or early completion instead of fixed delay + // Excel search runs in background and results are merged via readSearchResults const firstChunk = new Promise(resolve => { const onData = () => { session.process.stdout?.off('data', onData); @@ -177,8 +182,8 @@ export interface SearchSessionOptions { setTimeout(resolve, 40); // cap at 40ms instead of 50-100ms }); - // Wait for both ripgrep first chunk and Excel search - await Promise.all([firstChunk, excelSearchPromise].filter(Boolean)); + // Only wait for ripgrep first chunk - Excel results merge asynchronously + await firstChunk; return { sessionId, @@ -507,7 +512,7 @@ export interface SearchSessionOptions { * (has file extension and no glob wildcards) */ private isExactFilename(pattern: string): boolean { - return /\.[a-zA-Z0-9]+$/.test(pattern) && + return /\.[a-zA-Z0-9]+$/.test(pattern) && !this.isGlobPattern(pattern); } @@ -515,14 +520,46 @@ export interface SearchSessionOptions { * Detect if pattern contains glob wildcards */ private isGlobPattern(pattern: string): boolean { - return pattern.includes('*') || - pattern.includes('?') || - pattern.includes('[') || + return pattern.includes('*') || + pattern.includes('?') || + pattern.includes('[') || pattern.includes('{') || pattern.includes(']') || pattern.includes('}'); } + /** + * Determine if Excel search should be included based on context + * Only searches Excel files when: + * - filePattern explicitly targets Excel files (*.xlsx, *.xls, *.xlsm, *.xlsb) + * - or the rootPath itself is an Excel file + */ + private shouldIncludeExcelSearch(filePattern?: string, rootPath?: string): boolean { + const excelExtensions = ['.xlsx', '.xls', '.xlsm', '.xlsb']; + + // Check if rootPath is an Excel file + if (rootPath) { + const lowerPath = rootPath.toLowerCase(); + if (excelExtensions.some(ext => lowerPath.endsWith(ext))) { + return true; + } + } + + // Check if filePattern targets Excel files + if (filePattern) { + const lowerPattern = filePattern.toLowerCase(); + // Check for patterns like *.xlsx, *.xls, or explicit Excel extensions + if (excelExtensions.some(ext => + lowerPattern.includes(`*${ext}`) || + lowerPattern.endsWith(ext) + )) { + return true; + } + } + + return false; + } + private buildRipgrepArgs(options: SearchSessionOptions): string[] { const args: string[] = []; diff --git a/src/server.ts b/src/server.ts index e1b99d2b..9970cdbd 100644 --- a/src/server.ts +++ b/src/server.ts @@ -47,6 +47,7 @@ import { GetPromptsArgsSchema, GetRecentToolCallsArgsSchema, WritePdfArgsSchema, + ExecuteNodeArgsSchema, } from './tools/schemas.js'; import { getConfig, setConfigValue } from './tools/config.js'; import { getUsageStats } from './tools/usage.js'; diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts index 4a1d2c2d..344430ff 100644 --- a/src/tools/filesystem.ts +++ b/src/tools/filesystem.ts @@ -721,9 +721,21 @@ export async function readFileFromDisk( includeStatusMessage: true }); - // Return result - handler provides correct format for each type + // Return with content as string + // For images: content is already base64-encoded string from handler + // For text: content may be string or Buffer, convert to UTF-8 string + let content: string; + if (typeof result.content === 'string') { + content = result.content; + } else if (result.metadata?.isImage) { + // Image buffer should be base64 encoded, not UTF-8 converted + content = result.content.toString('base64'); + } else { + content = result.content.toString('utf8'); + } + return { - content: result.content, + content, mimeType: result.mimeType, metadata: result.metadata }; @@ -891,10 +903,19 @@ export async function readMultipleFiles(paths: string[]): Promise> { const validPath = await validatePath(filePath); + // Get fs.stat as a fallback for any missing fields + const stats = await fs.stat(validPath); + const fallbackInfo = { + size: stats.size, + created: stats.birthtime, + modified: stats.mtime, + accessed: stats.atime, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + permissions: stats.mode.toString(8).slice(-3), + fileType: 'text' as const, + metadata: undefined as Record | undefined, + }; + // Get appropriate handler for this file type const handler = getFileHandler(validPath); - // Use handler to get file info - const fileInfo = await handler.getInfo(validPath); + // Use handler to get file info, with fallback + let fileInfo; + try { + fileInfo = await handler.getInfo(validPath); + } catch (error) { + // If handler fails, use fallback stats + fileInfo = fallbackInfo; + } // Convert to legacy format (for backward compatibility) + // Use handler values with fallback to fs.stat values for any missing fields const info: Record = { - size: fileInfo.size, - created: fileInfo.created, - modified: fileInfo.modified, - accessed: fileInfo.accessed, - isDirectory: fileInfo.isDirectory, - isFile: fileInfo.isFile, - permissions: fileInfo.permissions, - fileType: fileInfo.fileType, + size: fileInfo.size ?? fallbackInfo.size, + created: fileInfo.created ?? fallbackInfo.created, + modified: fileInfo.modified ?? fallbackInfo.modified, + accessed: fileInfo.accessed ?? fallbackInfo.accessed, + isDirectory: fileInfo.isDirectory ?? fallbackInfo.isDirectory, + isFile: fileInfo.isFile ?? fallbackInfo.isFile, + permissions: fileInfo.permissions ?? fallbackInfo.permissions, + fileType: fileInfo.fileType ?? fallbackInfo.fileType, }; // Add type-specific metadata from file handler diff --git a/test/test-file-handlers.js b/test/test-file-handlers.js index 6f83f6e0..1adcf7ec 100644 --- a/test/test-file-handlers.js +++ b/test/test-file-handlers.js @@ -60,14 +60,26 @@ async function setup() { /** * Teardown function + * Always runs cleanup, restores config only if provided */ async function teardown(originalConfig) { - if (originalConfig) { - await configManager.updateConfig(originalConfig); + // Always clean up test directories, even if setup failed + try { + await cleanupTestDirectories(); + console.log('✓ Teardown: cleaned up test directories'); + } catch (error) { + console.error('Warning: Failed to clean up test directories:', error.message); } - await cleanupTestDirectories(); - console.log('✓ Teardown: cleaned up'); + // Restore config only if we have the original + if (originalConfig) { + try { + await configManager.updateConfig(originalConfig); + console.log('✓ Teardown: restored config'); + } catch (error) { + console.error('Warning: Failed to restore config:', error.message); + } + } } /** @@ -295,9 +307,9 @@ export default async function runTests() { console.error(error.stack); return false; } finally { - if (originalConfig) { - await teardown(originalConfig); - } + // Always run teardown to clean up test directories and restore config + // teardown handles the case where originalConfig is undefined + await teardown(originalConfig); } return true; } From cfd4867c59e7a8ca85355833153d31b2f2a63882 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Fri, 28 Nov 2025 20:35:10 +0200 Subject: [PATCH 13/17] removing dead code and prompts left from testing --- src/tools/edit.ts | 2 +- src/tools/filesystem.ts | 412 ++----------------------------------- src/utils/files/base.ts | 4 +- src/utils/files/binary.ts | 122 ++--------- src/utils/files/factory.ts | 90 +++++--- src/utils/files/text.ts | 34 +-- 6 files changed, 105 insertions(+), 559 deletions(-) diff --git a/src/tools/edit.ts b/src/tools/edit.ts index 05453cf0..590c1fb7 100644 --- a/src/tools/edit.ts +++ b/src/tools/edit.ts @@ -374,7 +374,7 @@ export async function handleEditBlock(args: unknown): Promise { if (parsed.range && parsed.content !== undefined) { try { const { getFileHandler } = await import('../utils/files/factory.js'); - const handler = getFileHandler(parsed.file_path); + const handler = await getFileHandler(parsed.file_path); // Parse content if it's a JSON string (AI often sends arrays as JSON strings) let content = parsed.content; diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts index 344430ff..96b41ef3 100644 --- a/src/tools/filesystem.ts +++ b/src/tools/filesystem.ts @@ -2,66 +2,28 @@ import fs from "fs/promises"; import path from "path"; import os from 'os'; import fetch from 'cross-fetch'; -import { createReadStream } from 'fs'; -import { createInterface } from 'readline'; -import { isBinaryFile } from 'isbinaryfile'; import { capture } from '../utils/capture.js'; import { withTimeout } from '../utils/withTimeout.js'; import { configManager } from '../config-manager.js'; -import { isPdfFile } from "./mime-types.js"; -import { editPdf, PdfOperations, PdfMetadata, parsePdfToMarkdown, parseMarkdownToPdf } from './pdf/index.js'; -import { getFileHandler } from '../utils/files/factory.js'; +import { getFileHandler, TextFileHandler } from '../utils/files/index.js'; import type { ReadOptions, FileResult, PdfPageItem } from '../utils/files/base.js'; +import { isPdfFile } from "./mime-types.js"; +import { parsePdfToMarkdown, editPdf, PdfOperations, PdfMetadata, parseMarkdownToPdf } from './pdf/index.js'; +import { isBinaryFile } from 'isbinaryfile'; // CONSTANTS SECTION - Consolidate all timeouts and thresholds const FILE_OPERATION_TIMEOUTS = { PATH_VALIDATION: 10000, // 10 seconds - URL_FETCH: 30000, // 30 seconds + URL_FETCH: 30000, // 30 seconds FILE_READ: 30000, // 30 seconds } as const; const FILE_SIZE_LIMITS = { - LARGE_FILE_THRESHOLD: 10 * 1024 * 1024, // 10MB LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting } as const; -const READ_PERFORMANCE_THRESHOLDS = { - SMALL_READ_THRESHOLD: 100, // For very small reads - DEEP_OFFSET_THRESHOLD: 1000, // For byte estimation - SAMPLE_SIZE: 10000, // Sample size for estimation - CHUNK_SIZE: 8192, // 8KB chunks for reverse reading -} as const; - // UTILITY FUNCTIONS - Eliminate duplication -/** - * Count lines in text content efficiently - * @param content Text content to count lines in - * @returns Number of lines - */ -function countLines(content: string): number { - return content.split('\n').length; -} - -/** - * Count lines in a file efficiently (for files under size limit) - * @param filePath Path to the file - * @returns Line count or undefined if file too large/can't read - */ -async function getFileLineCount(filePath: string): Promise { - try { - const stats = await fs.stat(filePath); - // Only count lines for reasonably sized files to avoid performance issues - if (stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { - const content = await fs.readFile(filePath, 'utf8'); - return countLines(content); - } - } catch (error) { - // If we can't read the file, just return undefined - } - return undefined; -} - /** * Get MIME type information for a file * @param filePath Path to the file @@ -93,22 +55,6 @@ async function getDefaultReadLength(): Promise { return config.fileReadLineLimit ?? 1000; // Default to 1000 lines if not set } -/** - * Generate instructions for handling binary files - * @param filePath Path to the binary file - * @param mimeType MIME type of the file - * @returns Instruction message for the LLM - */ -function getBinaryFileInstructions(filePath: string, mimeType: string): string { - const fileName = path.basename(filePath); - - return `Cannot read binary file as text: ${fileName} (${mimeType}) - -Use start_process + interact_with_process to analyze binary files with appropriate tools (Node.js or Python libraries, command-line utilities, etc.). - -The read_file tool only handles text files, images or PDFs.`; -} - // Initialize allowed directories from configuration async function getAllowedDirs(): Promise { try { @@ -288,7 +234,6 @@ export async function validatePath(requestedPath: string): Promise { // Re-export FileResult from base for consumers export type { FileResult } from '../utils/files/base.js'; -// Type alias for backward compatibility with PDF handling code type PdfPayload = { metadata: PdfMetadata; pages: PdfPageItem[]; @@ -369,298 +314,6 @@ export async function readFileFromUrl(url: string): Promise { } } - - -/** - * Generate enhanced status message with total and remaining line information - * @param readLines Number of lines actually read - * @param offset Starting offset (line number) - * @param totalLines Total lines in the file (if available) - * @param isNegativeOffset Whether this is a tail operation - * @returns Enhanced status message string - */ -function generateEnhancedStatusMessage( - readLines: number, - offset: number, - totalLines?: number, - isNegativeOffset: boolean = false -): string { - if (isNegativeOffset) { - // For tail operations (negative offset) - if (totalLines !== undefined) { - return `[Reading last ${readLines} lines (total: ${totalLines} lines)]`; - } else { - return `[Reading last ${readLines} lines]`; - } - } else { - // For normal reads (positive offset) - if (totalLines !== undefined) { - const endLine = offset + readLines; - const remainingLines = Math.max(0, totalLines - endLine); - - if (offset === 0) { - return `[Reading ${readLines} lines from start (total: ${totalLines} lines, ${remainingLines} remaining)]`; - } else { - return `[Reading ${readLines} lines from line ${offset} (total: ${totalLines} lines, ${remainingLines} remaining)]`; - } - } else { - // Fallback when total lines unknown - if (offset === 0) { - return `[Reading ${readLines} lines from start]`; - } else { - return `[Reading ${readLines} lines from line ${offset}]`; - } - } - } -} - -/** - * Read file content using smart positioning for optimal performance - * @param filePath Path to the file (already validated) - * @param offset Starting line number (negative for tail behavior) - * @param length Maximum number of lines to read - * @param mimeType MIME type of the file - * @param includeStatusMessage Whether to include status headers (default: true) - * @returns File result with content - */ -async function readFileWithSmartPositioning(filePath: string, offset: number, length: number, mimeType: string, includeStatusMessage: boolean = true): Promise { - const stats = await fs.stat(filePath); - const fileSize = stats.size; - - // Check if the file is binary (but allow images to pass through) - const { isImage } = await getMimeTypeInfo(filePath); - if (!isImage) { - const isBinary = await isBinaryFile(filePath); - if (isBinary) { - // Return instructions instead of trying to read binary content - const instructions = getBinaryFileInstructions(filePath, mimeType); - throw new Error(instructions); - } - } - - // Get total line count for enhanced status messages (only for smaller files) - const totalLines = await getFileLineCount(filePath); - - // For negative offsets (tail behavior), use reverse reading - if (offset < 0) { - const requestedLines = Math.abs(offset); - - if (fileSize > FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD && requestedLines <= READ_PERFORMANCE_THRESHOLDS.SMALL_READ_THRESHOLD) { - // Use efficient reverse reading for large files with small tail requests - return await readLastNLinesReverse(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); - } else { - // Use readline circular buffer for other cases - return await readFromEndWithReadline(filePath, requestedLines, mimeType, includeStatusMessage, totalLines); - } - } - - // For positive offsets - else { - // For small files or reading from start, use simple readline - if (fileSize < FILE_SIZE_LIMITS.LARGE_FILE_THRESHOLD || offset === 0) { - return await readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); - } - - // For large files with middle/end reads, try to estimate position - else { - // If seeking deep into file, try byte estimation - if (offset > READ_PERFORMANCE_THRESHOLDS.DEEP_OFFSET_THRESHOLD) { - return await readFromEstimatedPosition(filePath, offset, length, mimeType, includeStatusMessage, totalLines); - } else { - return await readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, totalLines); - } - } - } -} - -/** - * Read last N lines efficiently by reading file backwards in chunks - */ -async function readLastNLinesReverse(filePath: string, n: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - const fd = await fs.open(filePath, 'r'); - try { - const stats = await fd.stat(); - const fileSize = stats.size; - - let position = fileSize; - let lines: string[] = []; - let partialLine = ''; - - while (position > 0 && lines.length < n) { - const readSize = Math.min(READ_PERFORMANCE_THRESHOLDS.CHUNK_SIZE, position); - position -= readSize; - - const buffer = Buffer.alloc(readSize); - await fd.read(buffer, 0, readSize, position); - - const chunk = buffer.toString('utf-8'); - const text = chunk + partialLine; - const chunkLines = text.split('\n'); - - partialLine = chunkLines.shift() || ''; - lines = chunkLines.concat(lines); - } - - // Add the remaining partial line if we reached the beginning - if (position === 0 && partialLine) { - lines.unshift(partialLine); - } - - const result = lines.slice(-n); // Get exactly n lines - const content = includeStatusMessage - ? `${generateEnhancedStatusMessage(result.length, -n, fileTotalLines, true)}\n\n${result.join('\n')}` - : result.join('\n'); - - return { content, mimeType, metadata: { isImage: false } }; - } finally { - await fd.close(); - } -} - -/** - * Read from end using readline with circular buffer - */ -async function readFromEndWithReadline(filePath: string, requestedLines: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - const rl = createInterface({ - input: createReadStream(filePath), - crlfDelay: Infinity - }); - - const buffer: string[] = new Array(requestedLines); - let bufferIndex = 0; - let totalLines = 0; - - for await (const line of rl) { - buffer[bufferIndex] = line; - bufferIndex = (bufferIndex + 1) % requestedLines; - totalLines++; - } - - rl.close(); - - // Extract lines in correct order - let result: string[]; - if (totalLines >= requestedLines) { - result = [ - ...buffer.slice(bufferIndex), - ...buffer.slice(0, bufferIndex) - ].filter(line => line !== undefined); - } else { - result = buffer.slice(0, totalLines); - } - - const content = includeStatusMessage - ? `${generateEnhancedStatusMessage(result.length, -requestedLines, fileTotalLines, true)}\n\n${result.join('\n')}` - : result.join('\n'); - return { content, mimeType, metadata: { isImage: false } }; -} - -/** - * Read from start/middle using readline - */ -async function readFromStartWithReadline(filePath: string, offset: number, length: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - const rl = createInterface({ - input: createReadStream(filePath), - crlfDelay: Infinity - }); - - const result: string[] = []; - let lineNumber = 0; - - for await (const line of rl) { - if (lineNumber >= offset && result.length < length) { - result.push(line); - } - if (result.length >= length) break; // Early exit optimization - lineNumber++; - } - - rl.close(); - - if (includeStatusMessage) { - const statusMessage = generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false); - const content = `${statusMessage}\n\n${result.join('\n')}`; - return { content, mimeType, metadata: { isImage: false } }; - } else { - const content = result.join('\n'); - return { content, mimeType, metadata: { isImage: false } }; - } -} - -/** - * Read from estimated byte position for very large files - */ -async function readFromEstimatedPosition(filePath: string, offset: number, length: number, mimeType: string, includeStatusMessage: boolean = true, fileTotalLines?: number): Promise { - // First, do a quick scan to estimate lines per byte - const rl = createInterface({ - input: createReadStream(filePath), - crlfDelay: Infinity - }); - - let sampleLines = 0; - let bytesRead = 0; - - - - for await (const line of rl) { - bytesRead += Buffer.byteLength(line, 'utf-8') + 1; // +1 for newline - sampleLines++; - if (bytesRead >= READ_PERFORMANCE_THRESHOLDS.SAMPLE_SIZE) break; - } - - rl.close(); - - if (sampleLines === 0) { - // Fallback to simple read - return await readFromStartWithReadline(filePath, offset, length, mimeType, includeStatusMessage, fileTotalLines); - } - - // Estimate average line length and seek position - const avgLineLength = bytesRead / sampleLines; - const estimatedBytePosition = Math.floor(offset * avgLineLength); - - // Create a new stream starting from estimated position - const fd = await fs.open(filePath, 'r'); - try { - const stats = await fd.stat(); - const startPosition = Math.min(estimatedBytePosition, stats.size); - - const stream = createReadStream(filePath, { start: startPosition }); - const rl2 = createInterface({ - input: stream, - crlfDelay: Infinity - }); - - const result: string[] = []; - let lineCount = 0; - let firstLineSkipped = false; - - for await (const line of rl2) { - // Skip first potentially partial line if we didn't start at beginning - if (!firstLineSkipped && startPosition > 0) { - firstLineSkipped = true; - continue; - } - - if (result.length < length) { - result.push(line); - } else { - break; - } - lineCount++; - } - - rl2.close(); - - const content = includeStatusMessage - ? `${generateEnhancedStatusMessage(result.length, offset, fileTotalLines, false)}\n\n${result.join('\n')}` - : result.join('\n'); - return { content, mimeType, metadata: { isImage: false } }; - } finally { - await fd.close(); - } -} - /** * Read file content from the local filesystem * @param filePath Path to the file @@ -709,8 +362,8 @@ export async function readFileFromDisk( // Use withTimeout to handle potential hangs const readOperation = async () => { - // Get appropriate handler for this file type (now includes PDF handler) - const handler = getFileHandler(validPath); + // Get appropriate handler for this file type (async - includes binary detection) + const handler = await getFileHandler(validPath); // Use handler to read the file const result = await handler.read(validPath, { @@ -812,7 +465,7 @@ export async function readFileInternal(filePath: string, offset: number = 0, len } // Handle offset/length by splitting on line boundaries while preserving line endings - const lines = splitLinesPreservingEndings(content); + const lines = TextFileHandler.splitLinesPreservingEndings(content); // Apply offset and length const selectedLines = lines.slice(offset, offset + length); @@ -821,47 +474,6 @@ export async function readFileInternal(filePath: string, offset: number = 0, len return selectedLines.join(''); } -/** - * Split text into lines while preserving original line endings with each line - * @param content The text content to split - * @returns Array of lines, each including its original line ending - */ -function splitLinesPreservingEndings(content: string): string[] { - if (!content) return ['']; - - const lines: string[] = []; - let currentLine = ''; - - for (let i = 0; i < content.length; i++) { - const char = content[i]; - currentLine += char; - - // Check for line ending patterns - if (char === '\n') { - // LF or end of CRLF - lines.push(currentLine); - currentLine = ''; - } else if (char === '\r') { - // Could be CR or start of CRLF - if (i + 1 < content.length && content[i + 1] === '\n') { - // It's CRLF, include the \n as well - currentLine += content[i + 1]; - i++; // Skip the \n in next iteration - } - // Either way, we have a complete line - lines.push(currentLine); - currentLine = ''; - } - } - - // Handle any remaining content (file not ending with line ending) - if (currentLine) { - lines.push(currentLine); - } - - return lines; -} - export async function writeFile(filePath: string, content: string, mode: 'rewrite' | 'append' = 'rewrite'): Promise { const validPath = await validatePath(filePath); @@ -870,7 +482,7 @@ export async function writeFile(filePath: string, content: string, mode: 'rewrit // Calculate content metrics const contentBytes = Buffer.from(content).length; - const lineCount = countLines(content); + const lineCount = TextFileHandler.countLines(content); // Capture file extension and operation details in telemetry without capturing the file path capture('server_write_file', { @@ -880,8 +492,8 @@ export async function writeFile(filePath: string, content: string, mode: 'rewrit lineCount: lineCount }); - // Get appropriate handler for this file type - const handler = getFileHandler(validPath); + // Get appropriate handler for this file type (async - includes binary detection) + const handler = await getFileHandler(validPath); // Use handler to write the file await handler.write(validPath, content, mode); @@ -1153,7 +765,7 @@ export async function getFileInfo(filePath: string): Promise }; // Get appropriate handler for this file type - const handler = getFileHandler(validPath); + const handler = await getFileHandler(validPath); // Use handler to get file info, with fallback let fileInfo; diff --git a/src/utils/files/base.ts b/src/utils/files/base.ts index a275441d..775acdb6 100644 --- a/src/utils/files/base.ts +++ b/src/utils/files/base.ts @@ -58,9 +58,9 @@ export interface FileHandler { /** * Check if this handler can handle the given file * @param path File path - * @returns true if this handler supports this file type + * @returns true if this handler supports this file type (can be async for content-based checks) */ - canHandle(path: string): boolean; + canHandle(path: string): boolean | Promise; } // ============================================================================ diff --git a/src/utils/files/binary.ts b/src/utils/files/binary.ts index a934e833..245bb01f 100644 --- a/src/utils/files/binary.ts +++ b/src/utils/files/binary.ts @@ -1,11 +1,13 @@ /** * Binary file handler - * Catch-all handler for unsupported binary files + * Handles binary files that aren't supported by other handlers (Excel, Image) + * Uses isBinaryFile for content-based detection * Returns instructions to use start_process with appropriate tools */ import fs from "fs/promises"; import path from "path"; +import { isBinaryFile } from 'isbinaryfile'; import { FileHandler, ReadOptions, @@ -15,12 +17,17 @@ import { /** * Binary file handler implementation - * This is a catch-all handler for binary files that aren't supported by other handlers + * Uses content-based detection via isBinaryFile */ export class BinaryFileHandler implements FileHandler { - canHandle(path: string): boolean { - // Binary handler is the catch-all - handles everything not handled by other handlers - return true; + async canHandle(filePath: string): Promise { + // Content-based binary detection using isBinaryFile + try { + return await isBinaryFile(filePath); + } catch (error) { + // If we can't check (file doesn't exist, etc.), don't handle it + return false; + } } async read(filePath: string, options?: ReadOptions): Promise { @@ -62,112 +69,11 @@ export class BinaryFileHandler implements FileHandler { */ private getBinaryInstructions(filePath: string): string { const fileName = path.basename(filePath); - const ext = path.extname(filePath).toLowerCase(); - - // Get MIME type suggestion based on extension - const mimeType = this.guessMimeType(ext); - - let specificGuidance = ''; - - // Provide specific guidance based on file type - switch (ext) { - case '.pdf': - specificGuidance = ` -PDF FILES: -- Python: PyPDF2, pdfplumber - start_process("python -i") - interact_with_process(pid, "import pdfplumber") - interact_with_process(pid, "pdf = pdfplumber.open('${filePath}')") - interact_with_process(pid, "print(pdf.pages[0].extract_text())") - -- Node.js: pdf-parse - start_process("node -i") - interact_with_process(pid, "const pdf = require('pdf-parse')")`; - break; - case '.doc': - case '.docx': - specificGuidance = ` -WORD DOCUMENTS: -- Python: python-docx - start_process("python -i") - interact_with_process(pid, "import docx") - interact_with_process(pid, "doc = docx.Document('${filePath}')") - interact_with_process(pid, "for para in doc.paragraphs: print(para.text)") + return `Cannot read binary file as text: ${fileName} -- Node.js: mammoth - start_process("node -i") - interact_with_process(pid, "const mammoth = require('mammoth')")`; - break; - - case '.zip': - case '.tar': - case '.gz': - specificGuidance = ` -ARCHIVE FILES: -- Python: zipfile, tarfile - start_process("python -i") - interact_with_process(pid, "import zipfile") - interact_with_process(pid, "with zipfile.ZipFile('${filePath}') as z: print(z.namelist())") - -- Command-line: - start_process("unzip -l ${filePath}") # For ZIP files - start_process("tar -tzf ${filePath}") # For TAR files`; - break; - - case '.db': - case '.sqlite': - case '.sqlite3': - specificGuidance = ` -SQLITE DATABASES: -- Python: sqlite3 - start_process("python -i") - interact_with_process(pid, "import sqlite3") - interact_with_process(pid, "conn = sqlite3.connect('${filePath}')") - interact_with_process(pid, "cursor = conn.cursor()") - interact_with_process(pid, "cursor.execute('SELECT * FROM sqlite_master')") - -- Command-line: - start_process("sqlite3 ${filePath} '.tables'")`; - break; - - default: - specificGuidance = ` -GENERIC BINARY FILES: -- Use appropriate libraries based on file type -- Python libraries: Check PyPI for ${ext} support -- Node.js libraries: Check npm for ${ext} support -- Command-line tools: Use file-specific utilities`; - } - - return `Cannot read binary file as text: ${fileName} (${mimeType}) - -Use start_process + interact_with_process to analyze binary files with appropriate tools. -${specificGuidance} +Use start_process + interact_with_process to analyze binary files with appropriate tools (Node.js or Python libraries, command-line utilities, etc.). The read_file tool only handles text files, images, and Excel files.`; } - - /** - * Guess MIME type based on file extension - */ - private guessMimeType(ext: string): string { - const mimeTypes: { [key: string]: string } = { - '.pdf': 'application/pdf', - '.doc': 'application/msword', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.zip': 'application/zip', - '.tar': 'application/x-tar', - '.gz': 'application/gzip', - '.db': 'application/x-sqlite3', - '.sqlite': 'application/x-sqlite3', - '.sqlite3': 'application/x-sqlite3', - '.mp3': 'audio/mpeg', - '.mp4': 'video/mp4', - '.avi': 'video/x-msvideo', - '.mkv': 'video/x-matroska', - }; - - return mimeTypes[ext] || 'application/octet-stream'; - } } diff --git a/src/utils/files/factory.ts b/src/utils/files/factory.ts index d911e1ae..5fba5ff5 100644 --- a/src/utils/files/factory.ts +++ b/src/utils/files/factory.ts @@ -1,6 +1,9 @@ /** * Factory pattern for creating appropriate file handlers * Routes file operations to the correct handler based on file type + * + * Each handler implements canHandle() which can be sync (extension-based) + * or async (content-based like BinaryFileHandler using isBinaryFile) */ import { FileHandler } from './base.js'; @@ -11,55 +14,80 @@ import { ExcelFileHandler } from './excel.js'; import { PdfFileHandler } from './pdf.js'; // Singleton instances of each handler -let handlers: FileHandler[] | null = null; +let excelHandler: ExcelFileHandler | null = null; +let imageHandler: ImageFileHandler | null = null; +let textHandler: TextFileHandler | null = null; +let binaryHandler: BinaryFileHandler | null = null; +let pdfHandler: PdfFileHandler | null = null; /** * Initialize handlers (lazy initialization) */ -function initializeHandlers(): FileHandler[] { - if (handlers) { - return handlers; - } +function getExcelHandler(): ExcelFileHandler { + if (!excelHandler) excelHandler = new ExcelFileHandler(); + return excelHandler; +} + +function getImageHandler(): ImageFileHandler { + if (!imageHandler) imageHandler = new ImageFileHandler(); + return imageHandler; +} + +function getTextHandler(): TextFileHandler { + if (!textHandler) textHandler = new TextFileHandler(); + return textHandler; +} - handlers = [ - // Order matters! More specific handlers first - new PdfFileHandler(), // Check PDF first - new ExcelFileHandler(), // Check Excel (before binary) - new ImageFileHandler(), // Then images - new TextFileHandler(), // Then text (handles most files) - new BinaryFileHandler(), // Finally binary (catch-all) - ]; +function getBinaryHandler(): BinaryFileHandler { + if (!binaryHandler) binaryHandler = new BinaryFileHandler(); + return binaryHandler; +} - return handlers; +function getPdfHandler(): PdfFileHandler { + if (!pdfHandler) pdfHandler = new PdfFileHandler(); + return pdfHandler; } /** * Get the appropriate file handler for a given file path * - * This function checks each handler in priority order and returns the first - * handler that can handle the file type. + * Each handler's canHandle() determines if it can process the file. + * Extension-based handlers (Excel, Image) return sync boolean. + * BinaryFileHandler uses async isBinaryFile for content-based detection. * * Priority order: - * 1. Excel files (xlsx, xls, xlsm) - * 2. Image files (png, jpg, gif, webp) - * 3. Text files (most other files) - * 4. Binary files (catch-all for unsupported formats) + * 1. PDF files (extension based) + * 2. Excel files (xlsx, xls, xlsm) - extension based + * 3. Image files (png, jpg, gif, webp) - extension based + * 4. Binary files - content-based detection via isBinaryFile + * 5. Text files (default) * - * @param path File path (can be before or after validation) + * @param filePath File path to get handler for * @returns FileHandler instance that can handle this file */ -export function getFileHandler(path: string): FileHandler { - const allHandlers = initializeHandlers(); +export async function getFileHandler(filePath: string): Promise { + // Check PDF first (extension-based, sync) + if (getPdfHandler().canHandle(filePath)) { + return getPdfHandler(); + } + + // Check Excel (extension-based, sync) + if (getExcelHandler().canHandle(filePath)) { + return getExcelHandler(); + } - // Try each handler in order - for (const handler of allHandlers) { - if (handler.canHandle(path)) { - return handler; - } + // Check Image (extension-based, sync - images are binary but handled specially) + if (getImageHandler().canHandle(filePath)) { + return getImageHandler(); } - // Fallback to binary handler (should never reach here due to binary catch-all) - return allHandlers[allHandlers.length - 1]; + // Check Binary (content-based, async via isBinaryFile) + if (await getBinaryHandler().canHandle(filePath)) { + return getBinaryHandler(); + } + + // Default to text handler + return getTextHandler(); } /** @@ -88,4 +116,4 @@ export function isImageFile(path: string): boolean { ext.endsWith('.webp') || ext.endsWith('.bmp') || ext.endsWith('.svg'); -} +} \ No newline at end of file diff --git a/src/utils/files/text.ts b/src/utils/files/text.ts index 14b5b844..3e00b11b 100644 --- a/src/utils/files/text.ts +++ b/src/utils/files/text.ts @@ -2,6 +2,9 @@ * Text file handler * Handles reading, writing, and editing text files * + * Binary detection is handled at the factory level (factory.ts) using isBinaryFile. + * This handler only receives files that have been confirmed as text. + * * TECHNICAL DEBT: * This handler is missing editRange() - text search/replace logic currently lives in * src/tools/edit.ts (performSearchReplace function) instead of here. @@ -11,9 +14,9 @@ */ import fs from "fs/promises"; +import path from "path"; import { createReadStream } from 'fs'; import { createInterface } from 'readline'; -import { isBinaryFile } from 'isbinaryfile'; import { FileHandler, ReadOptions, @@ -37,27 +40,22 @@ const READ_PERFORMANCE_THRESHOLDS = { /** * Text file handler implementation + * Binary detection is done at the factory level - this handler assumes file is text */ export class TextFileHandler implements FileHandler { - canHandle(path: string): boolean { - // Text handler is the default - handles most files - // Only returns false for known non-text formats (checked by other handlers) + canHandle(_path: string): boolean { + // Text handler accepts all files that pass the factory's binary check + // The factory routes binary files to BinaryFileHandler before reaching here return true; } - async read(path: string, options?: ReadOptions): Promise { + async read(filePath: string, options?: ReadOptions): Promise { const offset = options?.offset ?? 0; const length = options?.length ?? 1000; // Default from config const includeStatusMessage = options?.includeStatusMessage ?? true; - // Check if file is binary - const isBinary = await isBinaryFile(path); - if (isBinary) { - throw new Error('Cannot read binary file as text. Use appropriate handler.'); - } - - // Read with smart positioning - return this.readFileWithSmartPositioning(path, offset, length, 'text/plain', includeStatusMessage); + // Binary detection is done at factory level - just read as text + return this.readFileWithSmartPositioning(filePath, offset, length, 'text/plain', includeStatusMessage); } async write(path: string, content: string, mode: 'rewrite' | 'append' = 'rewrite'): Promise { @@ -87,7 +85,7 @@ export class TextFileHandler implements FileHandler { if (stats.isFile() && stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { try { const content = await fs.readFile(path, 'utf8'); - const lineCount = this.countLines(content); + const lineCount = TextFileHandler.countLines(content); info.metadata!.lineCount = lineCount; } catch (error) { // If reading fails, skip line count @@ -103,8 +101,9 @@ export class TextFileHandler implements FileHandler { /** * Count lines in text content + * Made static and public for use by other modules (e.g., writeFile telemetry in filesystem.ts) */ - private countLines(content: string): number { + static countLines(content: string): number { return content.split('\n').length; } @@ -116,7 +115,7 @@ export class TextFileHandler implements FileHandler { const stats = await fs.stat(filePath); if (stats.size < FILE_SIZE_LIMITS.LINE_COUNT_LIMIT) { const content = await fs.readFile(filePath, 'utf8'); - return this.countLines(content); + return TextFileHandler.countLines(content); } } catch (error) { // If we can't read the file, return undefined @@ -161,8 +160,9 @@ export class TextFileHandler implements FileHandler { /** * Split text into lines while preserving line endings + * Made static and public for use by other modules (e.g., readFileInternal in filesystem.ts) */ - private splitLinesPreservingEndings(content: string): string[] { + static splitLinesPreservingEndings(content: string): string[] { if (!content) return ['']; const lines: string[] = []; From 44372d7e4128cdac9f8ecbc0ecea838f9ce08ce8 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Sat, 29 Nov 2025 19:24:51 +0200 Subject: [PATCH 14/17] Replace execute_node tool with node:local command and add Python detection - Remove execute_node tool in favor of start_process("node:local") - Add pythonInfo to system info for LLM to check Python availability - Add node:local virtual session handling in improved-process-tools.ts - Add validatePath to edit.ts range+content branch - Fix parameter guard (range !== undefined instead of truthy) - Clean up: Remove node-handlers.ts and ExecuteNodeArgsSchema - Simplify tool descriptions, document node:local fallback The node:local approach is cleaner - LLM uses existing start_process/ interact_with_process flow instead of a separate tool. This is the final fallback when users dont have node and dont have python, we run the code on server, that's the idea. Python detection lets LLM decide when to fall back to Node.js execution. --- src/handlers/index.ts | 1 - src/handlers/node-handlers.ts | 90 -------------------- src/server.ts | 59 +++---------- src/tools/edit.ts | 24 ++++-- src/tools/improved-process-tools.ts | 123 +++++++++++++++++++++++++++- src/tools/schemas.ts | 5 -- src/utils/files/factory.ts | 18 ++-- src/utils/files/text.ts | 5 +- src/utils/system-info.ts | 51 +++++++++++- 9 files changed, 207 insertions(+), 169 deletions(-) delete mode 100644 src/handlers/node-handlers.ts diff --git a/src/handlers/index.ts b/src/handlers/index.ts index 10f6eebd..1ac19090 100644 --- a/src/handlers/index.ts +++ b/src/handlers/index.ts @@ -5,4 +5,3 @@ export * from './process-handlers.js'; export * from './edit-search-handlers.js'; export * from './search-handlers.js'; export * from './history-handlers.js'; -export * from './node-handlers.js'; diff --git a/src/handlers/node-handlers.ts b/src/handlers/node-handlers.ts deleted file mode 100644 index cbb76c7d..00000000 --- a/src/handlers/node-handlers.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { spawn } from 'child_process'; -import fs from 'fs/promises'; -import path from 'path'; -import { fileURLToPath } from 'url'; - -import { ExecuteNodeArgsSchema } from '../tools/schemas.js'; -import { ServerResult } from '../types.js'; - -// Get the directory where the MCP is installed (for requiring packages like exceljs) -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const mcpRoot = path.resolve(__dirname, '..', '..'); - -/** - * Handle execute_node command - * Executes Node.js code using the same Node runtime as the MCP - */ -export async function handleExecuteNode(args: unknown): Promise { - const parsed = ExecuteNodeArgsSchema.parse(args); - const { code, timeout_ms } = parsed; - - // Create temp file IN THE MCP DIRECTORY so ES module imports resolve correctly - // (ES modules resolve packages relative to file location, not NODE_PATH or cwd) - const tempFile = path.join(mcpRoot, `.mcp-exec-${Date.now()}-${Math.random().toString(36).slice(2)}.mjs`); - - // User code runs directly - imports will resolve from mcpRoot/node_modules - const wrappedCode = code; - - try { - await fs.writeFile(tempFile, wrappedCode, 'utf8'); - - const result = await new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { - const proc = spawn(process.execPath, [tempFile], { - cwd: mcpRoot, - timeout: timeout_ms - }); - - let stdout = ''; - let stderr = ''; - - proc.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - proc.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - proc.on('close', (exitCode) => { - resolve({ stdout, stderr, exitCode: exitCode ?? 1 }); - }); - - proc.on('error', (err) => { - resolve({ stdout, stderr: stderr + '\n' + err.message, exitCode: 1 }); - }); - }); - - // Clean up temp file - await fs.unlink(tempFile).catch(() => {}); - - if (result.exitCode !== 0) { - return { - content: [{ - type: "text", - text: `Execution failed (exit code ${result.exitCode}):\n${result.stderr}\n${result.stdout}` - }], - isError: true - }; - } - - return { - content: [{ - type: "text", - text: result.stdout || '(no output)' - }] - }; - - } catch (error) { - // Clean up temp file on error - await fs.unlink(tempFile).catch(() => {}); - - return { - content: [{ - type: "text", - text: `Failed to execute Node.js code: ${error instanceof Error ? error.message : String(error)}` - }], - isError: true - }; - } -} diff --git a/src/server.ts b/src/server.ts index 9970cdbd..6cfc47a4 100644 --- a/src/server.ts +++ b/src/server.ts @@ -47,7 +47,6 @@ import { GetPromptsArgsSchema, GetRecentToolCallsArgsSchema, WritePdfArgsSchema, - ExecuteNodeArgsSchema, } from './tools/schemas.js'; import { getConfig, setConfigValue } from './tools/config.js'; import { getUsageStats } from './tools/usage.js'; @@ -668,7 +667,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { EXCEL FILES (.xlsx, .xls, .xlsm) - Range Update mode: Takes: - file_path: Path to the Excel file - - range: "SheetName!A1:C10" or "SheetName" for whole sheet + - range: ALWAYS use FROM:TO format - "SheetName!A1:C10" or "SheetName!C1:C1" - content: 2D array, e.g., [["H1","H2"],["R1","R2"]] TEXT FILES - Find/Replace mode: @@ -764,8 +763,15 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { - Which detection mechanism triggered early exit Use this to identify missed optimization opportunities and improve detection patterns. - ALWAYS USE FOR: Local file analysis, CSV processing, data exploration, system commands - NEVER USE ANALYSIS TOOL FOR: Local file access (analysis tool is browser-only and WILL FAIL) + NODE.JS FALLBACK (node:local): + When Python is unavailable or fails, use start_process("node:local") instead. + - Runs on MCP server where Node.js is guaranteed + - interact_with_process(pid, "complete self-contained script") + - STATELESS: Each call is fresh - include ALL imports/setup/processing in ONE call + - Use ES module imports: import ExcelJS from 'exceljs' + - ExcelJS available for Excel files (NOT xlsx library) + - All Node.js built-ins available (fs, path, http, crypto, etc.) + - Use console.log() for output ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, @@ -937,9 +943,9 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { name: "kill_process", description: ` Terminate a running process by PID. - + Use with caution as this will forcefully terminate the specified process. - + ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(KillProcessArgsSchema), annotations: { @@ -1060,35 +1066,6 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(GetPromptsArgsSchema), - }, - { - name: "execute_node", - description: ` - Execute Node.js code directly using the MCP server's Node runtime. - - PRIMARY TOOL FOR EXCEL FILES AND COMPLEX CALCULATIONS - Use this tool for ANY Excel file (.xlsx, .xls) operations and complex data calculations. - ExcelJS library is built-in and ready to use. - - Code runs as ES module (.mjs) with top-level await support. - Uses the same Node.js environment that runs the MCP server. - - Available libraries: ExcelJS (for Excel file manipulation), and all Node.js built-ins. - - Use cases: Excel file reading/writing/analysis, data transformations, bulk file operations, - complex calculations, JSON processing, or any task better suited to code than tools. - - Output: Use console.log() to return results. Stdout is captured and returned. - - ${PATH_GUIDANCE} - ${CMD_PREFIX_DESCRIPTION}`, - inputSchema: zodToJsonSchema(ExecuteNodeArgsSchema), - annotations: { - title: "Execute Node.js Code", - readOnlyHint: false, - destructiveHint: true, - openWorldHint: true, - }, } ]; @@ -1278,18 +1255,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request: CallToolRequest) result = await handlers.handleKillProcess(args); break; - case "execute_node": - try { - result = await handlers.handleExecuteNode(args); - } catch (error) { - capture('server_request_error', {message: `Error in execute_node handler: ${error}`}); - result = { - content: [{type: "text", text: `Error executing Node.js code: ${error instanceof Error ? error.message : String(error)}`}], - isError: true, - }; - } - break; - // Note: REPL functionality removed in favor of using general terminal commands // Filesystem tools diff --git a/src/tools/edit.ts b/src/tools/edit.ts index 590c1fb7..4411f160 100644 --- a/src/tools/edit.ts +++ b/src/tools/edit.ts @@ -371,10 +371,13 @@ export async function handleEditBlock(args: unknown): Promise { const parsed = EditBlockArgsSchema.parse(args); // Structured files: Range rewrite - if (parsed.range && parsed.content !== undefined) { + if (parsed.range !== undefined && parsed.content !== undefined) { try { + // Validate path before any filesystem operations + const validatedPath = await validatePath(parsed.file_path); + const { getFileHandler } = await import('../utils/files/factory.js'); - const handler = await getFileHandler(parsed.file_path); + const handler = await getFileHandler(validatedPath); // Parse content if it's a JSON string (AI often sends arrays as JSON strings) let content = parsed.content; @@ -388,7 +391,7 @@ export async function handleEditBlock(args: unknown): Promise { // Check if handler supports range editing if ('editRange' in handler && typeof handler.editRange === 'function') { - await handler.editRange(parsed.file_path, parsed.range, content, parsed.options); + await handler.editRange(validatedPath, parsed.range, content, parsed.options); return { content: [{ type: "text", @@ -417,9 +420,20 @@ export async function handleEditBlock(args: unknown): Promise { } // Text files: String replacement + // Validate required parameters for text replacement + if (parsed.old_string === undefined || parsed.new_string === undefined) { + return { + content: [{ + type: "text", + text: `Error: Text replacement requires both old_string and new_string parameters` + }], + isError: true + }; + } + const searchReplace = { - search: parsed.old_string!, - replace: parsed.new_string! + search: parsed.old_string, + replace: parsed.new_string }; return performSearchReplace(parsed.file_path, searchReplace, parsed.expected_replacements); diff --git a/src/tools/improved-process-tools.ts b/src/tools/improved-process-tools.ts index 3764c489..344a4fde 100644 --- a/src/tools/improved-process-tools.ts +++ b/src/tools/improved-process-tools.ts @@ -4,9 +4,91 @@ import { StartProcessArgsSchema, ReadProcessOutputArgsSchema, InteractWithProces import { capture } from "../utils/capture.js"; import { ServerResult } from '../types.js'; import { analyzeProcessState, cleanProcessOutput, formatProcessStateMessage, ProcessState } from '../utils/process-detection.js'; -import { getSystemInfo } from '../utils/system-info.js'; import * as os from 'os'; import { configManager } from '../config-manager.js'; +import { spawn } from 'child_process'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +// Get the directory where the MCP is installed (for ES module imports) +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const mcpRoot = path.resolve(__dirname, '..', '..'); + +// Track virtual Node sessions (PIDs that are actually Node fallback sessions) +const virtualNodeSessions = new Map(); +let virtualPidCounter = -1000; // Use negative PIDs for virtual sessions + +/** + * Execute Node.js code via temp file (fallback when Python unavailable) + * Creates temp .mjs file in MCP directory for ES module import access + */ +async function executeNodeCode(code: string, timeout_ms: number = 30000): Promise { + const tempFile = path.join(mcpRoot, `.mcp-exec-${Date.now()}-${Math.random().toString(36).slice(2)}.mjs`); + + try { + await fs.writeFile(tempFile, code, 'utf8'); + + const result = await new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { + const proc = spawn(process.execPath, [tempFile], { + cwd: mcpRoot, + timeout: timeout_ms + }); + + let stdout = ''; + let stderr = ''; + + proc.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + proc.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + proc.on('close', (exitCode) => { + resolve({ stdout, stderr, exitCode: exitCode ?? 1 }); + }); + + proc.on('error', (err) => { + resolve({ stdout, stderr: stderr + '\n' + err.message, exitCode: 1 }); + }); + }); + + // Clean up temp file + await fs.unlink(tempFile).catch(() => {}); + + if (result.exitCode !== 0) { + return { + content: [{ + type: "text", + text: `Execution failed (exit code ${result.exitCode}):\n${result.stderr}\n${result.stdout}` + }], + isError: true + }; + } + + return { + content: [{ + type: "text", + text: result.stdout || '(no output)' + }] + }; + + } catch (error) { + // Clean up temp file on error + await fs.unlink(tempFile).catch(() => {}); + + return { + content: [{ + type: "text", + text: `Failed to execute Node.js code: ${error instanceof Error ? error.message : String(error)}` + }], + isError: true + }; + } +} /** * Start a new process (renamed from execute_command) @@ -42,6 +124,31 @@ export async function startProcess(args: unknown): Promise { }; } + const commandToRun = parsed.data.command; + + // Handle node:local - runs Node.js code directly on MCP server + if (commandToRun.trim() === 'node:local') { + const virtualPid = virtualPidCounter--; + virtualNodeSessions.set(virtualPid, { timeout_ms: parsed.data.timeout_ms || 30000 }); + + return { + content: [{ + type: "text", + text: `Node.js session started with PID ${virtualPid} (MCP server execution) + + IMPORTANT: Each interact_with_process call runs as a FRESH script. + State is NOT preserved between calls. Include ALL code in ONE call: + - imports, file reading, processing, and output together. + + Available libraries: + - ExcelJS for Excel files: import ExcelJS from 'exceljs' + - All Node.js built-ins: fs, path, http, crypto, etc. + +🔄 Ready for code - send complete self-contained script via interact_with_process.` + }], + }; + } + let shellUsed: string | undefined = parsed.data.shell; if (!shellUsed) { @@ -61,7 +168,7 @@ export async function startProcess(args: unknown): Promise { } const result = await terminalManager.executeCommand( - parsed.data.command, + commandToRun, parsed.data.timeout_ms, shellUsed, parsed.data.verbose_timing || false @@ -419,6 +526,18 @@ export async function interactWithProcess(args: unknown): Promise verbose_timing = false } = parsed.data; + // Check if this is a virtual Node session (Python fallback) + if (virtualNodeSessions.has(pid)) { + const session = virtualNodeSessions.get(pid)!; + capture('server_interact_with_process_node_fallback', { + pid: pid, + inputLength: input.length + }); + + // Execute code via temp file approach + return executeNodeCode(input, session.timeout_ms); + } + // Timing telemetry const startTime = Date.now(); let firstOutputTime: number | undefined; diff --git a/src/tools/schemas.ts b/src/tools/schemas.ts index d76748b5..88b2ffef 100644 --- a/src/tools/schemas.ts +++ b/src/tools/schemas.ts @@ -203,8 +203,3 @@ export const GetRecentToolCallsArgsSchema = z.object({ since: z.string().datetime().optional(), }); -// Execute Node.js code schema -export const ExecuteNodeArgsSchema = z.object({ - code: z.string(), - timeout_ms: z.number().optional().default(30000), -}); \ No newline at end of file diff --git a/src/utils/files/factory.ts b/src/utils/files/factory.ts index 5fba5ff5..34a2e1f4 100644 --- a/src/utils/files/factory.ts +++ b/src/utils/files/factory.ts @@ -92,28 +92,20 @@ export async function getFileHandler(filePath: string): Promise { /** * Check if a file path is an Excel file + * Delegates to ExcelFileHandler.canHandle to avoid duplicating extension logic * @param path File path * @returns true if file is Excel format */ export function isExcelFile(path: string): boolean { - const ext = path.toLowerCase(); - return ext.endsWith('.xlsx') || ext.endsWith('.xls') || ext.endsWith('.xlsm'); + return getExcelHandler().canHandle(path); } /** * Check if a file path is an image file + * Delegates to ImageFileHandler.canHandle to avoid duplicating extension logic * @param path File path * @returns true if file is an image format */ export function isImageFile(path: string): boolean { - // This will be implemented by checking MIME type - // For now, use extension-based check - const ext = path.toLowerCase(); - return ext.endsWith('.png') || - ext.endsWith('.jpg') || - ext.endsWith('.jpeg') || - ext.endsWith('.gif') || - ext.endsWith('.webp') || - ext.endsWith('.bmp') || - ext.endsWith('.svg'); -} \ No newline at end of file + return getImageHandler().canHandle(path); +} diff --git a/src/utils/files/text.ts b/src/utils/files/text.ts index 3e00b11b..18d408f7 100644 --- a/src/utils/files/text.ts +++ b/src/utils/files/text.ts @@ -24,8 +24,9 @@ import { FileInfo } from './base.js'; -// Import constants from filesystem.ts -// These will be imported after we organize the code +// TODO: Centralize these constants with filesystem.ts to avoid silent drift +// These duplicate concepts from filesystem.ts and should be moved to a shared +// constants module (e.g., src/utils/files/constants.ts) during reorganization const FILE_SIZE_LIMITS = { LARGE_FILE_THRESHOLD: 10 * 1024 * 1024, // 10MB LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting diff --git a/src/utils/system-info.ts b/src/utils/system-info.ts index 862e2b0a..5567f29b 100644 --- a/src/utils/system-info.ts +++ b/src/utils/system-info.ts @@ -1,6 +1,7 @@ import os from 'os'; import fs from 'fs'; import path from 'path'; +import { execSync } from 'child_process'; export interface DockerMount { hostPath: string; @@ -43,6 +44,11 @@ export interface SystemInfo { path: string; npmVersion?: string; }; + pythonInfo?: { + available: boolean; + command: string; + version?: string; + }; processInfo: { pid: number; arch: string; @@ -399,13 +405,13 @@ function detectNodeInfo(): SystemInfo['nodeInfo'] { try { // Get Node.js version from current process const version = process.version.replace('v', ''); // Remove 'v' prefix - + // Get Node.js executable path from current process const path = process.execPath; - + // Get npm version from environment if available const npmVersion = process.env.npm_version; - + return { version, path, @@ -416,6 +422,39 @@ function detectNodeInfo(): SystemInfo['nodeInfo'] { } } +/** + * Detect Python installation and version and put on systeminfo.pythonInfo + */ +function detectPythonInfo(): SystemInfo['pythonInfo'] { + // Try python commands in order of preference + const pythonCommands = process.platform === 'win32' + ? ['python', 'python3', 'py'] // Windows: 'python' is common, 'py' launcher + : ['python3', 'python']; // Unix: prefer python3 + + for (const cmd of pythonCommands) { + try { + const version = execSync(`${cmd} --version`, { + encoding: 'utf8', + timeout: 5000, + stdio: ['pipe', 'pipe', 'pipe'] + }).trim(); + + // Verify it's Python 3.x + if (version.includes('Python 3')) { + return { + available: true, + command: cmd, + version: version.replace('Python ', '') + }; + } + } catch { + // Command not found or failed, try next + } + } + + return { available: false, command: '' }; +} + /** * Get comprehensive system information for tool prompts */ @@ -509,7 +548,10 @@ export function getSystemInfo(): SystemInfo { // Detect Node.js installation from current process const nodeInfo = detectNodeInfo(); - + + // Detect Python installation + const pythonInfo = detectPythonInfo(); + // Get process information const processInfo = { pid: process.pid, @@ -538,6 +580,7 @@ export function getSystemInfo(): SystemInfo { }, isDXT: !!process.env.MCP_DXT, nodeInfo, + pythonInfo, processInfo, examplePaths }; From eaac88e9627d55747224395f88eca17dc2722d95 Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Sat, 29 Nov 2025 19:30:15 +0200 Subject: [PATCH 15/17] Replace execute_node tool with node:local command and add Python detection --- src/server.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/server.ts b/src/server.ts index 6cfc47a4..99c17005 100644 --- a/src/server.ts +++ b/src/server.ts @@ -773,6 +773,9 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { - All Node.js built-ins available (fs, path, http, crypto, etc.) - Use console.log() for output + ALWAYS USE FOR: Local file analysis, CSV processing, data exploration, system commands + NEVER USE ANALYSIS TOOL FOR: Local file access (analysis tool is browser-only and WILL FAIL) + ${PATH_GUIDANCE} ${CMD_PREFIX_DESCRIPTION}`, inputSchema: zodToJsonSchema(StartProcessArgsSchema), From d7543095b39350f59bc95103f0de90bd43f7d5bd Mon Sep 17 00:00:00 2001 From: edgarsskore Date: Sat, 29 Nov 2025 19:39:01 +0200 Subject: [PATCH 16/17] Fix virtual session handling for node:local --- src/tools/improved-process-tools.ts | 49 +++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/src/tools/improved-process-tools.ts b/src/tools/improved-process-tools.ts index 344a4fde..ace21706 100644 --- a/src/tools/improved-process-tools.ts +++ b/src/tools/improved-process-tools.ts @@ -526,7 +526,7 @@ export async function interactWithProcess(args: unknown): Promise verbose_timing = false } = parsed.data; - // Check if this is a virtual Node session (Python fallback) + // Check if this is a virtual Node session (node:local) if (virtualNodeSessions.has(pid)) { const session = virtualNodeSessions.get(pid)!; capture('server_interact_with_process_node_fallback', { @@ -535,7 +535,9 @@ export async function interactWithProcess(args: unknown): Promise }); // Execute code via temp file approach - return executeNodeCode(input, session.timeout_ms); + // Respect per-call timeout if provided, otherwise use session default + const effectiveTimeout = timeout_ms ?? session.timeout_ms; + return executeNodeCode(input, effectiveTimeout); } // Timing telemetry @@ -758,13 +760,26 @@ export async function forceTerminate(args: unknown): Promise { }; } - const success = terminalManager.forceTerminate(parsed.data.pid); + const pid = parsed.data.pid; + + // Handle virtual Node.js sessions (node:local) + if (virtualNodeSessions.has(pid)) { + virtualNodeSessions.delete(pid); + return { + content: [{ + type: "text", + text: `Cleared virtual Node.js session ${pid}` + }], + }; + } + + const success = terminalManager.forceTerminate(pid); return { content: [{ type: "text", text: success - ? `Successfully initiated termination of session ${parsed.data.pid}` - : `No active session found for PID ${parsed.data.pid}` + ? `Successfully initiated termination of session ${pid}` + : `No active session found for PID ${pid}` }], }; } @@ -774,14 +789,30 @@ export async function forceTerminate(args: unknown): Promise { */ export async function listSessions(): Promise { const sessions = terminalManager.listActiveSessions(); + + // Include virtual Node.js sessions + const virtualSessions = Array.from(virtualNodeSessions.entries()).map(([pid, session]) => ({ + pid, + type: 'node:local', + timeout_ms: session.timeout_ms + })); + + const realSessionsText = sessions.map(s => + `PID: ${s.pid}, Blocked: ${s.isBlocked}, Runtime: ${Math.round(s.runtime / 1000)}s` + ); + + const virtualSessionsText = virtualSessions.map(s => + `PID: ${s.pid} (node:local), Timeout: ${s.timeout_ms}ms` + ); + + const allSessions = [...realSessionsText, ...virtualSessionsText]; + return { content: [{ type: "text", - text: sessions.length === 0 + text: allSessions.length === 0 ? 'No active sessions' - : sessions.map(s => - `PID: ${s.pid}, Blocked: ${s.isBlocked}, Runtime: ${Math.round(s.runtime / 1000)}s` - ).join('\n') + : allSessions.join('\n') }], }; } \ No newline at end of file From 91da3e2323eed9eec4323cf7de2ec7b6cb99dbb0 Mon Sep 17 00:00:00 2001 From: Eduard Ruzga Date: Fri, 5 Dec 2025 16:40:46 +0200 Subject: [PATCH 17/17] Test fixes --- package-lock.json | 515 +++++++++++++++++++++++++++++++++---- test/test-excel-files.js | 4 +- test/test-file-handlers.js | 8 +- 3 files changed, 478 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index 373b8358..a9b57e32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@opendocsg/pdf2md": "^0.2.2", "@vscode/ripgrep": "^1.15.9", "cross-fetch": "^4.1.0", + "exceljs": "^4.4.0", "fastest-levenshtein": "^1.0.16", "file-type": "^21.1.1", "glob": "^10.3.10", @@ -138,6 +139,47 @@ "tslib": "^2.4.0" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, "node_modules/@img/colour": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", @@ -1990,7 +2032,6 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", - "dev": true, "license": "MIT", "dependencies": { "archiver-utils": "^2.1.0", @@ -2009,7 +2050,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", - "dev": true, "license": "MIT", "dependencies": { "glob": "^7.1.4", @@ -2031,7 +2071,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -2043,7 +2082,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -2064,7 +2102,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -2077,7 +2114,6 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -2093,14 +2129,12 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, "license": "MIT" }, "node_modules/archiver-utils/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" @@ -2162,7 +2196,6 @@ "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "dev": true, "license": "MIT" }, "node_modules/b4a": { @@ -2290,7 +2323,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -2316,6 +2348,15 @@ "node": ">=10.0.0" } }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -2326,6 +2367,19 @@ "node": "*" } }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -2342,7 +2396,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, "license": "MIT", "dependencies": { "buffer": "^5.5.0", @@ -2350,6 +2403,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", @@ -2444,7 +2503,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -2506,6 +2564,23 @@ "dev": true, "license": "MIT" }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2642,6 +2717,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -3042,7 +3129,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", - "dev": true, "license": "MIT", "dependencies": { "buffer-crc32": "^0.2.13", @@ -3121,7 +3207,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true, "license": "MIT" }, "node_modules/cors": { @@ -3185,7 +3270,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -3198,7 +3282,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", - "dev": true, "license": "MIT", "dependencies": { "crc-32": "^1.2.0", @@ -3246,6 +3329,12 @@ "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==", "license": "MIT" }, + "node_modules/dayjs": { + "version": "1.11.19", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", + "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -3922,6 +4011,45 @@ "node": ">= 0.4" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/duplexer3": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz", @@ -4251,6 +4379,35 @@ "node": ">=18.0.0" } }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/exceljs/node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/express": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/express/-/express-5.0.1.tgz", @@ -4440,6 +4597,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4742,7 +4912,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true, "license": "MIT" }, "node_modules/fs-extra": { @@ -4790,7 +4959,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "devOptional": true, "license": "ISC" }, "node_modules/fsevents": { @@ -4807,6 +4975,90 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/fstream/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/fstream/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fstream/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/fstream/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/fstream/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -5130,7 +5382,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, "license": "ISC" }, "node_modules/gray-matter": { @@ -5353,6 +5604,12 @@ "dev": true, "license": "ISC" }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -5412,7 +5669,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "devOptional": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -5637,7 +5893,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true, "license": "MIT" }, "node_modules/isbinaryfile": { @@ -5802,6 +6057,48 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -5825,7 +6122,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "^2.0.5" @@ -5838,7 +6134,6 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -5854,25 +6149,38 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, "license": "MIT" }, "node_modules/lazystream/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "license": "MIT" }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "license": "ISC" + }, "node_modules/listr": { "version": "0.14.3", "resolved": "https://registry.npmjs.org/listr/-/listr-0.14.3.tgz", @@ -6073,35 +6381,79 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "dev": true, "license": "MIT" }, "node_modules/lodash.difference": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "license": "MIT" }, "node_modules/lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", "license": "MIT" }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", "license": "MIT" }, "node_modules/lodash.union": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", - "dev": true, + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", "license": "MIT" }, "node_modules/log-symbols": { @@ -7905,7 +8257,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "devOptional": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -8092,7 +8443,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true, "license": "MIT" }, "node_modules/progress": { @@ -8325,7 +8675,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "devOptional": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -8340,7 +8689,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", - "dev": true, "license": "Apache-2.0", "dependencies": { "minimatch": "^5.1.0" @@ -8350,7 +8698,6 @@ "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -8731,6 +9078,18 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/schema-utils": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", @@ -8941,6 +9300,12 @@ "license": "ISC", "optional": true }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -9399,7 +9764,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "devOptional": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -9654,7 +10018,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, "license": "MIT", "dependencies": { "bl": "^4.0.3", @@ -9897,6 +10260,15 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/trim-repeated": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz", @@ -10230,6 +10602,54 @@ "node": ">= 0.8" } }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", @@ -10297,7 +10717,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "devOptional": true, "license": "MIT" }, "node_modules/utils-merge": { @@ -10309,6 +10728,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -10837,6 +11265,12 @@ } } }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -10958,7 +11392,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", - "dev": true, "license": "MIT", "dependencies": { "archiver-utils": "^3.0.4", @@ -10973,7 +11406,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", - "dev": true, "license": "MIT", "dependencies": { "glob": "^7.2.3", @@ -10995,7 +11427,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -11007,7 +11438,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -11028,7 +11458,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" diff --git a/test/test-excel-files.js b/test/test-excel-files.js index 8913d552..bed007e8 100644 --- a/test/test-excel-files.js +++ b/test/test-excel-files.js @@ -81,12 +81,12 @@ async function teardown(originalConfig) { async function testFileHandlerFactory() { console.log('\n--- Test 1: File Handler Factory ---'); - const handler = getFileHandler('test.xlsx'); + const handler = await getFileHandler('test.xlsx'); assert.ok(handler, 'Handler should be returned for .xlsx file'); assert.ok(handler.constructor.name === 'ExcelFileHandler', `Expected ExcelFileHandler but got ${handler.constructor.name}`); - const txtHandler = getFileHandler('test.txt'); + const txtHandler = await getFileHandler('test.txt'); assert.ok(txtHandler.constructor.name === 'TextFileHandler', `Expected TextFileHandler for .txt but got ${txtHandler.constructor.name}`); diff --git a/test/test-file-handlers.js b/test/test-file-handlers.js index 1adcf7ec..240d72fc 100644 --- a/test/test-file-handlers.js +++ b/test/test-file-handlers.js @@ -107,7 +107,7 @@ async function testHandlerFactory() { ]; for (const { file, expected } of testCases) { - const handler = getFileHandler(file); + const handler = await getFileHandler(file); assert.strictEqual(handler.constructor.name, expected, `${file} should use ${expected} but got ${handler.constructor.name}`); } @@ -168,9 +168,9 @@ async function testReadOptionsInterface() { async function testCanHandle() { console.log('\n--- Test 4: canHandle Method ---'); - const excelHandler = getFileHandler('test.xlsx'); - const textHandler = getFileHandler('test.txt'); - const imageHandler = getFileHandler('test.png'); + const excelHandler = await getFileHandler('test.xlsx'); + const textHandler = await getFileHandler('test.txt'); + const imageHandler = await getFileHandler('test.png'); // Excel handler should handle xlsx assert.ok(excelHandler.canHandle('anything.xlsx'), 'Excel handler should handle .xlsx');