From c2e26f863ae675b34a5cd0ef58f61afb9b520d7c Mon Sep 17 00:00:00 2001 From: Ogenbertrand Date: Sun, 21 Sep 2025 15:27:01 +0100 Subject: [PATCH] feat: set up eslint and prettier --- .eslintrc.json | 32 +++ .prettierignore | 66 ++++++ .prettierrc | 22 ++ README.md | 123 +++++++--- docs/todo-phase1.md | 10 +- package-lock.json | 442 +++++++++++++++++++++++++++++++++++ package.json | 14 +- src/chroma-vector-store.ts | 45 ++-- src/cli-main.ts | 2 +- src/cli.ts | 205 +++++++++------- src/code-parser.ts | 278 +++++++++++++++------- src/code-storage.ts | 8 +- src/config.ts | 69 ++++-- src/embedding-service.ts | 33 ++- src/file-watcher.ts | 34 +-- src/index.ts | 89 ++++--- src/logger.ts | 55 +++-- src/semantic-search.ts | 34 ++- src/tree-sitter-types.d.ts | 2 +- src/types.ts | 8 +- src/vector-store.ts | 57 ++--- test/debug-watcher.ts | 43 ++-- test/test-embedding.ts | 19 +- test/test-file-watcher.ts | 112 ++++----- test/test-integration.ts | 354 ++++++++++++++-------------- test/test-parser.ts | 46 ++-- test/test-semantic-search.ts | 62 +++-- test/test-storage.ts | 13 +- test/test-vector-store.ts | 47 ++-- tsconfig.json | 14 +- 30 files changed, 1601 insertions(+), 737 deletions(-) create mode 100644 .eslintrc.json create mode 100644 .prettierignore create mode 100644 .prettierrc diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..c9afbcb --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,32 @@ +{ + "env": { + "node": true, + "es2022": true + }, + "extends": ["eslint:recommended"], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 2022, + "sourceType": "module" + }, + "plugins": ["@typescript-eslint", "prettier"], + "rules": { + "prettier/prettier": "warn", + "no-console": "off", + "no-debugger": "error", + "prefer-const": "error", + "no-var": "error", + "object-shorthand": "warn", + "prefer-arrow-callback": "warn", + "prefer-template": "warn", + "template-curly-spacing": "error", + "arrow-spacing": "error", + "comma-dangle": ["warn", "always-multiline"], + "quotes": ["error", "single", { "avoidEscape": true }], + "semi": ["error", "always"], + "no-unused-vars": "warn", + "no-empty": "warn", + "no-case-declarations": "warn" + }, + "ignorePatterns": ["dist/", "node_modules/", "*.js", "coverage/", ".nyc_output/"] +} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..f2b8616 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,66 @@ +# Dependencies +node_modules/ +package-lock.json + +# Build outputs +dist/ +build/ +*.js +*.js.map + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory used by tools like istanbul +coverage/ +.nyc_output/ + +# Dependency directories +jspm_packages/ + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# IDE files +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Test corpus (large files) +test-corpus/ \ No newline at end of file diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..f9a2a7c --- /dev/null +++ b/.prettierrc @@ -0,0 +1,22 @@ +{ + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "printWidth": 100, + "tabWidth": 2, + "useTabs": false, + "bracketSpacing": true, + "arrowParens": "avoid", + "endOfLine": "lf", + "quoteProps": "as-needed", + "jsxSingleQuote": true, + "bracketSameLine": false, + "overrides": [ + { + "files": "*.json", + "options": { + "printWidth": 200 + } + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index dcbb762..24e525c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ -

MCP Local Context Engine

Semantic Code Search and Analysis Platform

-
GitHub Node.js @@ -25,6 +23,7 @@ MCP Local Context Engine is a comprehensive semantic code search and analysis platform that leverages the Model Context Protocol (MCP) for intelligent code understanding. Built with TypeScript and powered by vector databases, it provides advanced code search capabilities across multiple programming languages using AI-powered embeddings. ### Key Features + - **Multi-Language Support**: Comprehensive parsing for JavaScript, TypeScript, Python, Java, C/C++, Rust, Go, Ruby, and PHP - **Semantic Code Search**: Natural language queries with vector similarity matching - **Real-time Indexing**: File system watching with automatic re-indexing @@ -33,29 +32,32 @@ MCP Local Context Engine is a comprehensive semantic code search and analysis pl - **Comprehensive CLI**: Full-featured command-line interface with shell completions ### Architecture Overview + The engine combines multiple advanced technologies to provide a unified platform for code analysis and search. It uses Tree-sitter for multi-language parsing, Transformers.js for AI embeddings, and ChromaDB for efficient vector storage and retrieval. ## 2. Technical Specifications
-| | | -|:---:|:---:| -| **Core Language** | TypeScript | -| **Runtime** | Node.js 18+ | -| **Vector Database** | ChromaDB | -| **Embedding Model** | Xenova/all-MiniLM-L6-v2 | -| **Embedding Dimension** | 384 | -| **Parsing Engine** | Tree-sitter | -| **Supported Languages** | 10+ | -| **CLI Framework** | Commander.js | -| **Configuration** | JSON-based | -| **Logging** | Winston | +| | | +| :---------------------: | :---------------------: | +| **Core Language** | TypeScript | +| **Runtime** | Node.js 18+ | +| **Vector Database** | ChromaDB | +| **Embedding Model** | Xenova/all-MiniLM-L6-v2 | +| **Embedding Dimension** | 384 | +| **Parsing Engine** | Tree-sitter | +| **Supported Languages** | 10+ | +| **CLI Framework** | Commander.js | +| **Configuration** | JSON-based | +| **Logging** | Winston | +
## 3. Performance Metrics ### Code Analysis Performance +
@@ -96,6 +98,7 @@ The engine combines multiple advanced technologies to provide a unified platform ### Language Support Matrix +
@@ -178,11 +181,13 @@ The engine combines multiple advanced technologies to provide a unified platform ## 4. Deployment ### Prerequisites + - Node.js 18 or higher - Docker and Docker Compose - Git ### Installation + ```bash # Clone repository git clone https://github.com/AssahBismarkabah/42context.git @@ -199,9 +204,11 @@ npm run build ``` ### Configuration + The engine provides a **comprehensive three-tier configuration system** with automatic validation and flexible customization options. #### Configuration Hierarchy (Priority Order) + 1. **Environment Variables** (highest priority) - `DEV_CONTEXT_*` prefix 2. **JSON Configuration File** - Loaded via `--config` flag 3. **Default Values** (lowest priority) - Optimized for immediate use @@ -209,12 +216,14 @@ The engine provides a **comprehensive three-tier configuration system** with aut #### Configuration Methods **1. JSON Configuration File** + ```bash # Create custom configuration file node dist/src/cli-main.js --config my-config.json search "authentication" ``` **Example custom configuration** (`test-config.json`): + ```json { "projectPath": "/path/to/project", @@ -252,6 +261,7 @@ node dist/src/cli-main.js --config my-config.json search "authentication" ``` **2. Environment Variables** + ```bash # Set embedding batch size export DEV_CONTEXT_EMBEDDING_BATCH_SIZE=128 @@ -267,6 +277,7 @@ node dist/src/cli-main.js search "authentication" ``` **3. CLI Configuration Commands** + ```bash # View all settings node dist/src/cli-main.js config list @@ -283,19 +294,16 @@ node dist/src/cli-main.js config get embedding.batchSize #### Key Configuration Sections -| Section | Purpose | Key Settings | -|---------|---------|--------------| -| **vectorStore** | ChromaDB connection and vector storage | host, port, collectionName, embeddingDimension | -| **embedding** | AI model settings for code embeddings | modelName, batchSize, maxRetries, retryDelay | -| **parser** | Code analysis and chunking | maxFileSize, chunkSize, chunkOverlap, supportedLanguages | -| **fileWatcher** | Real-time file monitoring | ignored patterns, maxFileSize, polling settings | -| **semanticSearch** | Search behavior tuning | maxResults, minSimilarity, enableCaching | -| **performance** | Resource usage limits | maxConcurrentOperations, memoryLimit, cpuLimit | -| **security** | File access and content filtering | allowedFileExtensions, enableSandbox, maxFileSize | -| **logging** | Debug and monitoring output | level, enableConsole, enableFile | - - - +| Section | Purpose | Key Settings | +| ------------------ | -------------------------------------- | -------------------------------------------------------- | +| **vectorStore** | ChromaDB connection and vector storage | host, port, collectionName, embeddingDimension | +| **embedding** | AI model settings for code embeddings | modelName, batchSize, maxRetries, retryDelay | +| **parser** | Code analysis and chunking | maxFileSize, chunkSize, chunkOverlap, supportedLanguages | +| **fileWatcher** | Real-time file monitoring | ignored patterns, maxFileSize, polling settings | +| **semanticSearch** | Search behavior tuning | maxResults, minSimilarity, enableCaching | +| **performance** | Resource usage limits | maxConcurrentOperations, memoryLimit, cpuLimit | +| **security** | File access and content filtering | allowedFileExtensions, enableSandbox, maxFileSize | +| **logging** | Debug and monitoring output | level, enableConsole, enableFile | ## 5. Usage @@ -356,12 +364,67 @@ node dist/src/cli-main.js stats --detailed node dist/src/cli-main.js debug test-connection ``` +## 6. Development & CI/CD + +### Continuous Integration + +The project uses GitHub Actions for comprehensive CI/CD automation: + +#### ๐Ÿ”„ CI Pipeline + +- **Build & Test**: Automated testing with ChromaDB service +- **Code Quality**: ESLint, Prettier, TypeScript checks +- **Security**: npm audit, vulnerability scanning +- **Docker**: Container building and testing +- **Integration**: CLI functionality testing + +#### ๐Ÿ” Code Quality Pipeline + +- **Static Analysis**: CodeQL analysis +- **Security Scanning**: Dependency review, secret detection +- **Performance**: Build and CLI performance testing +- **License Compliance**: Dependency license checking + +#### ๐Ÿš€ Release Pipeline + +- **Automated Releases**: Tag-based and manual releases +- **Multi-platform**: Cross-platform binary packages +- **Container Registry**: Docker images to GitHub Container Registry +- **NPM Publishing**: Automated package publishing + +### Quick Setup + +```bash +# Install dependencies +npm install + +# Run quality checks +npm run lint +npm run format:check +npm run type-check + +# Run tests +npm test + +# Build project +npm run build + +# Start development +npm run dev +``` + +### CI/CD Documentation + +For detailed CI/CD setup and usage instructions, see: -## 6. License +- [CI/CD Setup Guide](CI_CD_SETUP.md) - Quick start and configuration +- [CI/CD Documentation](CI_CD_DOCUMENTATION.md) - Technical details + +## 7. License This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details. -## 7. Citation +## 8. Citation If you use MCP Local Context Engine in your research, please cite: @@ -373,4 +436,4 @@ If you use MCP Local Context Engine in your research, please cite: url={https://github.com/your-org/mcp-local-context-engine}, note={Open source semantic code search engine} } -``` \ No newline at end of file +``` diff --git a/docs/todo-phase1.md b/docs/todo-phase1.md index fee7efe..1bd3cc1 100644 --- a/docs/todo-phase1.md +++ b/docs/todo-phase1.md @@ -33,11 +33,11 @@ - [ ] Build request routing and handling compliance with MCP spec ### 4. PocketFlow Integration -- [ ] Integrate PocketFlow orchestration framework for multi-agent workflows -- [ ] Define task delegation mechanism to agents for code analysis, search, and completions -- [ ] Implement basic PocketFlow sample workflows (e.g., code search workflow) -- [ ] Ensure robust error handling and retry in task executions -- [ ] Set max agents and workflow timeout according to config +- [x] Integrate PocketFlow orchestration framework for multi-agent workflows +- [x] Define task delegation mechanism to agents for code analysis, search, and completions +- [x] Implement basic PocketFlow sample workflows (e.g., code search workflow) +- [x] Ensure robust error handling and retry in task executions +- [x] Set max agents and workflow timeout according to config ### 5. CLI Interface and Developer Tools - [x] Build MVP CLI tool for administrative tasks and debug commands diff --git a/package-lock.json b/package-lock.json index 0c32382..2419df6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,12 @@ "devDependencies": { "@types/chokidar": "^2.1.3", "@types/node": "^20.11.0", + "@typescript-eslint/eslint-plugin": "^8.44.0", + "@typescript-eslint/parser": "^8.44.0", "eslint": "^8.57.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-prettier": "^5.5.4", + "prettier": "^3.6.2", "typescript": "^5.3.3" }, "engines": { @@ -185,6 +190,19 @@ "node": ">= 8" } }, + "node_modules/@pkgr/core": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", + "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/pkgr" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -275,6 +293,277 @@ "undici-types": "~6.21.0" } }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.44.0.tgz", + "integrity": "sha512-EGDAOGX+uwwekcS0iyxVDmRV9HX6FLSM5kzrAToLTsr9OWCIKG/y3lQheCq18yZ5Xh78rRKJiEpP0ZaCs4ryOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "8.44.0", + "@typescript-eslint/type-utils": "8.44.0", + "@typescript-eslint/utils": "8.44.0", + "@typescript-eslint/visitor-keys": "8.44.0", + "graphemer": "^1.4.0", + "ignore": "^7.0.0", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.44.0", + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.44.0.tgz", + "integrity": "sha512-VGMpFQGUQWYT9LfnPcX8ouFojyrZ/2w3K5BucvxL/spdNehccKhB4jUyB1yBCXpr2XFm0jkECxgrpXBW2ipoAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.44.0", + "@typescript-eslint/types": "8.44.0", + "@typescript-eslint/typescript-estree": "8.44.0", + "@typescript-eslint/visitor-keys": "8.44.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.44.0.tgz", + "integrity": "sha512-ZeaGNraRsq10GuEohKTo4295Z/SuGcSq2LzfGlqiuEvfArzo/VRrT0ZaJsVPuKZ55lVbNk8U6FcL+ZMH8CoyVA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.44.0", + "@typescript-eslint/types": "^8.44.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.44.0.tgz", + "integrity": "sha512-87Jv3E+al8wpD+rIdVJm/ItDBe/Im09zXIjFoipOjr5gHUhJmTzfFLuTJ/nPTMc2Srsroy4IBXwcTCHyRR7KzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.44.0", + "@typescript-eslint/visitor-keys": "8.44.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.44.0.tgz", + "integrity": "sha512-x5Y0+AuEPqAInc6yd0n5DAcvtoQ/vyaGwuX5HE9n6qAefk1GaedqrLQF8kQGylLUb9pnZyLf+iEiL9fr8APDtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.44.0.tgz", + "integrity": "sha512-9cwsoSxJ8Sak67Be/hD2RNt/fsqmWnNE1iHohG8lxqLSNY8xNfyY7wloo5zpW3Nu9hxVgURevqfcH6vvKCt6yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.44.0", + "@typescript-eslint/typescript-estree": "8.44.0", + "@typescript-eslint/utils": "8.44.0", + "debug": "^4.3.4", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.44.0.tgz", + "integrity": "sha512-ZSl2efn44VsYM0MfDQe68RKzBz75NPgLQXuGypmym6QVOWL5kegTZuZ02xRAT9T+onqvM6T8CdQk0OwYMB6ZvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.44.0.tgz", + "integrity": "sha512-lqNj6SgnGcQZwL4/SBJ3xdPEfcBuhCG8zdcwCPgYcmiPLgokiNDKlbPzCwEwu7m279J/lBYWtDYL+87OEfn8Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.44.0", + "@typescript-eslint/tsconfig-utils": "8.44.0", + "@typescript-eslint/types": "8.44.0", + "@typescript-eslint/visitor-keys": "8.44.0", + "debug": "^4.3.4", + "fast-glob": "^3.3.2", + "is-glob": "^4.0.3", + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.44.0.tgz", + "integrity": "sha512-nktOlVcg3ALo0mYlV+L7sWUD58KG4CMj1rb2HUVOO4aL3K/6wcD+NERqd0rrA5Vg06b42YhF6cFxeixsp9Riqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.7.0", + "@typescript-eslint/scope-manager": "8.44.0", + "@typescript-eslint/types": "8.44.0", + "@typescript-eslint/typescript-estree": "8.44.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.44.0.tgz", + "integrity": "sha512-zaz9u8EJ4GBmnehlrpoKvj/E3dNbuQ7q0ucyZImm3cLqJ8INTc970B1qEqDX/Rzq65r3TvVTN7kHWPBoyW7DWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.44.0", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -911,6 +1200,53 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint-config-prettier": { + "version": "10.1.8", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", + "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", + "dev": true, + "license": "MIT", + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "funding": { + "url": "https://opencollective.com/eslint-config-prettier" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-plugin-prettier": { + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.5.4.tgz", + "integrity": "sha512-swNtI95SToIz05YINMA6Ox5R057IMAmWZ26GqPxusAp1TZzj+IdY9tXNWWD3vkF/wEqydCONcwjTFpxybBqZsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.11.7" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": ">= 7.0.0 <10.0.0 || >=10.1.0", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", @@ -1034,12 +1370,36 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/fast-fifo": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", "license": "MIT" }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -1492,6 +1852,30 @@ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", "license": "Apache-2.0" }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, "node_modules/mimic-response": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", @@ -1833,6 +2217,35 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/protobufjs": { "version": "6.11.4", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", @@ -2218,6 +2631,22 @@ "node": ">=8" } }, + "node_modules/synckit": { + "version": "0.11.11", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.11.tgz", + "integrity": "sha512-MeQTA1r0litLUf0Rp/iisCaL8761lKAZHaimlbGK4j0HysC4PLfqygQj9srcs0m2RdtDYnF8UuYyKpbjHYp7Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.2.9" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/synckit" + } + }, "node_modules/tar-fs": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", @@ -2359,6 +2788,19 @@ "tree-sitter": "^0.20.6" } }, + "node_modules/ts-api-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", + "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index 6f8af1e..fecb95a 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,14 @@ "dev": "tsc && node --watch dist/index.js", "test": "tsc && node --test dist/test/*.js", "test:watch": "tsc && node --test --watch", - "clean": "rm -rf dist" + "clean": "rm -rf dist", + "lint": "eslint src/**/*.ts test/**/*.ts --ext .ts", + "lint:fix": "eslint src/**/*.ts test/**/*.ts --ext .ts --fix", + "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"*.md\" \"*.json\"", + "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"*.md\" \"*.json\"", + "type-check": "tsc --noEmit", + "lint-and-format": "npm run lint:fix && npm run format", + "pre-commit": "npm run type-check && npm run lint-and-format" }, "keywords": [ "mcp", @@ -43,7 +50,12 @@ "devDependencies": { "@types/chokidar": "^2.1.3", "@types/node": "^20.11.0", + "@typescript-eslint/eslint-plugin": "^8.44.0", + "@typescript-eslint/parser": "^8.44.0", "eslint": "^8.57.0", + "eslint-config-prettier": "^10.1.8", + "eslint-plugin-prettier": "^5.5.4", + "prettier": "^3.6.2", "typescript": "^5.3.3" }, "engines": { diff --git a/src/chroma-vector-store.ts b/src/chroma-vector-store.ts index c473fda..78f0bec 100644 --- a/src/chroma-vector-store.ts +++ b/src/chroma-vector-store.ts @@ -9,19 +9,24 @@ export class ChromaVectorStore implements VectorStore { private port: number; private authToken: string; - constructor(collectionName: string = 'code_vectors', host: string = 'localhost', port: number = 8000, authToken: string = 'test-token') { + constructor( + collectionName: string = 'code_vectors', + host: string = 'localhost', + port: number = 8000, + authToken: string = 'test-token' + ) { this.collectionName = collectionName; this.host = host; this.port = port; this.authToken = authToken; - + // Initialize ChromaDB client with proper authentication this.client = new ChromaClient({ path: `http://${this.host}:${this.port}`, auth: { provider: 'token', - credentials: this.authToken - } + credentials: this.authToken, + }, }); } @@ -30,7 +35,7 @@ export class ChromaVectorStore implements VectorStore { // Create or get collection this.collection = await this.client.getOrCreateCollection({ name: this.collectionName, - metadata: { 'hnsw:space': 'cosine' } // Use cosine similarity for code vectors + metadata: { 'hnsw:space': 'cosine' }, // Use cosine similarity for code vectors }); console.log(`ChromaDB collection '${this.collectionName}' initialized successfully`); } catch (error) { @@ -57,7 +62,7 @@ export class ChromaVectorStore implements VectorStore { type: v.type, lineStart: v.lineStart, lineEnd: v.lineEnd, - timestamp: v.timestamp + timestamp: v.timestamp, })); // Add vectors to collection @@ -65,7 +70,7 @@ export class ChromaVectorStore implements VectorStore { ids, embeddings, documents, - metadatas + metadatas, }); console.log(`Added ${vectors.length} vectors to ChromaDB collection`); @@ -75,7 +80,11 @@ export class ChromaVectorStore implements VectorStore { } } - async searchSimilar(queryVector: number[], topK: number = 5, language?: string): Promise { + async searchSimilar( + queryVector: number[], + topK: number = 5, + language?: string + ): Promise { if (!this.collection) { throw new Error('Collection not initialized. Call initialize() first.'); } @@ -84,12 +93,12 @@ export class ChromaVectorStore implements VectorStore { // Build query parameters const queryParams: any = { queryEmbeddings: [queryVector], - nResults: topK + nResults: topK, }; // Add language filter if specified if (language) { - queryParams.where = { language: language }; + queryParams.where = { language }; } // Perform similarity search @@ -97,14 +106,14 @@ export class ChromaVectorStore implements VectorStore { // Transform results to our format const searchResults: VectorSearchResult[] = []; - + if (results.ids && results.ids[0]) { for (let i = 0; i < results.ids[0].length; i++) { const id = results.ids[0][i]; const document = results.documents?.[0]?.[i] || ''; const metadata = results.metadatas?.[0]?.[i] as any; const distance = results.distances?.[0]?.[i] || 0; - + searchResults.push({ id, content: document, @@ -114,7 +123,7 @@ export class ChromaVectorStore implements VectorStore { lineStart: metadata?.lineStart || 0, lineEnd: metadata?.lineEnd || 0, similarity: 1 - distance, // Convert distance to similarity score - timestamp: metadata?.timestamp || Date.now() + timestamp: metadata?.timestamp || Date.now(), }); } } @@ -134,9 +143,9 @@ export class ChromaVectorStore implements VectorStore { try { // Delete all vectors for a specific file path await this.collection.delete({ - where: { filePath: filePath } + where: { filePath }, }); - + console.log(`Deleted vectors for file: ${filePath}`); } catch (error) { console.error('Failed to delete vectors from ChromaDB:', error); @@ -162,7 +171,7 @@ export class ChromaVectorStore implements VectorStore { type: v.type, lineStart: v.lineStart, lineEnd: v.lineEnd, - timestamp: v.timestamp + timestamp: v.timestamp, })); // Update vectors in collection @@ -170,7 +179,7 @@ export class ChromaVectorStore implements VectorStore { ids, embeddings, documents, - metadatas + metadatas, }); console.log(`Updated ${vectors.length} vectors in ChromaDB collection`); @@ -214,4 +223,4 @@ export class ChromaVectorStore implements VectorStore { this.collection = null; console.log('ChromaDB connection closed'); } -} \ No newline at end of file +} diff --git a/src/cli-main.ts b/src/cli-main.ts index d51ebb1..907fd33 100644 --- a/src/cli-main.ts +++ b/src/cli-main.ts @@ -12,4 +12,4 @@ program.parse(process.argv); // If no command is provided, show help if (!process.argv.slice(2).length) { program.outputHelp(); -} \ No newline at end of file +} diff --git a/src/cli.ts b/src/cli.ts index bc1ea53..e297867 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,3 @@ - import { Command } from 'commander'; import { ContextEngine } from './index.js'; import { ConfigManager } from './config.js'; @@ -37,7 +36,7 @@ export class CLIInterface { .version('0.1.0') .option('-d, --debug', 'Enable debug mode') .option('-c, --config ', 'Configuration file path') - .hook('preAction', (thisCommand) => { + .hook('preAction', thisCommand => { const options = thisCommand.opts(); if (options.debug) { logger.setLevel(LogLevel.DEBUG); @@ -50,7 +49,7 @@ export class CLIInterface { .command('start') .description('Start the context engine with file watching') .option('-p, --project-path ', 'Project path to watch', process.cwd()) - .action(async (options) => { + .action(async options => { try { await this.startEngine(options.projectPath, program.opts().debug); } catch (error) { @@ -132,7 +131,7 @@ export class CLIInterface { .command('stats') .description('Show system statistics') .option('--format ', 'Output format (json, table)', 'table') - .action(async (options) => { + .action(async options => { try { await this.statsCommand(options); } catch (error) { @@ -148,7 +147,7 @@ export class CLIInterface { .option('--vectors', 'Clear vector store') .option('--cache', 'Clear embedding cache') .option('--all', 'Clear everything') - .action(async (options) => { + .action(async options => { try { await this.clearCommand(options); } catch (error) { @@ -177,7 +176,7 @@ export class CLIInterface { .command('completion') .description('Generate shell completion script') .argument('', 'Shell type: bash, zsh, fish') - .action((shell) => { + .action(shell => { this.generateCompletion(shell); }); } @@ -186,13 +185,13 @@ export class CLIInterface { * Start the context engine */ private async startEngine(projectPath: string, debug: boolean): Promise { - console.log(`Starting MCP Local Context Engine...`); + console.log('Starting MCP Local Context Engine...'); console.log(`Project path: ${path.resolve(projectPath)}`); - + this.engine = new ContextEngine({ projectPath: path.resolve(projectPath), - debug: debug, - configPath: program.opts().config + debug, + configPath: program.opts().config, }); await this.engine.start(); @@ -203,7 +202,7 @@ export class CLIInterface { */ private async searchCommand(query: string, options: any): Promise { console.log(`Searching for: "${query}"`); - + const config = this.configManager.getConfig(); const semanticSearch = new SemanticSearch(config.semanticSearch); await semanticSearch.initialize(); @@ -213,13 +212,13 @@ export class CLIInterface { language: options.language, chunkType: options.type, filePath: options.file, - minSimilarity: parseFloat(options.minSimilarity) + minSimilarity: parseFloat(options.minSimilarity), }; const results = await semanticSearch.search(query, searchOptions); - + this.displaySearchResults(results, options.format); - + await semanticSearch.close(); } @@ -228,7 +227,7 @@ export class CLIInterface { */ private async analyzeCommand(filePath: string | undefined, options: any): Promise { const parser = new CodeParser(); - + if (filePath) { // Analyze specific file if (!existsSync(filePath)) { @@ -238,7 +237,7 @@ export class CLIInterface { const content = readFileSync(filePath, 'utf-8'); const language = parser.detectLanguage(filePath); - + if (!language) { console.error(`Unsupported file type: ${filePath}`); return; @@ -246,9 +245,9 @@ export class CLIInterface { console.log(`Analyzing file: ${filePath}`); console.log(`Language: ${language}`); - + const chunks = await parser.parseFile(filePath, content); - + this.displayAnalysisResults(chunks, options.format); } else { // Analyze project structure @@ -262,21 +261,21 @@ export class CLIInterface { */ private async indexCommand(indexPath: string, options: any): Promise { console.log(`Indexing: ${indexPath}`); - + const config = this.configManager.getConfig(); const semanticSearch = new SemanticSearch(config.semanticSearch); await semanticSearch.initialize(); try { const resolvedPath = path.resolve(indexPath); - + if (!existsSync(resolvedPath)) { console.error(`Path not found: ${resolvedPath}`); return; } const stats = statSync(resolvedPath); - + if (stats.isFile()) { // Index single file await this.indexFile(semanticSearch, resolvedPath); @@ -302,7 +301,7 @@ export class CLIInterface { */ private async indexFile(semanticSearch: SemanticSearch, filePath: string): Promise { console.log(`Indexing file: ${filePath}`); - + // Check if file is supported if (!this.isSupportedFile(filePath)) { console.log(`Skipping unsupported file: ${filePath}`); @@ -311,7 +310,7 @@ export class CLIInterface { try { const content = readFileSync(filePath, 'utf-8'); - + // Check file size limit const config = this.configManager.getConfig(); if (content.length > config.security.maxFileSize) { @@ -330,14 +329,18 @@ export class CLIInterface { /** * Index a directory recursively */ - private async indexDirectory(semanticSearch: SemanticSearch, dirPath: string, recursive: boolean): Promise { + private async indexDirectory( + semanticSearch: SemanticSearch, + dirPath: string, + recursive: boolean + ): Promise { console.log(`Indexing directory: ${dirPath} (recursive: ${recursive})`); - + const files = this.getFilesInDirectory(dirPath, recursive); const supportedFiles = files.filter(file => this.isSupportedFile(file)); - + console.log(`Found ${files.length} total files, ${supportedFiles.length} supported files`); - + if (supportedFiles.length === 0) { console.log('No supported files found to index'); return; @@ -346,34 +349,36 @@ export class CLIInterface { // Index files in batches to avoid memory issues const batchSize = 50; let processed = 0; - + for (let i = 0; i < supportedFiles.length; i += batchSize) { const batch = supportedFiles.slice(i, i + batchSize); const batchNumber = Math.floor(i / batchSize) + 1; const totalBatches = Math.ceil(supportedFiles.length / batchSize); - + console.log(`Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`); - - await Promise.all(batch.map(async (filePath) => { - try { - const content = readFileSync(filePath, 'utf-8'); - - // Check file size limit - const config = this.configManager.getConfig(); - if (content.length > config.security.maxFileSize) { - console.warn(`File too large, skipping: ${filePath}`); - return; - } - await semanticSearch.indexFile(filePath, content); - processed++; - console.log(`โœ“ ${processed}/${supportedFiles.length}: ${filePath}`); - } catch (error) { - console.error(`โœ— Failed to index file ${filePath}:`, error); - } - })); + await Promise.all( + batch.map(async filePath => { + try { + const content = readFileSync(filePath, 'utf-8'); + + // Check file size limit + const config = this.configManager.getConfig(); + if (content.length > config.security.maxFileSize) { + console.warn(`File too large, skipping: ${filePath}`); + return; + } + + await semanticSearch.indexFile(filePath, content); + processed++; + console.log(`โœ“ ${processed}/${supportedFiles.length}: ${filePath}`); + } catch (error) { + console.error(`โœ— Failed to index file ${filePath}:`, error); + } + }) + ); } - + console.log(`Indexing completed: ${processed}/${supportedFiles.length} files processed`); } @@ -382,14 +387,14 @@ export class CLIInterface { */ private getFilesInDirectory(dirPath: string, recursive: boolean): string[] { const files: string[] = []; - + try { const entries = readdirSync(dirPath); - + for (const entry of entries) { const fullPath = path.join(dirPath, entry); const stats = statSync(fullPath); - + if (stats.isFile()) { files.push(fullPath); } else if (stats.isDirectory() && recursive) { @@ -401,7 +406,7 @@ export class CLIInterface { } catch (error) { console.error(`Error reading directory ${dirPath}:`, error); } - + return files; } @@ -410,10 +415,25 @@ export class CLIInterface { */ private isSupportedFile(filePath: string): boolean { const supportedExtensions = [ - '.js', '.jsx', '.ts', '.tsx', '.py', '.java', '.cpp', '.c', '.h', - '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala' + '.js', + '.jsx', + '.ts', + '.tsx', + '.py', + '.java', + '.cpp', + '.c', + '.h', + '.cs', + '.php', + '.rb', + '.go', + '.rs', + '.swift', + '.kt', + '.scala', ]; - + const ext = path.extname(filePath).toLowerCase(); return supportedExtensions.includes(ext); } @@ -462,17 +482,23 @@ export class CLIInterface { } else if (format === 'table') { console.log(`\nSearch Results (${results.resultCount} found):`); console.log('='.repeat(60)); - + results.results.forEach((result, index) => { - console.log(`${index + 1}. ${result.type} | ${result.filePath}:${result.lineStart}-${result.lineEnd}`); + console.log( + `${index + 1}. ${result.type} | ${result.filePath}:${result.lineStart}-${result.lineEnd}` + ); console.log(` Similarity: ${result.similarity.toFixed(4)}`); - console.log(` Content: ${result.content.substring(0, 100)}${result.content.length > 100 ? '...' : ''}`); + console.log( + ` Content: ${result.content.substring(0, 100)}${result.content.length > 100 ? '...' : ''}` + ); console.log(''); }); } else { // plain format - results.results.forEach((result) => { - console.log(`${result.filePath}:${result.lineStart}-${result.lineEnd} | ${result.type} | ${result.content}`); + results.results.forEach(result => { + console.log( + `${result.filePath}:${result.lineStart}-${result.lineEnd} | ${result.type} | ${result.content}` + ); }); } } @@ -484,9 +510,9 @@ export class CLIInterface { if (format === 'json') { console.log(JSON.stringify(chunks, null, 2)); } else if (format === 'tree') { - console.log(`\nCode Structure Analysis:`); + console.log('\nCode Structure Analysis:'); console.log('='.repeat(40)); - + chunks.forEach((chunk, index) => { console.log(`${index + 1}. ${chunk.type}: ${chunk.name}`); console.log(` Lines: ${chunk.startLine}-${chunk.endLine}`); @@ -503,7 +529,7 @@ export class CLIInterface { }); } else { // plain format - chunks.forEach((chunk) => { + chunks.forEach(chunk => { console.log(`${chunk.type}: ${chunk.name} (${chunk.startLine}-${chunk.endLine})`); }); } @@ -515,7 +541,7 @@ export class CLIInterface { private async statsCommand(options: any): Promise { console.log('System Statistics:'); console.log('=================='); - + try { const config = this.configManager.getConfig(); const vectorStore = new ChromaVectorStore( @@ -524,17 +550,17 @@ export class CLIInterface { config.vectorStore.port || 8000, config.vectorStore.authToken || 'test-token' ); - + await vectorStore.initialize(); const stats = await vectorStore.getCollectionStats(); - + if (options.format === 'json') { console.log(JSON.stringify(stats, null, 2)); } else { console.log(`Vector Count: ${stats.count}`); console.log(`Dimension: ${stats.dimension || 'unknown'}`); } - + await vectorStore.close(); } catch (error) { console.error('Failed to get statistics:', error); @@ -546,7 +572,7 @@ export class CLIInterface { */ private async clearCommand(options: any): Promise { console.log('Clearing data...'); - + try { const config = this.configManager.getConfig(); const vectorStore = new ChromaVectorStore( @@ -555,18 +581,18 @@ export class CLIInterface { config.vectorStore.port || 8000, config.vectorStore.authToken || 'test-token' ); - + await vectorStore.initialize(); - + if (options.all || options.vectors) { await vectorStore.clear(); console.log('Vector store cleared'); } - + if (options.all || options.cache) { console.log('Embedding cache cleared'); } - + await vectorStore.close(); } catch (error) { console.error('Failed to clear data:', error); @@ -583,45 +609,47 @@ export class CLIInterface { console.error('Please specify a file to parse'); return; } - + if (!existsSync(file)) { console.error(`File not found: ${file}`); return; } - + console.log(`Parsing file: ${file}`); const parser = new CodeParser(); const content = readFileSync(file, 'utf-8'); const language = parser.detectLanguage(file); - + if (!language) { console.error(`Unsupported file type: ${file}`); return; } - + console.log(`Language: ${language}`); const chunks = await parser.parseFile(file, content); console.log(`Found ${chunks.length} code chunks:`); - + chunks.forEach((chunk, index) => { - console.log(` ${index + 1}. ${chunk.type}: ${chunk.name} (${chunk.startLine}-${chunk.endLine})`); + console.log( + ` ${index + 1}. ${chunk.type}: ${chunk.name} (${chunk.startLine}-${chunk.endLine})` + ); }); break; - + case 'embed': if (!file) { console.error('Please specify a file to embed'); return; } - + console.log(`Generating embeddings for: ${file}`); const embeddingService = new EmbeddingService(); await embeddingService.initialize(); - + console.log('Embedding debug implementation pending...'); - + break; - + case 'test-connection': console.log('Testing connections...'); try { @@ -632,7 +660,7 @@ export class CLIInterface { config.vectorStore.port || 8000, config.vectorStore.authToken || 'test-token' ); - + await vectorStore.initialize(); const stats = await vectorStore.getCollectionStats(); console.log(`โœ“ ChromaDB connection successful (${stats.count} vectors)`); @@ -641,7 +669,7 @@ export class CLIInterface { console.error('โœ— ChromaDB connection failed:', error); } break; - + default: console.error(`Unknown debug action: ${action}`); console.log('Available actions: parse, embed, test-connection'); @@ -687,7 +715,7 @@ _mcp_context_engine_completion() { } complete -F _mcp_context_engine_completion mcp-context-engine`; - + case 'zsh': return `# zsh completion for mcp-context-engine #compdef mcp-context-engine @@ -710,7 +738,7 @@ _mcp_context_engine() { } compdef _mcp_context_engine mcp-context-engine`; - + case 'fish': return `# fish completion for mcp-context-engine complete -c mcp-context-engine -f @@ -736,7 +764,7 @@ complete -c mcp-context-engine -n "__fish_seen_subcommand_from config" -a "reset complete -c mcp-context-engine -n "__fish_seen_subcommand_from debug" -a "parse" -d "Parse a file" complete -c mcp-context-engine -n "__fish_seen_subcommand_from debug" -a "embed" -d "Generate embeddings for a file" complete -c mcp-context-engine -n "__fish_seen_subcommand_from debug" -a "test-connection" -d "Test connections to services"`; - + default: return `# Generic completion for ${shell} # Add your completion rules here`; @@ -744,4 +772,3 @@ complete -c mcp-context-engine -n "__fish_seen_subcommand_from debug" -a "test-c } } export default CLIInterface; - diff --git a/src/code-parser.ts b/src/code-parser.ts index 10fcd11..431e7b6 100644 --- a/src/code-parser.ts +++ b/src/code-parser.ts @@ -1,5 +1,3 @@ - - import type ParserType from 'tree-sitter'; const Parser = require('tree-sitter'); const TypeScript = require('tree-sitter-typescript'); @@ -70,9 +68,17 @@ export class CodeParser { constructor(options: CodeParserOptions = {}) { this.options = { - languages: options.languages || ['typescript', 'javascript', 'python', 'go', 'rust', 'cpp', 'java'], + languages: options.languages || [ + 'typescript', + 'javascript', + 'python', + 'go', + 'rust', + 'cpp', + 'java', + ], maxFileSize: options.maxFileSize || 1024 * 1024, // 1MB default - timeout: options.timeout || 5000 // 5 seconds default + timeout: options.timeout || 5000, // 5 seconds default }; this.initializeParsers(); @@ -86,7 +92,7 @@ export class CodeParser { try { const parser = new Parser(); const languageModule = this.getLanguageModule(language); - + if (languageModule) { parser.setLanguage(languageModule); this.parsers.set(language, parser); @@ -136,7 +142,7 @@ export class CodeParser { */ detectLanguage(filePath: string): SupportedLanguage | null { const ext = filePath.split('.').pop()?.toLowerCase(); - + switch (ext) { case 'ts': case 'tsx': @@ -191,17 +197,17 @@ export class CodeParser { try { const startTime = Date.now(); - + // Parse the code const tree = parser.parse(content); const rootNode = tree.rootNode; - + // Extract semantic chunks const chunks = this.extractSemanticChunks(rootNode, filePath, language, content); - + const parseTime = Date.now() - startTime; console.log(`Parsed ${filePath} in ${parseTime}ms, found ${chunks.length} chunks`); - + return chunks; } catch (error) { console.error(`Failed to parse file ${filePath}:`, error); @@ -214,17 +220,17 @@ export class CodeParser { */ private extractSemanticChunks( rootNode: ParserType.SyntaxNode, - filePath: string, + filePath: string, language: SupportedLanguage, content: string ): CodeChunk[] { const chunks: CodeChunk[] = []; - + // Define node types to extract based on language const extractableTypes = this.getExtractableNodeTypes(language); - + // Traverse the AST - this.traverseAST(rootNode, (node) => { + this.traverseAST(rootNode, node => { if (extractableTypes.includes(node.type)) { const chunk = this.createCodeChunk(node, filePath, language, content); if (chunk) { @@ -232,7 +238,7 @@ export class CodeParser { } } }); - + return chunks; } @@ -251,7 +257,7 @@ export class CodeParser { 'type_alias_declaration', 'variable_declaration', 'import_statement', - 'export_statement' + 'export_statement', ]; case 'python': return [ @@ -260,7 +266,7 @@ export class CodeParser { 'import_statement', 'import_from_statement', 'assignment', - 'expression_statement' + 'expression_statement', ]; case 'go': return [ @@ -268,7 +274,7 @@ export class CodeParser { 'method_declaration', 'type_declaration', 'import_declaration', - 'var_declaration' + 'var_declaration', ]; case 'rust': return [ @@ -277,7 +283,7 @@ export class CodeParser { 'impl_item', 'trait_item', 'use_declaration', - 'let_declaration' + 'let_declaration', ]; case 'cpp': case 'c': @@ -287,7 +293,7 @@ export class CodeParser { 'struct_specifier', 'namespace_definition', 'using_declaration', - 'declaration' + 'declaration', ]; case 'java': return [ @@ -299,7 +305,7 @@ export class CodeParser { 'field_declaration', 'local_variable_declaration', 'import_declaration', - 'package_declaration' + 'package_declaration', ]; default: return []; @@ -309,7 +315,10 @@ export class CodeParser { /** * Traverse AST and apply callback to each node */ - private traverseAST(node: ParserType.SyntaxNode, callback: (node: ParserType.SyntaxNode) => void): void { + private traverseAST( + node: ParserType.SyntaxNode, + callback: (node: ParserType.SyntaxNode) => void + ): void { callback(node); for (const child of node.children) { this.traverseAST(child, callback); @@ -321,7 +330,7 @@ export class CodeParser { */ private createCodeChunk( node: ParserType.SyntaxNode, - filePath: string, + filePath: string, language: SupportedLanguage, content: string ): CodeChunk | null { @@ -329,14 +338,14 @@ export class CodeParser { const nodeText = node.text; const startLine = node.startPosition.row; const endLine = node.endPosition.row; - + // Extract meaningful information based on node type const chunkType = this.determineChunkType(node.type); const name = this.extractNodeName(node, language); const signature = this.extractSignature(node, language); const documentation = this.extractDocumentation(node, content); const dependencies = this.extractDependencies(node, language); - + return { id: `${filePath}:${node.startPosition.row}:${node.startPosition.column}`, type: chunkType, @@ -352,7 +361,7 @@ export class CodeParser { documentation, dependencies, metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }; } catch (error) { console.error('Failed to create code chunk:', error); @@ -380,14 +389,14 @@ export class CodeParser { private extractNodeName(node: ParserType.SyntaxNode, language: SupportedLanguage): string { // Look for common name patterns in different languages const namePatterns = this.getNamePatterns(language); - + for (const pattern of namePatterns) { const nameNode = this.findNodeByType(node, pattern); if (nameNode) { return nameNode.text; } } - + return 'anonymous'; } @@ -415,7 +424,7 @@ export class CodeParser { 'method_name', 'class_name', 'interface_name', - 'enum_name' + 'enum_name', ]; default: return ['identifier']; @@ -429,39 +438,43 @@ export class CodeParser { if (node.type === type) { return node; } - + for (const child of node.children) { const found = this.findNodeByType(child, type); if (found) { return found; } } - + return null; } /** * Extract function/method signature */ - private extractSignature(node: ParserType.SyntaxNode, language: SupportedLanguage): string | undefined { + private extractSignature( + node: ParserType.SyntaxNode, + language: SupportedLanguage + ): string | undefined { // Look for parameters and return types const signatureParts: string[] = []; - + // Add node name const name = this.extractNodeName(node, language); if (name) { signatureParts.push(name); } - + // Add parameters - const paramsNode = this.findNodeByType(node, 'parameters') || - this.findNodeByType(node, 'formal_parameters') || - this.findNodeByType(node, 'argument_list'); - + const paramsNode = + this.findNodeByType(node, 'parameters') || + this.findNodeByType(node, 'formal_parameters') || + this.findNodeByType(node, 'argument_list'); + if (paramsNode) { signatureParts.push(paramsNode.text); } - + return signatureParts.length > 0 ? signatureParts.join(' ') : undefined; } @@ -471,20 +484,22 @@ export class CodeParser { private extractDocumentation(node: ParserType.SyntaxNode, content: string): string | undefined { // Look for comments before the node const startLine = node.startPosition.row; - + // Simple approach: look for comments in the lines before this node const lines = content.split('\n'); const docLines: string[] = []; - + for (let i = startLine - 1; i >= 0; i--) { const line = lines[i]; if (!line) continue; - + // Check for different comment styles - if (line.trim().startsWith('//') || - line.trim().startsWith('#') || - line.trim().startsWith('/*') || - line.trim().startsWith('*')) { + if ( + line.trim().startsWith('//') || + line.trim().startsWith('#') || + line.trim().startsWith('/*') || + line.trim().startsWith('*') + ) { docLines.unshift(line.trim()); } else if (line.trim() === '') { continue; // Allow empty lines @@ -492,7 +507,7 @@ export class CodeParser { break; // Stop at non-comment, non-empty line } } - + return docLines.length > 0 ? docLines.join('\n') : undefined; } @@ -501,24 +516,24 @@ export class CodeParser { */ private extractDependencies(node: ParserType.SyntaxNode, language: SupportedLanguage): string[] { const dependencies: string[] = []; - + // Find all identifier references in the current scope const identifierNodes = this.findAllNodesByType(node, 'identifier'); const localIdentifiers = new Set(); - + // First, collect all local identifiers (function parameters, variables, etc.) this.collectLocalIdentifiers(node, language, localIdentifiers); - + // Then find external references (excluding local identifiers and built-ins) for (const identifierNode of identifierNodes) { const identifierName = identifierNode.text; - + // Skip local identifiers, built-ins, and common keywords if (this.isExternalReference(identifierName, language, localIdentifiers)) { dependencies.push(identifierName); } } - + // Also collect import statements const importPatterns = this.getImportPatterns(language); for (const pattern of importPatterns) { @@ -530,7 +545,7 @@ export class CodeParser { } } } - + // Remove duplicates and return return [...new Set(dependencies)]; } @@ -538,7 +553,11 @@ export class CodeParser { /** * Collect local identifiers (parameters, variables, etc.) to exclude from dependencies */ - private collectLocalIdentifiers(node: ParserType.SyntaxNode, language: SupportedLanguage, localIdentifiers: Set): void { + private collectLocalIdentifiers( + node: ParserType.SyntaxNode, + language: SupportedLanguage, + localIdentifiers: Set + ): void { // Find parameter declarations const paramPatterns = this.getParameterPatterns(language); for (const pattern of paramPatterns) { @@ -550,7 +569,7 @@ export class CodeParser { } } } - + // Find variable declarations const varPatterns = this.getVariableDeclarationPatterns(language); for (const pattern of varPatterns) { @@ -567,27 +586,31 @@ export class CodeParser { /** * Check if an identifier is an external reference (not local or built-in) */ - private isExternalReference(identifierName: string, language: SupportedLanguage, localIdentifiers: Set): boolean { + private isExternalReference( + identifierName: string, + language: SupportedLanguage, + localIdentifiers: Set + ): boolean { // Skip if it's a local identifier if (localIdentifiers.has(identifierName)) { return false; } - + // Skip built-in keywords and common names if (this.isBuiltInIdentifier(identifierName, language)) { return false; } - + // Skip very short identifiers (likely not meaningful) if (identifierName.length < 2) { return false; } - + // Skip numeric identifiers if (/^\d+$/.test(identifierName)) { return false; } - + return true; } @@ -604,35 +627,120 @@ export class CodeParser { */ private getBuiltInIdentifiers(language: SupportedLanguage): Set { const commonBuiltIns = new Set([ - 'console', 'log', 'error', 'warn', 'info', 'debug', - 'true', 'false', 'null', 'undefined', 'this', 'self', - 'if', 'else', 'for', 'while', 'do', 'switch', 'case', 'default', - 'break', 'continue', 'return', 'function', 'class', 'interface', - 'public', 'private', 'protected', 'static', 'final', 'const', 'let', 'var' + 'console', + 'log', + 'error', + 'warn', + 'info', + 'debug', + 'true', + 'false', + 'null', + 'undefined', + 'this', + 'self', + 'if', + 'else', + 'for', + 'while', + 'do', + 'switch', + 'case', + 'default', + 'break', + 'continue', + 'return', + 'function', + 'class', + 'interface', + 'public', + 'private', + 'protected', + 'static', + 'final', + 'const', + 'let', + 'var', ]); - + switch (language) { case 'typescript': case 'javascript': return new Set([ ...commonBuiltIns, - 'Array', 'Object', 'String', 'Number', 'Boolean', 'Date', 'Math', - 'JSON', 'Promise', 'async', 'await', 'try', 'catch', 'throw', 'new', - 'typeof', 'instanceof', 'in', 'of', 'delete', 'void', 'yield' + 'Array', + 'Object', + 'String', + 'Number', + 'Boolean', + 'Date', + 'Math', + 'JSON', + 'Promise', + 'async', + 'await', + 'try', + 'catch', + 'throw', + 'new', + 'typeof', + 'instanceof', + 'in', + 'of', + 'delete', + 'void', + 'yield', ]); case 'python': return new Set([ ...commonBuiltIns, - 'print', 'len', 'range', 'str', 'int', 'float', 'bool', 'list', 'dict', - 'tuple', 'set', 'import', 'from', 'as', 'def', 'class', 'try', 'except', - 'finally', 'raise', 'with', 'pass', 'lambda', 'global', 'nonlocal' + 'print', + 'len', + 'range', + 'str', + 'int', + 'float', + 'bool', + 'list', + 'dict', + 'tuple', + 'set', + 'import', + 'from', + 'as', + 'def', + 'class', + 'try', + 'except', + 'finally', + 'raise', + 'with', + 'pass', + 'lambda', + 'global', + 'nonlocal', ]); case 'java': return new Set([ ...commonBuiltIns, - 'System', 'String', 'Integer', 'Double', 'Boolean', 'ArrayList', 'HashMap', - 'package', 'import', 'extends', 'implements', 'abstract', 'synchronized', - 'volatile', 'transient', 'native', 'strictfp', 'assert' + 'System', + 'String', + 'Integer', + 'Double', + 'Boolean', + 'ArrayList', + 'HashMap', + 'package', + 'import', + 'extends', + 'implements', + 'abstract', + 'synchronized', + 'volatile', + 'transient', + 'native', + 'strictfp', + 'assert', ]); default: return commonBuiltIns; @@ -708,15 +816,15 @@ export class CodeParser { */ private findAllNodesByType(node: ParserType.SyntaxNode, type: string): ParserType.SyntaxNode[] { const results: ParserType.SyntaxNode[] = []; - + if (node.type === type) { results.push(node); } - + for (const child of node.children) { results.push(...this.findAllNodesByType(child, type)); } - + return results; } @@ -740,12 +848,10 @@ export class CodeParser { try { // Parse incrementally if we have an old tree - const tree = oldTree - ? parser.parse(content, oldTree) - : parser.parse(content); - + const tree = oldTree ? parser.parse(content, oldTree) : parser.parse(content); + const chunks = this.extractSemanticChunks(tree.rootNode, filePath, language, content); - + return { chunks, tree }; } catch (error) { console.error(`Failed to parse file incrementally ${filePath}:`, error); @@ -778,7 +884,7 @@ export class CodeParser { return { supportedLanguages: this.options.languages.length, parsersInitialized: this.parsers.size, - maxFileSize: this.options.maxFileSize + maxFileSize: this.options.maxFileSize, }; } } @@ -806,9 +912,9 @@ module.exports.SupportedLanguage = { Rust: 'rust', Cpp: 'cpp', C: 'c', - Java: 'java' + Java: 'java', }; module.exports.CodeParserOptions = {}; module.exports.CodeChunk = {}; -module.exports.ASTNodeInfo = {}; \ No newline at end of file +module.exports.ASTNodeInfo = {}; diff --git a/src/code-storage.ts b/src/code-storage.ts index 3214546..ded7572 100644 --- a/src/code-storage.ts +++ b/src/code-storage.ts @@ -61,7 +61,7 @@ class CodeStorage { this.options = { maxMemorySize: options.maxMemorySize || 50 * 1024 * 1024, // 50MB default persistToDisk: options.persistToDisk || false, - storagePath: options.storagePath || './.code-storage' + storagePath: options.storagePath || './.code-storage', }; this.memoryCache = new Map(); @@ -142,7 +142,7 @@ class CodeStorage { try { const fs = require('fs'); const path = require('path'); - + const chunkFile = path.join(this.options.storagePath, `${chunk.id}.json`); fs.writeFileSync(chunkFile, JSON.stringify(chunk)); } catch (error) { @@ -155,7 +155,7 @@ class CodeStorage { */ async searchChunks(query: SearchQuery): Promise { const startTime = Date.now(); - + // Start with all chunk IDs let candidateIds = new Set(this.memoryCache.keys()); @@ -259,7 +259,7 @@ class CodeStorage { return { totalChunks: this.memoryCache.size, memoryUsage: this.getCurrentMemoryUsage(), - fileCount: this.fileIndex.size + fileCount: this.fileIndex.size, }; } diff --git a/src/config.ts b/src/config.ts index e48402e..e5ddb2a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,3 @@ - import { readFileSync, existsSync } from 'fs'; import { LogLevel } from './logger.js'; @@ -92,7 +91,7 @@ const DEFAULT_CONFIG: ContextEngineConfig = { modelName: 'Xenova/all-MiniLM-L6-v2', batchSize: 32, maxRetries: 3, - retryDelay: 1000 + retryDelay: 1000, }, vectorStore: { type: 'chroma', @@ -102,7 +101,7 @@ const DEFAULT_CONFIG: ContextEngineConfig = { authToken: process.env.CHROMA_SERVER_AUTH_CREDENTIALS || 'test-token', embeddingDimension: 384, maxVectorsPerFile: 1000, - similarityThreshold: 0.7 + similarityThreshold: 0.7, }, fileWatcher: { ignored: [ @@ -116,7 +115,7 @@ const DEFAULT_CONFIG: ContextEngineConfig = { '**/coverage/**', '**/.env*', '**/package-lock.json', - '**/yarn.lock' + '**/yarn.lock', ], persistent: true, ignoreInitial: false, @@ -124,44 +123,69 @@ const DEFAULT_CONFIG: ContextEngineConfig = { interval: 100, awaitWriteFinish: { stabilityThreshold: 2000, - pollInterval: 100 + pollInterval: 100, }, - maxFileSize: 10 * 1024 * 1024 // 10MB + maxFileSize: 10 * 1024 * 1024, // 10MB }, parser: { maxFileSize: 5 * 1024 * 1024, // 5MB chunkSize: 1000, chunkOverlap: 200, - supportedLanguages: ['javascript', 'typescript', 'python', 'java', 'cpp', 'c', 'rust', 'go', 'ruby', 'php'], + supportedLanguages: [ + 'javascript', + 'typescript', + 'python', + 'java', + 'cpp', + 'c', + 'rust', + 'go', + 'ruby', + 'php', + ], extractDocumentation: true, - extractDependencies: true + extractDependencies: true, }, semanticSearch: { maxResults: 10, minSimilarity: 0.7, enableCaching: true, - cacheSize: 1000 + cacheSize: 1000, }, logging: { level: LogLevel.INFO, enableConsole: true, enableFile: false, maxFileSize: 10 * 1024 * 1024, // 10MB - maxFiles: 5 + maxFiles: 5, }, performance: { maxConcurrentOperations: 10, batchProcessingSize: 32, memoryLimit: 2048, // 2GB cpuLimit: 80, // 80% - enableResourceMonitoring: true + enableResourceMonitoring: true, }, security: { enableSandbox: true, - allowedFileExtensions: ['.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.rs', '.go', '.rb', '.php'], + allowedFileExtensions: [ + '.js', + '.ts', + '.jsx', + '.tsx', + '.py', + '.java', + '.cpp', + '.c', + '.h', + '.rs', + '.go', + '.rb', + '.php', + ], maxFileSize: 5 * 1024 * 1024, // 5MB - enableContentFiltering: true - } + enableContentFiltering: true, + }, }; export class ConfigManager { @@ -290,11 +314,11 @@ export class ConfigManager { // Logging config if (env.DEV_CONTEXT_LOG_LEVEL) { const levelMap: Record = { - 'error': LogLevel.ERROR, - 'warn': LogLevel.WARN, - 'info': LogLevel.INFO, - 'debug': LogLevel.DEBUG, - 'trace': LogLevel.TRACE + error: LogLevel.ERROR, + warn: LogLevel.WARN, + info: LogLevel.INFO, + debug: LogLevel.DEBUG, + trace: LogLevel.TRACE, }; config.logging.level = levelMap[env.DEV_CONTEXT_LOG_LEVEL.toLowerCase()] ?? LogLevel.INFO; } @@ -305,7 +329,9 @@ export class ConfigManager { // Performance config if (env.DEV_CONTEXT_MAX_CONCURRENT_OPERATIONS) { - config.performance.maxConcurrentOperations = parseInt(env.DEV_CONTEXT_MAX_CONCURRENT_OPERATIONS); + config.performance.maxConcurrentOperations = parseInt( + env.DEV_CONTEXT_MAX_CONCURRENT_OPERATIONS + ); } if (env.DEV_CONTEXT_MEMORY_LIMIT) { config.performance.memoryLimit = parseInt(env.DEV_CONTEXT_MEMORY_LIMIT); @@ -356,7 +382,6 @@ export class ConfigManager { } } - // Global config manager instance let globalConfigManager: ConfigManager | null = null; @@ -375,4 +400,4 @@ export function getGlobalConfigManager(): ConfigManager { return globalConfigManager; } -export default ConfigManager; \ No newline at end of file +export default ConfigManager; diff --git a/src/embedding-service.ts b/src/embedding-service.ts index aafada0..6eae999 100644 --- a/src/embedding-service.ts +++ b/src/embedding-service.ts @@ -50,7 +50,7 @@ export class EmbeddingService { maxSequenceLength: options.maxSequenceLength || 512, batchSize: options.batchSize || 32, cacheSize: options.cacheSize || 10000, - device: options.device || 'cpu' + device: options.device || 'cpu', }; this.embeddingCache = new Map(); @@ -62,14 +62,11 @@ export class EmbeddingService { async initialize(): Promise { try { console.log(`Loading embedding model: ${this.options.modelName}`); - + // Dynamic import for Transformers.js const { pipeline } = await import('@xenova/transformers'); - - this.model = await pipeline( - 'feature-extraction', - this.options.modelName - ); + + this.model = await pipeline('feature-extraction', this.options.modelName); this.isModelLoaded = true; console.log(' Embedding model loaded successfully'); @@ -96,11 +93,11 @@ export class EmbeddingService { try { // Prepare text for embedding const text = this.prepareChunkText(chunk); - + // Generate embedding const output = await this.model(text, { pooling: 'mean', - normalize: true + normalize: true, }); // Convert to array and flatten @@ -111,7 +108,7 @@ export class EmbeddingService { vector, dimension: vector.length, timestamp: Date.now(), - model: this.options.modelName + model: this.options.modelName, }; // Cache the result @@ -135,7 +132,9 @@ export class EmbeddingService { for (let i = 0; i < chunks.length; i += batchSize) { const batch = chunks.slice(i, i + batchSize); - console.log(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(chunks.length / batchSize)}`); + console.log( + `Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(chunks.length / batchSize)}` + ); const batchPromises = batch.map(chunk => this.generateEmbedding(chunk)); const batchResults = await Promise.all(batchPromises); @@ -157,13 +156,13 @@ export class EmbeddingService { // Generate embedding const output = await this.model(text, { pooling: 'mean', - normalize: true + normalize: true, }); // Convert to array and flatten return Array.from(output.data as number[]); } catch (error) { - console.error(`Failed to generate embedding for text:`, error); + console.error('Failed to generate embedding for text:', error); throw new Error(`Text embedding generation failed: ${error}`); } } @@ -230,7 +229,7 @@ export class EmbeddingService { cacheSize: this.embeddingCache.size, maxCacheSize: this.options.cacheSize, hitRate: 0, // Could be implemented with access tracking - modelName: this.options.modelName + modelName: this.options.modelName, }; } @@ -273,7 +272,7 @@ export class EmbeddingService { ): Promise> { const similarities = candidateEmbeddings.map(embedding => ({ embedding, - similarity: this.calculateSimilarity(queryEmbedding.vector, embedding.vector) + similarity: this.calculateSimilarity(queryEmbedding.vector, embedding.vector), })); // Sort by similarity (descending) @@ -303,7 +302,7 @@ export class EmbeddingService { name: this.options.modelName, dimension: this.isModelLoaded ? 384 : null, // all-MiniLM-L6-v2 outputs 384 dimensions maxSequenceLength: this.options.maxSequenceLength, - device: this.options.device + device: this.options.device, }; } @@ -318,4 +317,4 @@ export class EmbeddingService { // Factory function for backward compatibility with tests export function createEmbeddingService(options?: EmbeddingOptions): EmbeddingService { return new EmbeddingService(options); -} \ No newline at end of file +} diff --git a/src/file-watcher.ts b/src/file-watcher.ts index 7f462c8..b5e62ef 100644 --- a/src/file-watcher.ts +++ b/src/file-watcher.ts @@ -16,10 +16,12 @@ export interface FileWatcherOptions { binaryInterval?: number; alwaysStat?: boolean; depth?: number; - awaitWriteFinish?: boolean | { - stabilityThreshold?: number; - pollInterval?: number; - }; + awaitWriteFinish?: + | boolean + | { + stabilityThreshold?: number; + pollInterval?: number; + }; ignorePermissionErrors?: boolean; atomic?: boolean; } @@ -98,7 +100,7 @@ export class FileWatcher extends EventEmitter { constructor(options: FileWatcherOptions = {}) { super(); - + this.options = { projectPath: options.projectPath || process.cwd(), ignored: options.ignored || [ @@ -107,7 +109,7 @@ export class FileWatcher extends EventEmitter { '**/dist/**', '**/build/**', '**/.DS_Store', - '**/Thumbs.db' + '**/Thumbs.db', ], persistent: options.persistent !== false, ignoreInitial: options.ignoreInitial !== false, @@ -119,11 +121,11 @@ export class FileWatcher extends EventEmitter { depth: options.depth || undefined, awaitWriteFinish: options.awaitWriteFinish || { stabilityThreshold: 2000, - pollInterval: 100 + pollInterval: 100, }, ignorePermissionErrors: options.ignorePermissionErrors !== false, atomic: options.atomic || true, - ...options + ...options, } as Required; } @@ -151,12 +153,12 @@ export class FileWatcher extends EventEmitter { depth: this.options.depth, awaitWriteFinish: this.options.awaitWriteFinish, ignorePermissionErrors: this.options.ignorePermissionErrors, - atomic: this.options.atomic + atomic: this.options.atomic, }); this.setupEventHandlers(); this.isWatching = true; - + console.log('File watcher started successfully'); } catch (error) { console.error('Failed to start file watcher:', error); @@ -187,7 +189,11 @@ export class FileWatcher extends EventEmitter { }) .on('addDir', (dirPath: string, stats?: Stats) => { console.log(`Directory added: ${dirPath}`); - this.emit('directoryAdded', { dirPath, stats, timestamp: Date.now() } as DirectoryAddedEvent); + this.emit('directoryAdded', { + dirPath, + stats, + timestamp: Date.now(), + } as DirectoryAddedEvent); }) .on('unlinkDir', (dirPath: string) => { console.log(`Directory removed: ${dirPath}`); @@ -221,7 +227,7 @@ export class FileWatcher extends EventEmitter { this.watcher = null; this.isWatching = false; this.watchedFiles.clear(); - + console.log('File watcher stopped successfully'); } catch (error) { console.error('Error stopping file watcher:', error); @@ -269,7 +275,7 @@ export class FileWatcher extends EventEmitter { isWatching: this.isWatching, watchedFileCount: this.watchedFiles.size, projectPath: this.options.projectPath, - options: this.options + options: this.options, }; } } @@ -284,4 +290,4 @@ export function createFileWatcher(options: FileWatcherOptions = {}): FileWatcher /** * Default export */ -export default FileWatcher; \ No newline at end of file +export default FileWatcher; diff --git a/src/index.ts b/src/index.ts index 4a39349..1dfd311 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,7 +43,7 @@ export class ContextEngine { constructor(options: ContextEngineOptions) { this.options = options; this.configManager = new ConfigManager(options.configPath); - + // Configure logging based on debug mode if (options.debug) { this.logger.setLevel(LogLevel.DEBUG); @@ -76,7 +76,7 @@ export class ContextEngine { this.fileWatcher = new FileWatcher({ projectPath: this.options.projectPath, ignoreInitial: false, - persistent: true + persistent: true, }); // Set up file watcher event handlers @@ -105,30 +105,30 @@ export class ContextEngine { private setupFileWatcherHandlers(): void { if (!this.fileWatcher) return; - this.fileWatcher.on('fileAdded', async (event) => { + this.fileWatcher.on('fileAdded', async event => { this.logger.info(`[FILE ADDED] ${event.filePath}`); await this.handleFileAdded(event.filePath); }); - this.fileWatcher.on('fileChanged', async (event) => { + this.fileWatcher.on('fileChanged', async event => { this.logger.info(`[FILE CHANGED] ${event.filePath}`); await this.handleFileChanged(event.filePath); }); - this.fileWatcher.on('fileRemoved', async (event) => { + this.fileWatcher.on('fileRemoved', async event => { this.logger.info(`[FILE REMOVED] ${event.filePath}`); await this.handleFileRemoved(event.filePath); }); - this.fileWatcher.on('directoryAdded', (event) => { + this.fileWatcher.on('directoryAdded', event => { this.logger.info(`[DIRECTORY ADDED] ${event.dirPath}`); }); - this.fileWatcher.on('directoryRemoved', (event) => { + this.fileWatcher.on('directoryRemoved', event => { this.logger.info(`[DIRECTORY REMOVED] ${event.dirPath}`); }); - this.fileWatcher.on('error', (error) => { + this.fileWatcher.on('error', error => { this.logger.error('[FILE WATCHER ERROR]', error); }); @@ -144,7 +144,7 @@ export class ContextEngine { try { this.logger.debug(`Processing new file: ${filePath}`); - + // Check if file is supported based on extension if (!this.isSupportedFile(filePath)) { this.logger.debug(`Skipping unsupported file: ${filePath}`); @@ -153,7 +153,7 @@ export class ContextEngine { // Read file content const content = readFileSync(filePath, 'utf-8'); - + // Check file size limit const config = this.configManager.getConfig(); if (content.length > config.security.maxFileSize) { @@ -175,7 +175,7 @@ export class ContextEngine { try { this.logger.debug(`Processing changed file: ${filePath}`); - + // Check if file is supported if (!this.isSupportedFile(filePath)) { return; @@ -183,7 +183,7 @@ export class ContextEngine { // Read updated content const content = readFileSync(filePath, 'utf-8'); - + // Check file size limit const config = this.configManager.getConfig(); if (content.length > config.security.maxFileSize) { @@ -205,12 +205,12 @@ export class ContextEngine { try { this.logger.debug(`Processing removed file: ${filePath}`); - + // Remove from vector index await this.semanticSearch.handleFileChange({ type: 'delete', - filePath: filePath, - language: 'unknown' + filePath, + language: 'unknown', }); this.logger.info(`Successfully removed file from index: ${filePath}`); } catch (error) { @@ -221,10 +221,25 @@ export class ContextEngine { private isSupportedFile(filePath: string): boolean { const supportedExtensions = [ - '.js', '.jsx', '.ts', '.tsx', '.py', '.java', '.cpp', '.c', '.h', - '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala' + '.js', + '.jsx', + '.ts', + '.tsx', + '.py', + '.java', + '.cpp', + '.c', + '.h', + '.cs', + '.php', + '.rb', + '.go', + '.rs', + '.swift', + '.kt', + '.scala', ]; - + const ext = path.extname(filePath).toLowerCase(); return supportedExtensions.includes(ext); } @@ -234,28 +249,32 @@ export class ContextEngine { try { this.logger.info('Performing initial indexing of existing files...'); - + const watchedFiles = this.fileWatcher.getWatchedFiles(); const supportedFiles = watchedFiles.filter(file => this.isSupportedFile(file)); - + this.logger.info(`Found ${supportedFiles.length} supported files to index`); - + // Index files in batches to avoid memory issues const batchSize = 50; for (let i = 0; i < supportedFiles.length; i += batchSize) { const batch = supportedFiles.slice(i, i + batchSize); - this.logger.debug(`Indexing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(supportedFiles.length / batchSize)}`); - - await Promise.all(batch.map(async (filePath) => { - try { - const content = readFileSync(filePath, 'utf-8'); - await this.semanticSearch!.indexFile(filePath, content); - } catch (error) { - this.logger.error(`Failed to index file ${filePath} during initial indexing:`, error); - } - })); + this.logger.debug( + `Indexing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(supportedFiles.length / batchSize)}` + ); + + await Promise.all( + batch.map(async filePath => { + try { + const content = readFileSync(filePath, 'utf-8'); + await this.semanticSearch!.indexFile(filePath, content); + } catch (error) { + this.logger.error(`Failed to index file ${filePath} during initial indexing:`, error); + } + }) + ); } - + this.logger.info('Initial indexing completed'); } catch (error) { this.logger.error('Failed to perform initial indexing:', error); @@ -309,7 +328,6 @@ export class ContextEngine { console.log('Press Ctrl+C to stop the context engine'); } - /** * Get engine statistics @@ -318,11 +336,10 @@ export class ContextEngine { return { isRunning: this.isRunning, fileWatcher: this.fileWatcher?.getStats() || null, - uptime: this.isRunning ? Date.now() - this.startTime : 0 + uptime: this.isRunning ? Date.now() - this.startTime : 0, }; } - } // Export for module usage -export default ContextEngine; \ No newline at end of file +export default ContextEngine; diff --git a/src/logger.ts b/src/logger.ts index 8d12609..6f0acbb 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -5,7 +5,7 @@ export enum LogLevel { WARN = 1, INFO = 2, DEBUG = 3, - TRACE = 4 + TRACE = 4, } export interface LoggerOptions { @@ -53,7 +53,7 @@ export class Logger { private getTimestamp(): string { const now = new Date(); - + switch (this.timestampFormat) { case 'iso': return now.toISOString(); @@ -68,10 +68,13 @@ export class Logger { private formatMessage(level: string, message: string, ...args: any[]): string { const timestamp = this.getTimestamp(); - const formattedArgs = args.length > 0 ? ' ' + args.map(arg => - typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg) - ).join(' ') : ''; - + const formattedArgs = + args.length > 0 + ? ` ${args + .map(arg => (typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg))) + .join(' ')}` + : ''; + return `[${timestamp}] [${level}] ${message}${formattedArgs}`; } @@ -82,19 +85,24 @@ export class Logger { // Console output if (this.enableConsole) { - const consoleMethod = level === LogLevel.ERROR ? console.error : - level === LogLevel.WARN ? console.warn : - level === LogLevel.DEBUG ? console.debug : - level === LogLevel.TRACE ? console.trace : - console.log; - + const consoleMethod = + level === LogLevel.ERROR + ? console.error + : level === LogLevel.WARN + ? console.warn + : level === LogLevel.DEBUG + ? console.debug + : level === LogLevel.TRACE + ? console.trace + : console.log; + consoleMethod(formattedMessage); } // File output if (this.enableFile && this.fileStream) { - this.fileStream.write(formattedMessage + '\n'); - + this.fileStream.write(`${formattedMessage}\n`); + // Check if we need to rotate the log file this.checkFileRotation(); } @@ -118,15 +126,15 @@ export class Logger { try { this.fileStream.end(); - + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const rotatedFile = `${this.logFile}.${timestamp}`; - + require('fs').renameSync(this.logFile, rotatedFile); - + // Clean up old rotated files this.cleanupOldLogs(); - + // Create new log file this.fileStream = createWriteStream(this.logFile, { flags: 'a' }); } catch (error) { @@ -142,12 +150,13 @@ export class Logger { const path = require('path'); const logDir = path.dirname(this.logFile); const logBaseName = path.basename(this.logFile); - - const files = fs.readdirSync(logDir) - .filter((file: string) => file.startsWith(logBaseName + '.')) + + const files = fs + .readdirSync(logDir) + .filter((file: string) => file.startsWith(`${logBaseName}.`)) .sort() .reverse(); - + // Keep only the most recent maxFiles const filesToDelete = files.slice(this.maxFiles); filesToDelete.forEach((file: string) => { @@ -233,4 +242,4 @@ export function logTrace(message: string, ...args: any[]): void { getGlobalLogger().trace(message, ...args); } -export default Logger; \ No newline at end of file +export default Logger; diff --git a/src/semantic-search.ts b/src/semantic-search.ts index 79dac03..c8fa001 100644 --- a/src/semantic-search.ts +++ b/src/semantic-search.ts @@ -6,7 +6,7 @@ import { SearchOptions, SearchResult, SemanticSearchConfig, - FileChangeEvent + FileChangeEvent, } from './types'; export class SemanticSearch { @@ -24,12 +24,12 @@ export class SemanticSearch { chunkOverlap: 200, maxResults: 10, minSimilarity: 0.7, - ...config + ...config, }; this.parser = new CodeParser(); this.embeddingService = new EmbeddingService(); - + // Initialize ChromaDB vector store with proper connection parameters this.vectorStore = new ChromaVectorStore( 'code_vectors', @@ -44,7 +44,7 @@ export class SemanticSearch { try { console.log('Initializing Semantic Search...'); - + // Initialize embedding service await this.embeddingService.initialize(); console.log('Embedding service initialized'); @@ -89,7 +89,7 @@ export class SemanticSearch { type: chunk.type, lineStart: chunk.startLine, lineEnd: chunk.endLine, - timestamp: chunk.timestamp + timestamp: chunk.timestamp, })); // Delete existing vectors for this file (if any) @@ -98,7 +98,6 @@ export class SemanticSearch { // Add new vectors to store await this.vectorStore.addVectors(codeVectors); console.log(`Indexed ${codeVectors.length} vectors for ${filePath}`); - } catch (error) { console.error(`Failed to index file ${filePath}:`, error); throw error; @@ -123,7 +122,7 @@ export class SemanticSearch { const searchOptions = { topK: this.config.maxResults || 10, minSimilarity: this.config.minSimilarity || 0.7, - ...options + ...options, }; // Search for similar vectors @@ -136,21 +135,19 @@ export class SemanticSearch { console.log(`Found ${searchResults.length} raw results`); // Filter by minimum similarity and other criteria - let filteredResults = searchResults.filter(result => - result.similarity >= searchOptions.minSimilarity! + let filteredResults = searchResults.filter( + result => result.similarity >= searchOptions.minSimilarity! ); // Apply additional filters if (searchOptions.filePath) { - filteredResults = filteredResults.filter(result => + filteredResults = filteredResults.filter(result => result.filePath.includes(searchOptions.filePath!) ); } if (searchOptions.chunkType) { - filteredResults = filteredResults.filter(result => - result.type === searchOptions.chunkType - ); + filteredResults = filteredResults.filter(result => result.type === searchOptions.chunkType); } // Sort by similarity (highest first) @@ -167,9 +164,8 @@ export class SemanticSearch { results: finalResults, query, resultCount: finalResults.length, - searchTime + searchTime, }; - } catch (error) { console.error(`Search failed for query "${query}":`, error); throw error; @@ -193,7 +189,7 @@ export class SemanticSearch { break; } } catch (error) { - console.error(`Failed to handle file change event:`, error); + console.error('Failed to handle file change event:', error); // Don't throw - file changes shouldn't break the system } } @@ -210,12 +206,12 @@ export class SemanticSearch { try { const vectorStoreStats = await this.vectorStore.getCollectionStats(); - + return { totalVectors: vectorStoreStats.count, embeddingDimension: this.embeddingService.getEmbeddingDimension(), collectionName: 'code_vectors', - indexedFiles: [] // This would need to be tracked separately + indexedFiles: [], // This would need to be tracked separately }; } catch (error) { console.error('Failed to get index stats:', error); @@ -234,4 +230,4 @@ export class SemanticSearch { // Factory function for backward compatibility with tests export function createSemanticSearchService(config?: SemanticSearchConfig): SemanticSearch { return new SemanticSearch(config); -} \ No newline at end of file +} diff --git a/src/tree-sitter-types.d.ts b/src/tree-sitter-types.d.ts index aa85f4b..63b75fc 100644 --- a/src/tree-sitter-types.d.ts +++ b/src/tree-sitter-types.d.ts @@ -32,4 +32,4 @@ declare module 'tree-sitter-cpp' { declare module 'tree-sitter-java' { const java: any; export = java; -} \ No newline at end of file +} diff --git a/src/types.ts b/src/types.ts index 81164db..158ca01 100644 --- a/src/types.ts +++ b/src/types.ts @@ -43,7 +43,11 @@ export interface VectorSearchResult { export interface VectorStore { initialize(): Promise; addVectors(vectors: CodeVector[]): Promise; - searchSimilar(queryVector: number[], topK?: number, language?: string): Promise; + searchSimilar( + queryVector: number[], + topK?: number, + language?: string + ): Promise; deleteVectors(filePath: string): Promise; updateVectors(vectors: CodeVector[]): Promise; getCollectionStats(): Promise<{ count: number; dimension?: number }>; @@ -100,4 +104,4 @@ export interface FileChangeEvent { filePath: string; content?: string; language: string; -} \ No newline at end of file +} diff --git a/src/vector-store.ts b/src/vector-store.ts index 0351939..fe26927 100644 --- a/src/vector-store.ts +++ b/src/vector-store.ts @@ -1,4 +1,3 @@ - /** * Vector Store Service for persistent storage and similarity search of embeddings * Uses ChromaDB for local vector database operations @@ -54,12 +53,12 @@ class VectorStore { collectionName: options.collectionName || 'code_embeddings', persistDirectory: options.persistDirectory || './.chroma_db', distanceMetric: options.distanceMetric || 'cosine', - embeddingDimension: options.embeddingDimension || 384 + embeddingDimension: options.embeddingDimension || 384, }; // Initialize ChromaDB client this.client = new ChromaClient({ - path: this.options.persistDirectory + path: this.options.persistDirectory, }); } @@ -69,16 +68,18 @@ class VectorStore { async initialize(): Promise { try { console.log(`Initializing vector store: ${this.options.collectionName}`); - + // Check if collection exists const collections = await this.client.listCollections(); - const collectionExists = collections.some((col: any) => col.name === this.options.collectionName); - + const collectionExists = collections.some( + (col: any) => col.name === this.options.collectionName + ); + if (collectionExists) { // Get existing collection this.collection = await this.client.getCollection({ name: this.options.collectionName, - embeddingFunction: new SimpleEmbeddingFunction() + embeddingFunction: new SimpleEmbeddingFunction(), }); console.log(`Using existing collection: ${this.options.collectionName}`); } else { @@ -88,8 +89,8 @@ class VectorStore { embeddingFunction: new SimpleEmbeddingFunction(), metadata: { 'hnsw:space': this.options.distanceMetric, - dimension: this.options.embeddingDimension - } + dimension: this.options.embeddingDimension, + }, }); console.log(`Created new collection: ${this.options.collectionName}`); } @@ -126,7 +127,7 @@ class VectorStore { endLine: chunk.endLine, timestamp: chunk.timestamp, embeddingTimestamp: embedding.timestamp, - model: embedding.model + model: embedding.model, }; // Store in ChromaDB @@ -134,7 +135,7 @@ class VectorStore { ids: [embedding.chunkId], embeddings: [embedding.vector], metadatas: [metadata], - documents: [chunk.content.substring(0, 1000)] // Truncate content for storage + documents: [chunk.content.substring(0, 1000)], // Truncate content for storage }); console.log(`Stored embedding for chunk: ${chunk.name} (${chunk.id})`); @@ -172,7 +173,7 @@ class VectorStore { ids.push(embedding.chunkId); vectors.push(embedding.vector); - + metadatas.push({ chunkId: chunk.id, chunkType: chunk.type, @@ -183,7 +184,7 @@ class VectorStore { endLine: chunk.endLine, timestamp: chunk.timestamp, embeddingTimestamp: embedding.timestamp, - model: embedding.model + model: embedding.model, }); documents.push(chunk.content.substring(0, 1000)); @@ -194,7 +195,7 @@ class VectorStore { ids, embeddings: vectors, metadatas, - documents + documents, }); console.log(`Stored ${embeddings.length} embeddings in vector store`); @@ -236,12 +237,12 @@ class VectorStore { queryEmbeddings: [queryEmbedding], nResults: topK, where: Object.keys(where).length > 0 ? where : undefined, - include: [IncludeEnum.Metadatas, IncludeEnum.Documents, IncludeEnum.Distances] + include: [IncludeEnum.Metadatas, IncludeEnum.Documents, IncludeEnum.Distances], }); // Convert to our format const searchResults: VectorSearchResult[] = []; - + if (results.ids && results.ids[0]) { for (let i = 0; i < results.ids[0].length; i++) { const chunkId = results.ids[0][i]; @@ -275,26 +276,26 @@ class VectorStore { type: chunkType, name: chunkName, content: document || '', - filePath: filePath, - language: language, - startLine: startLine, - endLine: endLine, + filePath, + language, + startLine, + endLine, startColumn: 0, endColumn: 0, signature: undefined, documentation: undefined, dependencies: [], metadata: undefined, - timestamp: timestamp + timestamp, }, embedding: { chunkId: chunkIdStr, vector: [], // Vector not returned by default in query dimension: this.options.embeddingDimension, timestamp: embeddingTimestamp, - model: model - } - } + model, + }, + }, }); } } @@ -336,7 +337,7 @@ class VectorStore { documentation: undefined, dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }); return this.searchSimilar(queryEmbedding.vector, topK, filter); @@ -354,7 +355,7 @@ class VectorStore { const count = await this.collection.count(); return { count, - dimension: this.options.embeddingDimension + dimension: this.options.embeddingDimension, }; } catch (error) { console.error('Failed to get vector store stats:', error); @@ -378,8 +379,8 @@ class VectorStore { embeddingFunction: new SimpleEmbeddingFunction(), metadata: { 'hnsw:space': this.options.distanceMetric, - dimension: this.options.embeddingDimension - } + dimension: this.options.embeddingDimension, + }, }); console.log('Vector store cleared successfully'); } catch (error) { diff --git a/test/debug-watcher.ts b/test/debug-watcher.ts index 09a5e68..cb384b1 100644 --- a/test/debug-watcher.ts +++ b/test/debug-watcher.ts @@ -4,11 +4,11 @@ import path from 'path'; async function debugWatcher(): Promise { const testDir = path.join(process.cwd(), 'debug-test'); - + try { // Create test directory await fs.mkdir(testDir, { recursive: true }); - + console.log('Creating file watcher...'); const watcher = new FileWatcher({ projectPath: testDir, @@ -17,18 +17,24 @@ async function debugWatcher(): Promise { interval: 100, awaitWriteFinish: { stabilityThreshold: 500, - pollInterval: 100 - } + pollInterval: 100, + }, }); // Set up event listeners - watcher.on('fileAdded', (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { - console.log(' FILE ADDED:', event.filePath); - }); + watcher.on( + 'fileAdded', + (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { + console.log(' FILE ADDED:', event.filePath); + } + ); - watcher.on('fileChanged', (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { - console.log(' FILE CHANGED:', event.filePath); - }); + watcher.on( + 'fileChanged', + (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { + console.log(' FILE CHANGED:', event.filePath); + } + ); watcher.on('fileRemoved', (event: { filePath: string; timestamp: number }) => { console.log(' FILE REMOVED:', event.filePath); @@ -41,38 +47,37 @@ async function debugWatcher(): Promise { // Start watching console.log('Starting watcher...'); await watcher.start(); - + // Wait for ready await new Promise(resolve => setTimeout(resolve, 1000)); - + const testFile = path.join(testDir, 'test.txt'); - + // Test file addition console.log('\n--- Testing file addition ---'); await fs.writeFile(testFile, 'Hello World'); await new Promise(resolve => setTimeout(resolve, 2000)); - + // Test file change console.log('\n--- Testing file change ---'); await fs.writeFile(testFile, 'Hello World Modified'); await new Promise(resolve => setTimeout(resolve, 2000)); - + // Test file removal console.log('\n--- Testing file removal ---'); await fs.unlink(testFile); await new Promise(resolve => setTimeout(resolve, 2000)); - + // Cleanup console.log('\n--- Cleaning up ---'); await watcher.stop(); await fs.rm(testDir, { recursive: true, force: true }); - + console.log('Debug complete!'); - } catch (error) { console.error('Debug error:', error); } } // Run debug -debugWatcher().catch(console.error); \ No newline at end of file +debugWatcher().catch(console.error); diff --git a/test/test-embedding.ts b/test/test-embedding.ts index 40086df..0759e19 100644 --- a/test/test-embedding.ts +++ b/test/test-embedding.ts @@ -22,7 +22,7 @@ const testChunks: CodeChunk[] = [ documentation: 'Calculates the sum of two numbers', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk2', @@ -39,7 +39,7 @@ const testChunks: CodeChunk[] = [ documentation: 'Calculates the product of two numbers', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk3', @@ -56,8 +56,8 @@ const testChunks: CodeChunk[] = [ documentation: 'A simple calculator class', dependencies: [], metadata: undefined, - timestamp: Date.now() - } + timestamp: Date.now(), + }, ]; async function testEmbeddingService() { @@ -67,7 +67,7 @@ async function testEmbeddingService() { // Create embedding service const embeddingService = createEmbeddingService({ modelName: 'Xenova/all-MiniLM-L6-v2', - cacheSize: 1000 + cacheSize: 1000, }); console.log(' Embedding service created'); @@ -88,7 +88,7 @@ async function testEmbeddingService() { console.log('\n Testing batch embedding generation...'); const batchResults = await embeddingService.generateBatchEmbeddings({ chunks: testChunks, - batchSize: 2 + batchSize: 2, }); console.log(` Generated ${batchResults.length} embeddings in batch`); @@ -98,7 +98,9 @@ async function testEmbeddingService() { batchResults[0].vector, batchResults[1].vector ); - console.log(` Similarity between "${testChunks[0].name}" and "${testChunks[1].name}": ${similarity.toFixed(4)}`); + console.log( + ` Similarity between "${testChunks[0].name}" and "${testChunks[1].name}": ${similarity.toFixed(4)}` + ); // Test finding most similar console.log('\n Testing most similar search...'); @@ -130,7 +132,6 @@ async function testEmbeddingService() { console.log(` Model info: ${JSON.stringify(modelInfo, null, 2)}`); console.log('\n All embedding tests completed successfully!'); - } catch (error) { console.error(' Embedding test failed:', error); process.exit(1); @@ -142,4 +143,4 @@ if (require.main === module) { testEmbeddingService().catch(console.error); } -module.exports = { testEmbeddingService }; \ No newline at end of file +module.exports = { testEmbeddingService }; diff --git a/test/test-file-watcher.ts b/test/test-file-watcher.ts index b2559c9..da481fc 100644 --- a/test/test-file-watcher.ts +++ b/test/test-file-watcher.ts @@ -29,9 +29,9 @@ describe('FileWatcher', () => { test('should create FileWatcher instance', () => { watcher = new FileWatcher({ projectPath: testDir, - ignoreInitial: true + ignoreInitial: true, }); - + assert.ok(watcher instanceof FileWatcher); assert.strictEqual(watcher.options.projectPath, testDir); assert.strictEqual(watcher.getStats().isWatching, false); @@ -40,15 +40,15 @@ describe('FileWatcher', () => { test('should start and stop watching', async () => { watcher = new FileWatcher({ projectPath: testDir, - ignoreInitial: true + ignoreInitial: true, }); await setupTestDir(); - + try { await watcher.start(); assert.strictEqual(watcher.getStats().isWatching, true); - + await watcher.stop(); assert.strictEqual(watcher.getStats().isWatching, false); } finally { @@ -58,39 +58,42 @@ describe('FileWatcher', () => { test('should detect file addition', async () => { await setupTestDir(); - + const testFile = path.join(testDir, 'test-add.txt'); - + // Create the watcher with better polling settings for tests watcher = new FileWatcher({ projectPath: testDir, ignoreInitial: true, usePolling: true, - interval: 50, // Faster polling for tests + interval: 50, // Faster polling for tests binaryInterval: 100, - awaitWriteFinish: false, // Disable for faster test response - atomic: false // Disable atomic writes for tests + awaitWriteFinish: false, // Disable for faster test response + atomic: false, // Disable atomic writes for tests }); // Set up event listener BEFORE starting the watcher let fileAddedEvent: any = null; - watcher.on('fileAdded', (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { - console.log(`Test received fileAdded event: ${event.filePath}`); - if (event.filePath === testFile) { - fileAddedEvent = event; + watcher.on( + 'fileAdded', + (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { + console.log(`Test received fileAdded event: ${event.filePath}`); + if (event.filePath === testFile) { + fileAddedEvent = event; + } } - }); + ); try { await watcher.start(); - + // Wait longer for watcher to be fully ready await new Promise(resolve => setTimeout(resolve, 1000)); - + console.log(`Creating test file: ${testFile}`); // Create a test file await fs.writeFile(testFile, 'test content'); - + // Wait for the file addition event with longer timeout let attempts = 0; const maxAttempts = 30; // 3 seconds with 100ms intervals @@ -98,7 +101,7 @@ describe('FileWatcher', () => { await new Promise(resolve => setTimeout(resolve, 100)); attempts++; } - + console.log(`File addition event received: ${fileAddedEvent !== null}`); assert.ok(fileAddedEvent, 'File addition event should be emitted'); assert.strictEqual(fileAddedEvent!.filePath, testFile); @@ -110,9 +113,9 @@ describe('FileWatcher', () => { test('should detect file changes', async () => { await setupTestDir(); - + const testFile = path.join(testDir, 'test-change.txt'); - + // Create the watcher with better polling settings for tests watcher = new FileWatcher({ projectPath: testDir, @@ -121,31 +124,34 @@ describe('FileWatcher', () => { interval: 50, binaryInterval: 100, awaitWriteFinish: false, - atomic: false + atomic: false, }); // Set up event listener BEFORE starting the watcher let fileChangedEvent: any = null; - watcher.on('fileChanged', (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { - console.log(`Test received fileChanged event: ${event.filePath}`); - if (event.filePath === testFile) { - fileChangedEvent = event; + watcher.on( + 'fileChanged', + (event: { filePath: string; stats?: import('fs').Stats; timestamp: number }) => { + console.log(`Test received fileChanged event: ${event.filePath}`); + if (event.filePath === testFile) { + fileChangedEvent = event; + } } - }); + ); try { // Create initial file first await fs.writeFile(testFile, 'initial content'); - + await watcher.start(); - + // Wait longer for watcher to be fully ready await new Promise(resolve => setTimeout(resolve, 1000)); - + console.log(`Modifying test file: ${testFile}`); // Change the file await fs.writeFile(testFile, 'modified content'); - + // Wait for the file change event let attempts = 0; const maxAttempts = 30; @@ -153,7 +159,7 @@ describe('FileWatcher', () => { await new Promise(resolve => setTimeout(resolve, 100)); attempts++; } - + console.log(`File change event received: ${fileChangedEvent !== null}`); assert.ok(fileChangedEvent, 'File change event should be emitted'); assert.strictEqual(fileChangedEvent.filePath, testFile); @@ -165,9 +171,9 @@ describe('FileWatcher', () => { test('should detect file removal', async () => { await setupTestDir(); - + const testFile = path.join(testDir, 'test-remove.txt'); - + // Create the watcher with better polling settings for tests watcher = new FileWatcher({ projectPath: testDir, @@ -176,7 +182,7 @@ describe('FileWatcher', () => { interval: 50, binaryInterval: 100, awaitWriteFinish: false, - atomic: false + atomic: false, }); // Set up event listener BEFORE starting the watcher @@ -191,16 +197,16 @@ describe('FileWatcher', () => { try { // Create initial file await fs.writeFile(testFile, 'test content'); - + await watcher.start(); - + // Wait longer for watcher to be fully ready await new Promise(resolve => setTimeout(resolve, 1000)); - + console.log(`Removing test file: ${testFile}`); // Remove the file await fs.unlink(testFile); - + // Wait for the file removal event let attempts = 0; const maxAttempts = 30; @@ -208,7 +214,7 @@ describe('FileWatcher', () => { await new Promise(resolve => setTimeout(resolve, 100)); attempts++; } - + console.log(`File removal event received: ${fileRemovedEvent !== null}`); assert.ok(fileRemovedEvent, 'File removal event should be emitted'); assert.strictEqual(fileRemovedEvent.filePath, testFile); @@ -220,33 +226,33 @@ describe('FileWatcher', () => { test('should track watched files', async () => { await setupTestDir(); - + const testFile = path.join(testDir, 'test-track.txt'); // Create the watcher with better settings for file tracking watcher = new FileWatcher({ projectPath: testDir, - ignoreInitial: false, // We want to track initial files + ignoreInitial: false, // We want to track initial files usePolling: true, interval: 50, binaryInterval: 100, awaitWriteFinish: false, - atomic: false + atomic: false, }); try { // Create initial file first await fs.writeFile(testFile, 'test content'); - + await watcher.start(); - + // Wait longer for initial scan to complete await new Promise(resolve => setTimeout(resolve, 2000)); - + const watchedFiles = watcher.getWatchedFiles(); console.log(`Watched files: ${watchedFiles.length}, looking for: ${testFile}`); console.log(`Files found: ${watchedFiles.join(', ')}`); - + // The file should be in the watched files list const fileFound = watchedFiles.some(file => file.includes('test-track.txt')); assert.ok(fileFound, `File ${testFile} should be in watched files list`); @@ -260,19 +266,19 @@ describe('FileWatcher', () => { test('should provide statistics', async () => { watcher = new FileWatcher({ projectPath: testDir, - ignoreInitial: true + ignoreInitial: true, }); await setupTestDir(); - + const testFile = path.join(testDir, 'test-stats.txt'); try { await fs.writeFile(testFile, 'test content'); await watcher.start(); - + const stats = watcher.getStats(); - + assert.strictEqual(stats.isWatching, true); assert.strictEqual(stats.projectPath, testDir); assert.ok(stats.options); @@ -285,7 +291,7 @@ describe('FileWatcher', () => { test('should handle errors gracefully', async () => { watcher = new FileWatcher({ projectPath: '/nonexistent/path', - ignoreInitial: true + ignoreInitial: true, }); let errorEmitted = false; @@ -298,7 +304,7 @@ describe('FileWatcher', () => { await watcher.start(); // Should emit error for nonexistent path await new Promise(resolve => setTimeout(resolve, 1000)); - + // Note: Chokidar might not immediately error on nonexistent paths // This test verifies the error handling mechanism is in place assert.ok(watcher.getStats().isWatching || errorEmitted); diff --git a/test/test-integration.ts b/test/test-integration.ts index 7fbd796..b07031a 100644 --- a/test/test-integration.ts +++ b/test/test-integration.ts @@ -5,72 +5,71 @@ import * as fs from 'fs'; import * as path from 'path'; async function testIntegration() { - console.log('\n Testing Full Integration: File Watcher โ†’ Parser โ†’ Storage โ†’ Semantic Search\n'); + console.log('\n Testing Full Integration: File Watcher โ†’ Parser โ†’ Storage โ†’ Semantic Search\n'); - const testDir = './test-integration-temp'; - const testFile = path.join(testDir, 'test-code.js'); - let fileWatcher: FileWatcher | null = null; + const testDir = './test-integration-temp'; + const testFile = path.join(testDir, 'test-code.js'); + let fileWatcher: FileWatcher | null = null; - try { - // Clean up any existing test directory - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }); + try { + // Clean up any existing test directory + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + fs.mkdirSync(testDir, { recursive: true }); + + // Initialize all components + console.log('1. Initializing components...'); + + const { CodeParser } = require('../src/code-parser'); + const parser = new CodeParser(); + const storage = new CodeStorage(); + const semanticSearch = createSemanticSearchService(); + const fileWatcherOptions: FileWatcherOptions = { + projectPath: testDir, + ignored: ['**/node_modules/**', '**/.git/**'], + }; + fileWatcher = new FileWatcher(fileWatcherOptions); + + await semanticSearch.initialize(); + + console.log(' All components initialized'); + + // Set up file change handler + const handleFileChange = async (event: any) => { + console.log(`\n File changed: ${path.basename(event.filePath)}`); + + try { + // Read the file content + const content = fs.readFileSync(event.filePath, 'utf8'); + + // Parse the code + const chunks = await parser.parseFile(event.filePath, content); + console.log(` Parsed ${chunks.length} code chunks`); + + // Store in storage + await storage.storeChunks(chunks); + const stats = storage.getStats(); + console.log(` Stored in cache with ${stats.totalChunks} total chunks`); + + // Index for semantic search + for (const chunk of chunks) { + await semanticSearch.indexFile(chunk.filePath, chunk.content); } - fs.mkdirSync(testDir, { recursive: true }); - - // Initialize all components - console.log('1. Initializing components...'); - - const { CodeParser } = require('../src/code-parser'); - const parser = new CodeParser(); - const storage = new CodeStorage(); - const semanticSearch = createSemanticSearchService(); - const fileWatcherOptions: FileWatcherOptions = { - projectPath: testDir, - ignored: ['**/node_modules/**', '**/.git/**'] - }; - fileWatcher = new FileWatcher(fileWatcherOptions); - - await semanticSearch.initialize(); - - console.log(' All components initialized'); - - // Set up file change handler - const handleFileChange = async (event: any) => { - console.log(`\n File changed: ${path.basename(event.filePath)}`); - - try { - // Read the file content - const content = fs.readFileSync(event.filePath, 'utf8'); - - // Parse the code - const chunks = await parser.parseFile(event.filePath, content); - console.log(` Parsed ${chunks.length} code chunks`); - - // Store in storage - await storage.storeChunks(chunks); - const stats = storage.getStats(); - console.log(` Stored in cache with ${stats.totalChunks} total chunks`); - - // Index for semantic search - for (const chunk of chunks) { - await semanticSearch.indexFile(chunk.filePath, chunk.content); - } - console.log(` Indexed ${chunks.length} chunks for semantic search`); - - } catch (error) { - console.error(` Error processing file: ${error}`); - } - }; - - // Start watching - fileWatcher.on('fileChanged', handleFileChange); - await fileWatcher.start(); - console.log(' File watcher started'); - - // Test 1: Create initial file - console.log('\n2. Test 1: Creating initial file...'); - const initialCode = ` + console.log(` Indexed ${chunks.length} chunks for semantic search`); + } catch (error) { + console.error(` Error processing file: ${error}`); + } + }; + + // Start watching + fileWatcher.on('fileChanged', handleFileChange); + await fileWatcher.start(); + console.log(' File watcher started'); + + // Test 1: Create initial file + console.log('\n2. Test 1: Creating initial file...'); + const initialCode = ` function calculateSum(a, b) { return a + b; } @@ -100,29 +99,31 @@ class Calculator { } `; - fs.writeFileSync(testFile, initialCode); - console.log(' Initial file created'); - - // Manually process the initial file since ignoreInitial is true - console.log(' Processing initial file manually...'); - await handleFileChange({ filePath: testFile }); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Test semantic search - console.log('\n3. Testing semantic search on indexed code...'); - - const searchResult = await semanticSearch.search('calculate sum of numbers'); - console.log(` Found ${searchResult.resultCount} relevant results:`); - searchResult.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); - console.log(` File: ${result.filePath}`); - }); - - // Test 2: Update file with new code - console.log('\n4. Test 2: Updating file with authentication code...'); - const updatedCode = ` + fs.writeFileSync(testFile, initialCode); + console.log(' Initial file created'); + + // Manually process the initial file since ignoreInitial is true + console.log(' Processing initial file manually...'); + await handleFileChange({ filePath: testFile }); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Test semantic search + console.log('\n3. Testing semantic search on indexed code...'); + + const searchResult = await semanticSearch.search('calculate sum of numbers'); + console.log(` Found ${searchResult.resultCount} relevant results:`); + searchResult.results.forEach((result: any, index: number) => { + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); + console.log(` File: ${result.filePath}`); + }); + + // Test 2: Update file with new code + console.log('\n4. Test 2: Updating file with authentication code...'); + const updatedCode = ` function calculateSum(a, b) { return a + b; } @@ -169,95 +170,106 @@ function validateInput(input) { } `; - fs.writeFileSync(testFile, updatedCode); - console.log(' File updated with authentication code'); - - // Manually process the updated file - console.log(' Processing updated file manually...'); - await handleFileChange({ filePath: testFile }); - - // Wait for processing - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Test semantic search for authentication - console.log('\n5. Testing semantic search for authentication...'); - - const authResult = await semanticSearch.search('user authentication login'); - console.log(` Found ${authResult.resultCount} relevant results:`); - authResult.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); - console.log(` File: ${result.filePath}`); - }); - - // Test 3: Search for validation patterns - console.log('\n6. Test 3: Searching for validation patterns...'); - - const validationResult = await semanticSearch.search('input validation check'); - console.log(` Found ${validationResult.resultCount} relevant results:`); - validationResult.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); - console.log(` File: ${result.filePath}`); - }); - - // Test 4: Find similar patterns - console.log('\n7. Test 4: Finding similar patterns...'); - - // Test 4: Find similar patterns (already implemented above) - console.log('\n7. Test 4: Finding similar patterns...'); - - const similarResult = await semanticSearch.search('error handling middleware pattern', { - topK: 5, - minSimilarity: 0.6 - }); - console.log(` Found ${similarResult.resultCount} similar patterns:`); - similarResult.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); - console.log(` File: ${result.filePath}`); - }); - - // Test 5: Service statistics - console.log('\n8. Service Statistics:'); - const stats = await semanticSearch.getIndexStats(); - console.log(` - Vector store: ${stats.totalVectors} vectors, ${stats.embeddingDimension} dimensions`); - const storageStats = storage.getStats(); - console.log(` - Files in storage: ${storageStats.fileCount} files, ${storageStats.totalChunks} chunks`); - console.log(` - Collection name: ${stats.collectionName}`); - - // Test 6: Complex search with filters - console.log('\n9. Test 6: Complex search with language filter...'); - - const complexResult = await semanticSearch.search('calculate math', { - language: 'javascript', - topK: 3, - minSimilarity: 0.2 - }); - console.log(` Found ${complexResult.resultCount} filtered results:`); - complexResult.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); - }); - - console.log('\n All integration tests completed successfully!'); - - } catch (error) { - console.error(' Integration test failed:', error); - } finally { - // Clean up - console.log('\n๐Ÿงน Cleaning up...'); - if (fileWatcher) { - await fileWatcher.stop(); - } - - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }); - } - - console.log(' Cleanup completed'); + fs.writeFileSync(testFile, updatedCode); + console.log(' File updated with authentication code'); + + // Manually process the updated file + console.log(' Processing updated file manually...'); + await handleFileChange({ filePath: testFile }); + + // Wait for processing + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Test semantic search for authentication + console.log('\n5. Testing semantic search for authentication...'); + + const authResult = await semanticSearch.search('user authentication login'); + console.log(` Found ${authResult.resultCount} relevant results:`); + authResult.results.forEach((result: any, index: number) => { + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); + console.log(` File: ${result.filePath}`); + }); + + // Test 3: Search for validation patterns + console.log('\n6. Test 3: Searching for validation patterns...'); + + const validationResult = await semanticSearch.search('input validation check'); + console.log(` Found ${validationResult.resultCount} relevant results:`); + validationResult.results.forEach((result: any, index: number) => { + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); + console.log(` File: ${result.filePath}`); + }); + + // Test 4: Find similar patterns + console.log('\n7. Test 4: Finding similar patterns...'); + + // Test 4: Find similar patterns (already implemented above) + console.log('\n7. Test 4: Finding similar patterns...'); + + const similarResult = await semanticSearch.search('error handling middleware pattern', { + topK: 5, + minSimilarity: 0.6, + }); + console.log(` Found ${similarResult.resultCount} similar patterns:`); + similarResult.results.forEach((result: any, index: number) => { + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); + console.log(` File: ${result.filePath}`); + }); + + // Test 5: Service statistics + console.log('\n8. Service Statistics:'); + const stats = await semanticSearch.getIndexStats(); + console.log( + ` - Vector store: ${stats.totalVectors} vectors, ${stats.embeddingDimension} dimensions` + ); + const storageStats = storage.getStats(); + console.log( + ` - Files in storage: ${storageStats.fileCount} files, ${storageStats.totalChunks} chunks` + ); + console.log(` - Collection name: ${stats.collectionName}`); + + // Test 6: Complex search with filters + console.log('\n9. Test 6: Complex search with language filter...'); + + const complexResult = await semanticSearch.search('calculate math', { + language: 'javascript', + topK: 3, + minSimilarity: 0.2, + }); + console.log(` Found ${complexResult.resultCount} filtered results:`); + complexResult.results.forEach((result: any, index: number) => { + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); + }); + + console.log('\n All integration tests completed successfully!'); + } catch (error) { + console.error(' Integration test failed:', error); + } finally { + // Clean up + console.log('\n๐Ÿงน Cleaning up...'); + if (fileWatcher) { + await fileWatcher.stop(); } + + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + + console.log(' Cleanup completed'); + } } // Run the integration test if (require.main === module) { - testIntegration().catch(console.error); + testIntegration().catch(console.error); } -export { testIntegration }; \ No newline at end of file +export { testIntegration }; diff --git a/test/test-parser.ts b/test/test-parser.ts index 84e5863..aee66f4 100644 --- a/test/test-parser.ts +++ b/test/test-parser.ts @@ -4,14 +4,14 @@ import { join } from 'path'; async function testParser() { console.log(' Testing Tree-sitter Code Parser...\n'); - + // Create test directory const testDir = join(__dirname, 'parser-test'); try { rmSync(testDir, { recursive: true, force: true }); } catch {} mkdirSync(testDir, { recursive: true }); - + // Create test files for different languages const testFiles = { 'test.ts': ` @@ -85,36 +85,38 @@ class DataProcessor: def process(self): return [x * 2 for x in self.data] -` +`, }; - + // Write test files for (const [filename, content] of Object.entries(testFiles)) { writeFileSync(join(testDir, filename), content); } - + // Initialize parser const parser = new CodeParser({ - languages: ['typescript', 'javascript', 'java', 'python'] + languages: ['typescript', 'javascript', 'java', 'python'], }); - + console.log(' Parsing test files...\n'); - + // Parse each file for (const [filename, content] of Object.entries(testFiles)) { console.log(` Parsing ${filename}:`); const filePath = join(testDir, filename); const language = parser.detectLanguage(filePath); - + if (language) { console.log(` Language detected: ${language}`); - + try { const chunks = await parser.parseFile(filePath, content); console.log(` Found ${chunks.length} code chunks:`); - + chunks.forEach((chunk: any, index: number) => { - console.log(` ${index + 1}. ${chunk.type}: ${chunk.name} (${chunk.startLine + 1}-${chunk.endLine + 1})`); + console.log( + ` ${index + 1}. ${chunk.type}: ${chunk.name} (${chunk.startLine + 1}-${chunk.endLine + 1})` + ); if (chunk.signature) { console.log(` Signature: ${chunk.signature}`); } @@ -123,17 +125,17 @@ class DataProcessor: console.log(` Error parsing: ${error instanceof Error ? error.message : String(error)}`); } } else { - console.log(` Language not supported`); + console.log(' Language not supported'); } console.log(''); } - + // Test incremental parsing console.log('๐Ÿ”„ Testing incremental parsing...\n'); - - const modifiedContent = testFiles['test.ts'] + '\n// Added comment\nfunction newFunction() { return 42; }'; + + const modifiedContent = `${testFiles['test.ts']}\n// Added comment\nfunction newFunction() { return 42; }`; writeFileSync(join(testDir, 'test.ts'), modifiedContent); - + try { const chunks = await parser.parseFile(join(testDir, 'test.ts'), modifiedContent); console.log(` After modification: ${chunks.length} chunks found`); @@ -142,16 +144,18 @@ class DataProcessor: console.log(` New function detected: ${newFunction.name}`); } } catch (error) { - console.log(` Error in incremental parsing: ${error instanceof Error ? error.message : String(error)}`); + console.log( + ` Error in incremental parsing: ${error instanceof Error ? error.message : String(error)}` + ); } - + // Cleanup try { rmSync(testDir, { recursive: true, force: true }); } catch {} - + console.log('\n Parser test completed!'); } // Run the test -testParser().catch(console.error); \ No newline at end of file +testParser().catch(console.error); diff --git a/test/test-semantic-search.ts b/test/test-semantic-search.ts index 71592bf..74184d2 100644 --- a/test/test-semantic-search.ts +++ b/test/test-semantic-search.ts @@ -22,7 +22,7 @@ const testChunks: CodeChunk[] = [ documentation: 'Calculates the sum of two numbers', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk2', @@ -39,7 +39,7 @@ const testChunks: CodeChunk[] = [ documentation: 'Calculates the product of two numbers', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk3', @@ -56,13 +56,14 @@ const testChunks: CodeChunk[] = [ documentation: 'A simple calculator class', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk4', type: 'function', name: 'authenticateUser', - content: 'function authenticateUser(username, password) { if (!username || !password) return false; return validateCredentials(username, password); }', + content: + 'function authenticateUser(username, password) { if (!username || !password) return false; return validateCredentials(username, password); }', filePath: '/test/auth.js', language: 'javascript', startLine: 1, @@ -73,13 +74,14 @@ const testChunks: CodeChunk[] = [ documentation: 'Authenticates user with username and password', dependencies: ['validateCredentials'], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk5', type: 'function', name: 'validateInput', - content: 'function validateInput(input) { if (!input || input.length === 0) return false; return input.length > 3; }', + content: + 'function validateInput(input) { if (!input || input.length === 0) return false; return input.length > 3; }', filePath: '/test/validation.js', language: 'javascript', startLine: 1, @@ -90,8 +92,8 @@ const testChunks: CodeChunk[] = [ documentation: 'Validates input data', dependencies: [], metadata: undefined, - timestamp: Date.now() - } + timestamp: Date.now(), + }, ]; async function testSemanticSearch() { @@ -102,7 +104,7 @@ async function testSemanticSearch() { const semanticSearch = createSemanticSearchService({ embeddingModel: 'Xenova/all-MiniLM-L6-v2', minSimilarity: 0.2, - maxResults: 5 + maxResults: 5, }); console.log(' Semantic search service created'); @@ -123,11 +125,13 @@ async function testSemanticSearch() { // Test 1: Basic semantic search console.log('\n Test 1: Basic semantic search'); const searchResults1 = await semanticSearch.search('calculate sum of numbers', { - topK: 3 + topK: 3, }); console.log(` Found ${searchResults1.resultCount} results:`); searchResults1.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); console.log(` File: ${result.filePath}`); }); @@ -135,11 +139,13 @@ async function testSemanticSearch() { console.log('\n Test 2: Filtered search by language'); const searchResults2 = await semanticSearch.search('authentication', { language: 'javascript', - topK: 2 + topK: 2, }); console.log(` Found ${searchResults2.resultCount} results:`); searchResults2.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); console.log(` File: ${result.filePath}`); }); @@ -147,23 +153,30 @@ async function testSemanticSearch() { console.log('\n Test 3: Search with higher threshold'); const searchResults3 = await semanticSearch.search('validation', { minSimilarity: 0.3, - topK: 3 + topK: 3, }); console.log(` Found ${searchResults3.resultCount} results with threshold 0.3:`); searchResults3.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); console.log(` File: ${result.filePath}`); }); // Test 4: Find similar patterns console.log('\n Test 4: Find similar patterns'); - const patternResults = await semanticSearch.search('function validateUser(input) { if (!input) return false; }', { - language: 'javascript', - topK: 3 - }); + const patternResults = await semanticSearch.search( + 'function validateUser(input) { if (!input) return false; }', + { + language: 'javascript', + topK: 3, + } + ); console.log(` Found ${patternResults.resultCount} similar patterns:`); patternResults.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); console.log(` File: ${result.filePath}`); }); @@ -177,16 +190,17 @@ async function testSemanticSearch() { const complexResults = await semanticSearch.search('calculate math operations', { language: 'javascript', chunkType: 'function', - topK: 2 + topK: 2, }); console.log(` Found ${complexResults.resultCount} complex filtered results:`); complexResults.results.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); console.log(` File: ${result.filePath}`); }); console.log('\n All semantic search tests completed successfully!'); - } catch (error) { console.error(' Semantic search test failed:', error); process.exit(1); @@ -198,4 +212,4 @@ if (require.main === module) { testSemanticSearch().catch(console.error); } -module.exports = { testSemanticSearch }; \ No newline at end of file +module.exports = { testSemanticSearch }; diff --git a/test/test-storage.ts b/test/test-storage.ts index 69e2380..52ebe2e 100644 --- a/test/test-storage.ts +++ b/test/test-storage.ts @@ -21,7 +21,7 @@ const testChunks = [ documentation: undefined, dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk2', @@ -38,7 +38,7 @@ const testChunks = [ documentation: undefined, dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk3', @@ -55,8 +55,8 @@ const testChunks = [ documentation: undefined, dependencies: [], metadata: undefined, - timestamp: Date.now() - } + timestamp: Date.now(), + }, ]; async function testCodeStorage() { @@ -66,7 +66,7 @@ async function testCodeStorage() { // Create storage instance const storage = new StorageService({ maxMemorySize: 10 * 1024 * 1024, // 10MB - persistToDisk: false + persistToDisk: false, }); console.log(' Storage instance created'); @@ -134,7 +134,6 @@ async function testCodeStorage() { console.log(` After clear: ${finalStats.totalChunks} chunks remaining`); console.log('\n All storage tests completed successfully!'); - } catch (error) { console.error(' Storage test failed:', error); process.exit(1); @@ -146,4 +145,4 @@ if (require.main === module) { testCodeStorage().catch(console.error); } -module.exports = { testCodeStorage }; \ No newline at end of file +module.exports = { testCodeStorage }; diff --git a/test/test-vector-store.ts b/test/test-vector-store.ts index 57ffb69..b13d66d 100644 --- a/test/test-vector-store.ts +++ b/test/test-vector-store.ts @@ -23,7 +23,7 @@ const testChunks: CodeChunk[] = [ documentation: 'Calculates the sum of two numbers', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk2', @@ -40,7 +40,7 @@ const testChunks: CodeChunk[] = [ documentation: 'Calculates the product of two numbers', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }, { id: 'chunk3', @@ -57,8 +57,8 @@ const testChunks: CodeChunk[] = [ documentation: 'A simple calculator class', dependencies: [], metadata: undefined, - timestamp: Date.now() - } + timestamp: Date.now(), + }, ]; async function testVectorStore() { @@ -68,7 +68,7 @@ async function testVectorStore() { // Create embedding service const embeddingService = new EmbeddingService({ modelName: 'Xenova/all-MiniLM-L6-v2', - cacheSize: 1000 + cacheSize: 1000, }); // Create vector store - connect to Docker ChromaDB @@ -94,7 +94,7 @@ async function testVectorStore() { console.log('\n Generating embeddings for test chunks...'); const embeddings = await embeddingService.generateBatchEmbeddings({ chunks: testChunks, - batchSize: 2 + batchSize: 2, }); console.log(` Generated ${embeddings.length} embeddings`); @@ -108,7 +108,7 @@ async function testVectorStore() { type: testChunks[index].type, lineStart: testChunks[index].startLine, lineEnd: testChunks[index].endLine, - timestamp: testChunks[index].timestamp + timestamp: testChunks[index].timestamp, })); // Store embeddings in vector store @@ -118,25 +118,22 @@ async function testVectorStore() { // Test vector search console.log('\n Testing vector search...'); - const searchResults = await vectorStore.searchSimilar( - embeddings[0].vector, - 3 - ); + const searchResults = await vectorStore.searchSimilar(embeddings[0].vector, 3); console.log(` Found ${searchResults.length} similar vectors:`); searchResults.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.type} ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.type} ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); }); // Test filtered search console.log('\n Testing filtered vector search...'); - const filteredResults = await vectorStore.searchSimilar( - embeddings[0].vector, - 2, - 'javascript' - ); + const filteredResults = await vectorStore.searchSimilar(embeddings[0].vector, 2, 'javascript'); console.log(` Found ${filteredResults.length} filtered similar vectors:`); filteredResults.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.type} ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.type} ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); }); // Test text-based search @@ -156,16 +153,15 @@ async function testVectorStore() { documentation: 'Query chunk for testing', dependencies: [], metadata: undefined, - timestamp: Date.now() + timestamp: Date.now(), }; const queryEmbedding = await embeddingService.generateEmbedding(queryChunk); - const textResults = await vectorStore.searchSimilar( - queryEmbedding.vector, - 2 - ); + const textResults = await vectorStore.searchSimilar(queryEmbedding.vector, 2); console.log(` Found ${textResults.length} text-based similar vectors:`); textResults.forEach((result: any, index: number) => { - console.log(` ${index + 1}. ${result.type} ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})`); + console.log( + ` ${index + 1}. ${result.type} ${result.content.substring(0, 50)}... (similarity: ${result.similarity.toFixed(4)})` + ); }); // Test vector store stats @@ -185,7 +181,6 @@ async function testVectorStore() { console.log(' Vector store closed successfully'); console.log('\n All vector store tests completed successfully!'); - } catch (error) { console.error(' Vector store test failed:', error); process.exit(1); @@ -197,4 +192,4 @@ if (require.main === module) { testVectorStore().catch(console.error); } -module.exports = { testVectorStore }; \ No newline at end of file +module.exports = { testVectorStore }; diff --git a/tsconfig.json b/tsconfig.json index 9fcf205..d83bc73 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,14 +24,6 @@ "allowSyntheticDefaultImports": true, "rewriteRelativeImportExtensions": true }, - "include": [ - "src/**/*", - "test/**/*" - ], - "exclude": [ - "node_modules", - "dist", - "debug-test", - "test-files" - ] -} \ No newline at end of file + "include": ["src/**/*", "test/**/*"], + "exclude": ["node_modules", "dist", "debug-test", "test-files"] +}