Local directory support (#56)

Author: brendan-kellam

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Local directory indexing support. ([#56](https://github.com/sourcebot-dev/sourcebot/pull/56))
+
 ## [2.2.0] - 2024-10-30
 
 ### Added

Dockerfile

@@ -76,6 +76,7 @@ COPY --from=zoekt-builder \
     /cmd/zoekt-mirror-gitlab \
     /cmd/zoekt-mirror-gerrit \
     /cmd/zoekt-webserver \
+    /cmd/zoekt-index \
     /usr/local/bin/
 
 # Configure the webapp

README.md

@@ -267,6 +267,37 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
 
 If you're using a self-hosted GitLab or GitHub instance with a custom domain, you can specify the domain in your config file. See [configs/self-hosted.json](configs/self-hosted.json) for examples.
 
+## Searching a local directory
+
+Local directories can be searched by using the `local` type in your config file:
+
+```jsonc
+{
+    "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "local",
+            "path": "/repos/my-repo",
+            // re-index files when a change is detected
+            "watch": true,
+            "exclude": {
+                // exclude paths from being indexed
+                "paths": [
+                    "node_modules",
+                    "build"
+                ]
+            }
+        }
+    ]
+}
+```
+
+You'll need to mount the directory as a volume when running Sourcebot:
+
+<pre>
+docker run <b>-v /path/to/my-repo:/repos/my-repo</b> /* additional args */ ghcr.io/sourcebot-dev/sourcebot:latest
+</pre>
+
 ## Build from source
 >[!NOTE]
 > Building from source is only required if you'd like to contribute. The recommended way to use Sourcebot is to use the [pre-built docker image](https://github.com/sourcebot-dev/sourcebot/pkgs/container/sourcebot).

configs/auth.json

@@ -17,6 +17,13 @@
                 "my-group"
             ]
         },
+        {
+            "type": "gitea",
+            "token": "gitea-token",
+            "orgs": [
+                "my-org"
+            ]
+        },
 
         // You can also store the token in a environment variable and then
         // references it from the config.
@@ -34,6 +41,15 @@
             "groups": [
                 "my-group"
             ]
+        },
+        {
+            "type": "gitea",
+            "token": {
+                "env": "GITEA_TOKEN_ENV_VAR"
+            },
+            "orgs": [
+                "my-org"
+            ]
         }
     ]
 }

configs/basic.json

@@ -37,6 +37,28 @@
             "projects": [
                 "my-group/project1"
             ]
+        },
+        // From Gitea, include:
+        // - all public repos owned by user `my-user`
+        // - all public repos owned by organization `my-org`
+        // - repo `my-org/my-repo`
+        {
+            "type": "gitea",
+            "token": "my-token",
+            "users": [
+                "my-user"
+            ],
+            "orgs": [
+                "my-org"
+            ],
+            "repos": [
+                "my-org/my-repo"
+            ]
+        },
+        // Index a local repository
+        {
+            "type": "local",
+            "path": "/path/to/local/repo"
         }
     ]
 }

configs/filter.json

@@ -37,6 +37,25 @@
                     "my-group/project2"
                 ]
             }
-        }
+        },
+
+        // Include all repos in my-org, except:
+        // - repo1 & repo2
+        // - repos that are archived or forks
+        {
+            "type": "gitea",
+            "token": "my-token",
+            "orgs": [
+                "my-org"
+            ],
+            "exclude": {
+                "archived": true,
+                "forks": true,
+                "repos": [
+                    "my-org/repo1",
+                    "my-org/repo2"
+                ]
+            }
+        },
     ]
 }

configs/local-repo.json

@@ -0,0 +1,32 @@
+{
+    "$schema": "../schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "local",
+            "path": "/path/to/local/repo"
+        },
+        // Relative paths are relative to the config file
+        {
+            "type": "local",
+            "path": "../../relative/path/to/local/repo"
+        },
+        // File watcher can be disabled (enabled by default)
+        {
+            "type": "local",
+            "path": "/path/to/local/repo",
+            "watch": false
+        },
+        // Exclude paths can be specified
+        {
+            "type": "local",
+            "path": "/path/to/local/repo",
+            "exclude": {
+                "paths": [
+                    ".git",
+                    "node_modules",
+                    "dist"
+                ]
+            }
+        }
+    ]
+}

configs/self-hosted.json

@@ -14,6 +14,13 @@
             "groups": [
                 "my-group"
             ]
+        },
+        {
+            "type": "gitea",
+            "url": "https://gitea.example.com",
+            "orgs": [
+                "my-org-name"
+            ]
         }
     ]
 }

packages/backend/src/db.ts

@@ -14,7 +14,7 @@ export const loadDB = async (ctx: AppContext): Promise<Database> => {
     const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, { repos: {} });
     return db;
 }
-export const updateRepository = async (repoId: string, data: Partial<Repository>, db: Database) => {
+export const updateRepository = async (repoId: string, data: Repository, db: Database) => {
     db.data.repos[repoId] = {
         ...db.data.repos[repoId],
         ...data,

packages/backend/src/git.ts

@@ -1,11 +1,11 @@
-import { Repository } from './types.js';
+import { GitRepository } from './types.js';
 import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { existsSync } from 'fs';
 import { createLogger } from './logger.js';
 
 const logger = createLogger('git');
 
-export const cloneRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
+export const cloneRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
     if (existsSync(repo.path)) {
         logger.warn(`${repo.id} already exists. Skipping clone.`)
         return;
@@ -34,7 +34,7 @@ export const cloneRepository = async (repo: Repository, onProgress?: (event: Sim
 }
 
 
-export const fetchRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
+export const fetchRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
     const git = simpleGit({
         progress: onProgress,
     });