chore: speedup builds for binary and prepackage

.vscode/tasks.json

@@ -103,6 +103,9 @@
         "clear": true
       },
       "options": {
+        "env": {
+          "SKIP_INSTALLS": "true"
+        },
         "cwd": "${workspaceFolder}/extensions/vscode"
       }
     },

binary/build.js

@@ -3,13 +3,13 @@ const fs = require("fs");
 const path = require("path");
 const ncp = require("ncp").ncp;
 const { rimrafSync } = require("rimraf");
-const {
-  validateFilesPresent,
-  execCmdSync,
-  autodetectPlatformAndArch,
-} = require("../scripts/util");
-const { downloadRipgrep } = require("./utils/ripgrep");
+const { validateFilesPresent } = require("../scripts/util");
 const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
+const { fork } = require("child_process");
+const {
+  installAndCopyNodeModules,
+} = require("../extensions/vscode/scripts/install-copy-nodemodule");
+const { bundleBinary } = require("./utils/bundle-binary");
 
 const bin = path.join(__dirname, "bin");
 const out = path.join(__dirname, "out");
@@ -29,8 +29,6 @@ function cleanSlate() {
 const esbuildOutputFile = "out/index.js";
 let targets = [...ALL_TARGETS];
 
-const [currentPlatform, currentArch] = autodetectPlatformAndArch();
-
 const assetBackups = [
   "node_modules/win-ca/lib/crypt32-ia32.node.bak",
   "node_modules/win-ca/lib/crypt32-x64.node.bak",
@@ -78,85 +76,6 @@ async function buildWithEsbuild() {
   });
 }
 
-async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
-  console.log(`Copying ${packageName} to ${toCopy}`);
-  // This is a way to install only one package without npm trying to install all the dependencies
-  // Create a temporary directory for installing the package
-  const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
-  const tempDir = path.join(
-    __dirname,
-    "tmp",
-    `continue-node_modules-${adjustedName}`,
-  );
-  const currentDir = process.cwd();
-
-  // // Remove the dir we will be copying to
-  // rimrafSync(`node_modules/${toCopy}`);
-
-  // // Ensure the temporary directory exists
-  fs.mkdirSync(tempDir, { recursive: true });
-
-  try {
-    // Move to the temporary directory
-    process.chdir(tempDir);
-
-    // Initialize a new package.json and install the package
-    execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
-
-    console.log(
-      `Contents of: ${packageName}`,
-      fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
-    );
-
-    // Without this it seems the file isn't completely written to disk
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-
-    // Copy the installed package back to the current directory
-    await new Promise((resolve, reject) => {
-      ncp(
-        path.join(tempDir, "node_modules", toCopy),
-        path.join(currentDir, "node_modules", toCopy),
-        { dereference: true },
-        (error) => {
-          if (error) {
-            console.error(
-              `[error] Error copying ${packageName} package`,
-              error,
-            );
-            reject(error);
-          } else {
-            resolve();
-          }
-        },
-      );
-    });
-  } finally {
-    // Clean up the temporary directory
-    // rimrafSync(tempDir);
-
-    // Return to the original directory
-    process.chdir(currentDir);
-  }
-}
-
-/**
- * Downloads and installs ripgrep binaries for the specified target
- *
- * @param {string} target - Target platform-arch (e.g., 'darwin-x64')
- * @param {string} targetDir - Directory to install ripgrep to
- * @returns {Promise<void>}
- */
-async function downloadRipgrepForTarget(target, targetDir) {
-  console.log(`[info] Downloading ripgrep for ${target}...`);
-  try {
-    await downloadRipgrep(target, targetDir);
-    console.log(`[info] Successfully installed ripgrep for ${target}`);
-  } catch (error) {
-    console.error(`[error] Failed to download ripgrep for ${target}:`, error);
-    throw error;
-  }
-}
-
 (async () => {
   if (esbuildOnly) {
     await buildWithEsbuild();
@@ -181,16 +100,21 @@ async function downloadRipgrepForTarget(target, targetDir) {
     ),
   );
 
-  console.log("[info] Downloading prebuilt lancedb...");
+  const copyLanceDBPromises = [];
   for (const target of targets) {
-    if (TARGET_TO_LANCEDB[target]) {
-      console.log(`[info] Downloading for ${target}...`);
-      await installNodeModuleInTempDirAndCopyToCurrent(
-        TARGET_TO_LANCEDB[target],
-        "@lancedb",
-      );
+    if (!TARGET_TO_LANCEDB[target]) {
+      continue;
     }
+    console.log(`[info] Downloading for ${target}...`);
+    copyLanceDBPromises.push(
+      installAndCopyNodeModules(TARGET_TO_LANCEDB[target], "@lancedb"),
+    );
   }
+  await Promise.all(copyLanceDBPromises).catch(() => {
+    console.error("[error] Failed to copy LanceDB");
+    process.exit(1);
+  });
+  console.log("[info] Copied all LanceDB");
 
   // tree-sitter-wasm
   const treeSitterWasmsDir = path.join(out, "tree-sitter-wasms");
@@ -252,59 +176,16 @@ async function downloadRipgrepForTarget(target, targetDir) {
     "out/llamaTokenizerWorkerPool.mjs",
   );
 
+  const buildBinaryPromises = [];
   console.log("[info] Building binaries with pkg...");
   for (const target of targets) {
-    const targetDir = `bin/${target}`;
-    fs.mkdirSync(targetDir, { recursive: true });
-    console.log(`[info] Building ${target}...`);
-    execCmdSync(
-      `npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
-    );
-
-    // Download and unzip prebuilt sqlite3 binary for the target
-    console.log("[info] Downloading node-sqlite3");
-
-    const downloadUrl =
-      // node-sqlite3 doesn't have a pre-built binary for win32-arm64
-      target === "win32-arm64"
-        ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
-        : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
-            target
-          }.tar.gz`;
-
-    execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
-    execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
-
-    // Copy to build directory for testing
-    try {
-      const [platform, arch] = target.split("-");
-      if (platform === currentPlatform && arch === currentArch) {
-        fs.copyFileSync(
-          `${targetDir}/build/Release/node_sqlite3.node`,
-          `build/node_sqlite3.node`,
-        );
-      }
-    } catch (error) {
-      console.log("[warn] Could not copy node_sqlite to build");
-      console.log(error);
-    }
-
-    fs.unlinkSync(`${targetDir}/build.tar.gz`);
-
-    // copy @lancedb to bin folders
-    console.log("[info] Copying @lancedb files to bin");
-    fs.copyFileSync(
-      `node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
-      `${targetDir}/index.node`,
-    );
-
-    // Download and install ripgrep for the target
-    await downloadRipgrepForTarget(target, targetDir);
-
-    // Informs the `continue-binary` of where to look for node_sqlite3.node
-    // https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
-    fs.writeFileSync(`${targetDir}/package.json`, "");
+    buildBinaryPromises.push(bundleBinary(target));
   }
+  await Promise.all(buildBinaryPromises).catch(() => {
+    console.error("[error] Failed to build binaries");
+    process.exit(1);
+  });
+  console.log("[info] All binaries built");
 
   // Cleanup - this is needed when running locally
   fs.rmSync("out/package.json");
@@ -331,4 +212,5 @@ async function downloadRipgrepForTarget(target, targetDir) {
   validateFilesPresent(pathsToVerify);
 
   console.log("[info] Done!");
+  process.exit(0);
 })();

binary/utils/bundle-binary.js

@@ -0,0 +1,103 @@
+/**
+ * @file Builds the binary for the specified target. It is also intended to run as a child process.
+ */
+
+const {
+  execCmdSync,
+  autodetectPlatformAndArch,
+} = require("../../scripts/util");
+const { downloadRipgrep } = require("./ripgrep");
+const { TARGET_TO_LANCEDB } = require("../utils/targets");
+const fs = require("fs");
+const {
+  downloadSqlite,
+} = require("../../extensions/vscode/scripts/download-copy-sqlite-esbuild");
+const { fork } = require("child_process");
+
+async function downloadNodeSqlite(target, targetDir) {
+  const [currentPlatform, currentArch] = autodetectPlatformAndArch();
+
+  // Download and unzip prebuilt sqlite3 binary for the target
+  console.log("[info] Downloading node-sqlite3");
+
+  await downloadSqlite(target, `${targetDir}/build.tar.gz`);
+
+  execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
+
+  // Copy to build directory for testing
+  try {
+    const [platform, arch] = target.split("-");
+    if (platform === currentPlatform && arch === currentArch) {
+      fs.copyFileSync(
+        `${targetDir}/build/Release/node_sqlite3.node`,
+        `build/node_sqlite3.node`,
+      );
+    }
+  } catch (error) {
+    console.log("[warn] Could not copy node_sqlite to build");
+    console.log(error);
+  }
+  fs.unlinkSync(`${targetDir}/build.tar.gz`);
+}
+
+/**
+ * @param {string} target the platform to build for
+ */
+async function bundleForBinary(target) {
+  const targetDir = `bin/${target}`;
+  fs.mkdirSync(targetDir, { recursive: true });
+  console.log(`[info] Building ${target}...`);
+  execCmdSync(
+    `npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
+  );
+
+  // copy @lancedb to bin folders
+  console.log("[info] Copying @lancedb files to bin");
+  fs.copyFileSync(
+    `node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
+    `${targetDir}/index.node`,
+  );
+
+  const downloadPromises = [];
+  downloadPromises.push(downloadRipgrep(target, targetDir));
+  downloadPromises.push(downloadNodeSqlite(target, targetDir));
+  await Promise.all(downloadPromises);
+
+  // Informs the `continue-binary` of where to look for node_sqlite3.node
+  // https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
+  fs.writeFileSync(`${targetDir}/package.json`, "");
+}
+
+process.on("message", (msg) => {
+  bundleForBinary(msg.payload.target)
+    .then(() => process.send({ done: true }))
+    .catch((error) => {
+      console.error(error); // show the error in the parent process
+      process.send({ error: true });
+    });
+});
+
+/**
+ * @param {string} target the platform to bundle for
+ */
+async function bundleBinary(target) {
+  const child = fork(__filename, { stdio: "inherit" });
+  child.send({
+    payload: {
+      target,
+    },
+  });
+  return new Promise((resolve, reject) => {
+    child.on("message", (msg) => {
+      if (msg.error) {
+        reject();
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+module.exports = {
+  bundleBinary,
+};

extensions/vscode/.gitignore

@@ -9,6 +9,7 @@ src/client
 exe
 bin
 assets
+tmp
 
 gui/**