Skip to content

Commit 78426cc

Browse files
sestinjsestinj
authored
Merge pull request #6022 from uinstinct/speedup-builds
chore: speedup builds for binary and prepackage
2 parents 5845e21 + a6dbe94 commit 78426cc

File tree

12 files changed

+688
-362
lines changed

12 files changed

+688
-362
lines changed

‎.vscode/tasks.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@
103103
"clear": true
104104
},
105105
"options": {
106+
"env": {
107+
"SKIP_INSTALLS": "true"
108+
},
106109
"cwd": "${workspaceFolder}/extensions/vscode"
107110
}
108111
},

‎binary/build.js

Lines changed: 26 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ const fs = require("fs");
33
const path = require("path");
44
const ncp = require("ncp").ncp;
55
const { rimrafSync } = require("rimraf");
6-
const {
7-
validateFilesPresent,
8-
execCmdSync,
9-
autodetectPlatformAndArch,
10-
} = require("../scripts/util");
11-
const { downloadRipgrep } = require("./utils/ripgrep");
6+
const { validateFilesPresent } = require("../scripts/util");
127
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
8+
const { fork } = require("child_process");
9+
const {
10+
installAndCopyNodeModules,
11+
} = require("../extensions/vscode/scripts/install-copy-nodemodule");
12+
const { bundleBinary } = require("./utils/bundle-binary");
1313

1414
const bin = path.join(__dirname, "bin");
1515
const out = path.join(__dirname, "out");
@@ -29,8 +29,6 @@ function cleanSlate() {
2929
const esbuildOutputFile = "out/index.js";
3030
let targets = [...ALL_TARGETS];
3131

32-
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
33-
3432
const assetBackups = [
3533
"node_modules/win-ca/lib/crypt32-ia32.node.bak",
3634
"node_modules/win-ca/lib/crypt32-x64.node.bak",
@@ -78,85 +76,6 @@ async function buildWithEsbuild() {
7876
});
7977
}
8078

81-
async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
82-
console.log(`Copying ${packageName} to ${toCopy}`);
83-
// This is a way to install only one package without npm trying to install all the dependencies
84-
// Create a temporary directory for installing the package
85-
const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
86-
const tempDir = path.join(
87-
__dirname,
88-
"tmp",
89-
`continue-node_modules-${adjustedName}`,
90-
);
91-
const currentDir = process.cwd();
92-
93-
// // Remove the dir we will be copying to
94-
// rimrafSync(`node_modules/${toCopy}`);
95-
96-
// // Ensure the temporary directory exists
97-
fs.mkdirSync(tempDir, { recursive: true });
98-
99-
try {
100-
// Move to the temporary directory
101-
process.chdir(tempDir);
102-
103-
// Initialize a new package.json and install the package
104-
execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
105-
106-
console.log(
107-
`Contents of: ${packageName}`,
108-
fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
109-
);
110-
111-
// Without this it seems the file isn't completely written to disk
112-
await new Promise((resolve) => setTimeout(resolve, 2000));
113-
114-
// Copy the installed package back to the current directory
115-
await new Promise((resolve, reject) => {
116-
ncp(
117-
path.join(tempDir, "node_modules", toCopy),
118-
path.join(currentDir, "node_modules", toCopy),
119-
{ dereference: true },
120-
(error) => {
121-
if (error) {
122-
console.error(
123-
`[error] Error copying ${packageName} package`,
124-
error,
125-
);
126-
reject(error);
127-
} else {
128-
resolve();
129-
}
130-
},
131-
);
132-
});
133-
} finally {
134-
// Clean up the temporary directory
135-
// rimrafSync(tempDir);
136-
137-
// Return to the original directory
138-
process.chdir(currentDir);
139-
}
140-
}
141-
142-
/**
143-
* Downloads and installs ripgrep binaries for the specified target
144-
*
145-
* @param {string} target - Target platform-arch (e.g., 'darwin-x64')
146-
* @param {string} targetDir - Directory to install ripgrep to
147-
* @returns {Promise<void>}
148-
*/
149-
async function downloadRipgrepForTarget(target, targetDir) {
150-
console.log(`[info] Downloading ripgrep for ${target}...`);
151-
try {
152-
await downloadRipgrep(target, targetDir);
153-
console.log(`[info] Successfully installed ripgrep for ${target}`);
154-
} catch (error) {
155-
console.error(`[error] Failed to download ripgrep for ${target}:`, error);
156-
throw error;
157-
}
158-
}
159-
16079
(async () => {
16180
if (esbuildOnly) {
16281
await buildWithEsbuild();
@@ -181,16 +100,21 @@ async function downloadRipgrepForTarget(target, targetDir) {
181100
),
182101
);
183102

184-
console.log("[info] Downloading prebuilt lancedb...");
103+
const copyLanceDBPromises = [];
185104
for (const target of targets) {
186-
if (TARGET_TO_LANCEDB[target]) {
187-
console.log(`[info] Downloading for ${target}...`);
188-
await installNodeModuleInTempDirAndCopyToCurrent(
189-
TARGET_TO_LANCEDB[target],
190-
"@lancedb",
191-
);
105+
if (!TARGET_TO_LANCEDB[target]) {
106+
continue;
192107
}
108+
console.log(`[info] Downloading for ${target}...`);
109+
copyLanceDBPromises.push(
110+
installAndCopyNodeModules(TARGET_TO_LANCEDB[target], "@lancedb"),
111+
);
193112
}
113+
await Promise.all(copyLanceDBPromises).catch(() => {
114+
console.error("[error] Failed to copy LanceDB");
115+
process.exit(1);
116+
});
117+
console.log("[info] Copied all LanceDB");
194118

195119
// tree-sitter-wasm
196120
const treeSitterWasmsDir = path.join(out, "tree-sitter-wasms");
@@ -252,59 +176,16 @@ async function downloadRipgrepForTarget(target, targetDir) {
252176
"out/llamaTokenizerWorkerPool.mjs",
253177
);
254178

179+
const buildBinaryPromises = [];
255180
console.log("[info] Building binaries with pkg...");
256181
for (const target of targets) {
257-
const targetDir = `bin/${target}`;
258-
fs.mkdirSync(targetDir, { recursive: true });
259-
console.log(`[info] Building ${target}...`);
260-
execCmdSync(
261-
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
262-
);
263-
264-
// Download and unzip prebuilt sqlite3 binary for the target
265-
console.log("[info] Downloading node-sqlite3");
266-
267-
const downloadUrl =
268-
// node-sqlite3 doesn't have a pre-built binary for win32-arm64
269-
target === "win32-arm64"
270-
? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
271-
: `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
272-
target
273-
}.tar.gz`;
274-
275-
execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
276-
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
277-
278-
// Copy to build directory for testing
279-
try {
280-
const [platform, arch] = target.split("-");
281-
if (platform === currentPlatform && arch === currentArch) {
282-
fs.copyFileSync(
283-
`${targetDir}/build/Release/node_sqlite3.node`,
284-
`build/node_sqlite3.node`,
285-
);
286-
}
287-
} catch (error) {
288-
console.log("[warn] Could not copy node_sqlite to build");
289-
console.log(error);
290-
}
291-
292-
fs.unlinkSync(`${targetDir}/build.tar.gz`);
293-
294-
// copy @lancedb to bin folders
295-
console.log("[info] Copying @lancedb files to bin");
296-
fs.copyFileSync(
297-
`node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
298-
`${targetDir}/index.node`,
299-
);
300-
301-
// Download and install ripgrep for the target
302-
await downloadRipgrepForTarget(target, targetDir);
303-
304-
// Informs the `continue-binary` of where to look for node_sqlite3.node
305-
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
306-
fs.writeFileSync(`${targetDir}/package.json`, "");
182+
buildBinaryPromises.push(bundleBinary(target));
307183
}
184+
await Promise.all(buildBinaryPromises).catch(() => {
185+
console.error("[error] Failed to build binaries");
186+
process.exit(1);
187+
});
188+
console.log("[info] All binaries built");
308189

309190
// Cleanup - this is needed when running locally
310191
fs.rmSync("out/package.json");
@@ -331,4 +212,5 @@ async function downloadRipgrepForTarget(target, targetDir) {
331212
validateFilesPresent(pathsToVerify);
332213

333214
console.log("[info] Done!");
215+
process.exit(0);
334216
})();

‎binary/utils/bundle-binary.js

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/**
2+
* @file Builds the binary for the specified target. It is also intended to run as a child process.
3+
*/
4+
5+
const {
6+
execCmdSync,
7+
autodetectPlatformAndArch,
8+
} = require("../../scripts/util");
9+
const { downloadRipgrep } = require("./ripgrep");
10+
const { TARGET_TO_LANCEDB } = require("../utils/targets");
11+
const fs = require("fs");
12+
const {
13+
downloadSqlite,
14+
} = require("../../extensions/vscode/scripts/download-copy-sqlite-esbuild");
15+
const { fork } = require("child_process");
16+
17+
async function downloadNodeSqlite(target, targetDir) {
18+
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
19+
20+
// Download and unzip prebuilt sqlite3 binary for the target
21+
console.log("[info] Downloading node-sqlite3");
22+
23+
await downloadSqlite(target, `${targetDir}/build.tar.gz`);
24+
25+
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
26+
27+
// Copy to build directory for testing
28+
try {
29+
const [platform, arch] = target.split("-");
30+
if (platform === currentPlatform && arch === currentArch) {
31+
fs.copyFileSync(
32+
`${targetDir}/build/Release/node_sqlite3.node`,
33+
`build/node_sqlite3.node`,
34+
);
35+
}
36+
} catch (error) {
37+
console.log("[warn] Could not copy node_sqlite to build");
38+
console.log(error);
39+
}
40+
fs.unlinkSync(`${targetDir}/build.tar.gz`);
41+
}
42+
43+
/**
44+
* @param {string} target the platform to build for
45+
*/
46+
async function bundleForBinary(target) {
47+
const targetDir = `bin/${target}`;
48+
fs.mkdirSync(targetDir, { recursive: true });
49+
console.log(`[info] Building ${target}...`);
50+
execCmdSync(
51+
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
52+
);
53+
54+
// copy @lancedb to bin folders
55+
console.log("[info] Copying @lancedb files to bin");
56+
fs.copyFileSync(
57+
`node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
58+
`${targetDir}/index.node`,
59+
);
60+
61+
const downloadPromises = [];
62+
downloadPromises.push(downloadRipgrep(target, targetDir));
63+
downloadPromises.push(downloadNodeSqlite(target, targetDir));
64+
await Promise.all(downloadPromises);
65+
66+
// Informs the `continue-binary` of where to look for node_sqlite3.node
67+
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
68+
fs.writeFileSync(`${targetDir}/package.json`, "");
69+
}
70+
71+
process.on("message", (msg) => {
72+
bundleForBinary(msg.payload.target)
73+
.then(() => process.send({ done: true }))
74+
.catch((error) => {
75+
console.error(error); // show the error in the parent process
76+
process.send({ error: true });
77+
});
78+
});
79+
80+
/**
81+
* @param {string} target the platform to bundle for
82+
*/
83+
async function bundleBinary(target) {
84+
const child = fork(__filename, { stdio: "inherit" });
85+
child.send({
86+
payload: {
87+
target,
88+
},
89+
});
90+
return new Promise((resolve, reject) => {
91+
child.on("message", (msg) => {
92+
if (msg.error) {
93+
reject();
94+
} else {
95+
resolve();
96+
}
97+
});
98+
});
99+
}
100+
101+
module.exports = {
102+
bundleBinary,
103+
};

‎extensions/vscode/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ src/client
99
exe
1010
bin
1111
assets
12+
tmp
1213

1314
gui/**
1415

0 commit comments

Comments
 (0)