fix: spec loading for sea

Signed-off-by: Chapman Pendery <cpendery@vt.edu>

Author: cpendery

README.md

@@ -136,6 +136,10 @@ If you are using a [NerdFont](https://www.nerdfonts.com/) patched font, you can
 useNerdFont = true
 ```
 
+## Unsupported Specs
+
+Specs for the `az`, `gcloud`, & `aws` CLIs are not supported in inshellisense due to their large size.
+
 ## Contributing
 
 This project welcomes contributions and suggestions. Most contributions require you to agree to a

package.json

@@ -21,11 +21,11 @@
     "build": "tsc",
     "package": "tsx ./scripts/pkg.ts",
     "package:base": "tsx ./scripts/pkg-base.ts",
-    "dev": "node --import=tsx src/index.ts -V",
+    "dev": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --import=tsx src/index.ts -V",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "lint": "eslint src/ --ext .ts,.tsx && prettier src/ --check",
     "lint:fix": "eslint src/ --ext .ts,.tsx --fix && prettier src/ --write",
-    "debug": "node --inspect --import=tsx src/index.ts -V"
+    "debug": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --inspect --import=tsx src/index.ts -V"
   },
   "repository": {
     "type": "git",

scripts/pkg.ts

@@ -24,6 +24,7 @@ const PLATFORM_ARCH = `${process.platform}-${process.arch}`;
 const BINARY_NAME = process.platform === "win32" ? `inshellisense-${PLATFORM_ARCH}.exe` : `inshellisense-${PLATFORM_ARCH}`;
 const BINARY_PATH = path.join(PKG_DIR, BINARY_NAME);
 const NODE_VERSION = "22.21.1";
+const ASSET_PATH_SEP = "____";
 
 /** SHA-256 checksums for Node.js binaries from https://nodejs.org/dist/vX.X.X/SHASUMS256.txt */
 const NODE_SHASUMS: Record<string, string> = {
@@ -36,6 +37,7 @@ const NODE_SHASUMS: Record<string, string> = {
 };
 
 const getNodePtyPath = (): string => path.dirname(require.resolve("node-pty"));
+const getAutocompletePath = (): string => path.dirname(require.resolve("@withfig/autocomplete"));
 
 const getVersion = (): string => {
   const packageJson = JSON.parse(fs.readFileSync("package.json", "utf-8"));
@@ -189,20 +191,41 @@ const copyNodePtyNatives = (): void => {
   console.log(`Copied natives: ${srcDir} -> ${destDir}`);
 };
 
+const copyAutocompleteSpecs = (): void => {
+  const srcDir = path.join(getAutocompletePath());
+  if (!fs.existsSync(srcDir)) return;
+
+  const destDir = path.join(PKG_DIR, "specs");
+  fs.mkdirSync(path.dirname(destDir), { recursive: true });
+  fs.cpSync(srcDir, destDir, { recursive: true });
+  console.log(`Copied specs: ${srcDir} -> ${destDir}`);
+};
+
 const generateSeaConfig = (): void => {
   const shellAssets = fs.readdirSync("shell");
   const prebuildsDir = path.join(PKG_DIR, "prebuilds", PLATFORM_ARCH);
+  const specsDir = path.join(PKG_DIR, "specs");
 
   const nativeAssets = fs
     .readdirSync(prebuildsDir, { recursive: true })
     .map(String)
     .filter((file) => !file.endsWith(".pdb"))
     .filter((file) => fs.statSync(path.join(prebuildsDir, file)).isFile());
 
+  const specAssets = fs
+    .readdirSync(specsDir, { recursive: true })
+    .map(String)
+    .filter((file) => file.endsWith(".js"))
+    .filter((file) => fs.statSync(path.join(specsDir, file)).isFile())
+    .filter((file) => !["gcloud", "az", "aws"].some((name) => file.startsWith(name + path.sep)));
+  
   const assets: Record<string, string> = {};
   shellAssets.forEach((file) => (assets[file] = `shell/${file}`));
   nativeAssets.forEach((file) => {
-    assets[path.basename(file)] = `pkg/prebuilds/${PLATFORM_ARCH}/${file}`.replace(path.sep, "/");
+    assets[file.replaceAll(path.sep, ASSET_PATH_SEP)] = `pkg/prebuilds/${PLATFORM_ARCH}/${file}`.replaceAll(path.sep, "/");
+  });
+  specAssets.forEach((file) => {
+    assets[file.replaceAll(path.sep, ASSET_PATH_SEP)] = `pkg/specs/${file}`.replaceAll(path.sep, "/");
   });
 
   const seaConfig = {
@@ -269,6 +292,7 @@ const main = async (): Promise<void> => {
   await buildBundle();
   await applyBundlePatches();
   copyNodePtyNatives();
+  copyAutocompleteSpecs();
   await copyNodeExecutable();
   generateSeaConfig();
   generateSeaBlob();

src/commands/init.ts

@@ -3,16 +3,14 @@
 
 import { Command } from "commander";
 import { createShellConfigs, initSupportedShells as shells, getShellSourceCommand, Shell } from "../utils/shell.js";
-import { permissionNativeModules, unpackNativeModules, unpackShellFiles } from "../utils/node.js";
+import {unpackResources } from "../utils/node.js";
 import { render } from "../ui/ui-init.js";
 
 const supportedShells = shells.join(", ");
 
 const action = (program: Command) => async (shell: string | undefined) => {
   await createShellConfigs();
-  await unpackNativeModules();
-  await permissionNativeModules();
-  await unpackShellFiles();
+  unpackResources();
 
   if (shell == null) {
     await render();

src/runtime/runtime.ts

@@ -1,12 +1,13 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import speclist, {
-  diffVersionedCompletions as versionedSpeclist,
+import figSpecList, {
+  diffVersionedCompletions as figVersionedSpeclist,
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-ignore
 } from "@withfig/autocomplete/build/index.js";
 import path from "node:path";
+import { pathToFileURL } from "node:url";
 import { parseCommand, CommandToken } from "./parser.js";
 import { getArgDrivenRecommendation, getSubcommandDrivenRecommendation, SuggestionIcons } from "./suggestion.js";
 import { Suggestion, SuggestionBlob } from "./model.js";
@@ -15,9 +16,14 @@ import { Shell } from "../utils/shell.js";
 import { aliasExpand, getAliasNames } from "./alias.js";
 import { getConfig } from "../utils/config.js";
 import log from "../utils/log.js";
+import { specResourcesPath } from "../utils/constants.js";
+
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- recursive type, setting as any
 const specSet: any = {};
+const ignoredSpecs = ["gcloud", "az", "aws"];
+const speclist = figSpecList.filter((spec: string) => !ignoredSpecs.some((name) => spec.startsWith(name + "/")));
+const versionedSpeclist = figVersionedSpeclist.filter((spec: string) => !ignoredSpecs.some((name) => spec.startsWith(name)));
 
 function loadSpecsSet(speclist: string[], versionedSpeclist: string[], specsPath: string) {
   speclist.forEach((s) => {
@@ -29,7 +35,7 @@ function loadSpecsSet(speclist: string[], versionedSpeclist: string[], specsPath
       }
       if (idx === specRoutes.length - 1) {
         const prefix = versionedSpeclist.includes(s) ? "/index.js" : `.js`;
-        activeSet[route] = `${specsPath}/${s}${prefix}`;
+        activeSet[route] = `${specsPath}${path.sep}${s}${prefix}`;
       } else {
         activeSet[route] = activeSet[route] || {};
         activeSet = activeSet[route];
@@ -38,7 +44,7 @@ function loadSpecsSet(speclist: string[], versionedSpeclist: string[], specsPath
   });
 }
 
-loadSpecsSet(speclist as string[], versionedSpeclist, `@withfig/autocomplete/build`);
+loadSpecsSet(speclist as string[], versionedSpeclist, specResourcesPath);
 
 const loadedSpecs: { [key: string]: Fig.Spec } = {};
 
@@ -52,15 +58,19 @@ const loadSpec = async (cmd: CommandToken[]): Promise<Fig.Spec | undefined> => {
     return loadedSpecs[rootToken.token];
   }
   if (specSet[rootToken.token]) {
-    const spec = (await import(specSet[rootToken.token])).default;
+    const specPath = specSet[rootToken.token];
+    const importPath = path.isAbsolute(specPath) ? pathToFileURL(specPath).href : specPath;
+    const spec = (await import(importPath)).default;
     loadedSpecs[rootToken.token] = spec;
     return spec;
   }
 };
 
 // this load spec function should only be used for `loadSpec` on the fly as it is cacheless
 const lazyLoadSpec = async (key: string): Promise<Fig.Spec | undefined> => {
-  return (await import(`@withfig/autocomplete/build/${key}.js`)).default;
+  const specPath = path.join(specResourcesPath, `${key}.js`);
+  const importPath = path.isAbsolute(specPath) ? pathToFileURL(specPath).href : specPath;
+  return (await import(importPath)).default;
 };
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars -- will be implemented in below TODO
@@ -71,16 +81,18 @@ const lazyLoadSpecLocation = async (location: Fig.SpecLocation): Promise<Fig.Spe
 export const loadLocalSpecsSet = async () => {
   const specsPath = getConfig().specs.path;
   await Promise.allSettled(
-    specsPath.map((specPath) =>
-      import(path.join(specPath, "index.js"))
+    specsPath.map((specPath) => {
+      const indexPath = path.join(specPath, "index.js");
+      const importPath = path.isAbsolute(indexPath) ? pathToFileURL(indexPath).href : indexPath;
+      return import(importPath)
         .then((res) => {
           const { default: speclist, diffVersionedCompletions: versionedSpeclist } = res;
           loadSpecsSet(speclist, versionedSpeclist, specPath);
         })
         .catch((e) => {
           log.debug({ msg: `no local specs imported from '${specPath}', this will not break the current session`, e: (e as Error).message, specPath });
-        }),
-    ),
+        });
+    }),
   );
 };
 

src/ui/ui-reinit.ts

@@ -2,21 +2,21 @@
 // Licensed under the MIT License.
 
 import chalk from "chalk";
-import { permissionNativeModules, unpackNativeModules, unpackShellFiles } from "../utils/node.js";
+import { unpackResources } from "../utils/node.js";
 import { createShellConfigs } from "../utils/shell.js";
-import { shellResourcesPath, nativeResourcesPath, loggingResourcesPath, initResourcesPath } from "../utils/constants.js";
+import { shellResourcesPath, nativeResourcesPath, loggingResourcesPath, initResourcesPath, specResourcesPath } from "../utils/constants.js";
 import fs from "node:fs";
 
 export const render = async () => {
   fs.rmSync(shellResourcesPath, { recursive: true, force: true });
   fs.rmSync(nativeResourcesPath, { recursive: true, force: true });
   fs.rmSync(loggingResourcesPath, { recursive: true, force: true });
   fs.rmSync(initResourcesPath, { recursive: true, force: true });
+  fs.rmSync(specResourcesPath, { recursive: true, force: true });
   process.stdout.write(chalk.green("✓") + " removed old inshellisense resources \n");
 
   await createShellConfigs();
-  await unpackNativeModules();
-  await permissionNativeModules();
-  await unpackShellFiles();
+  await unpackResources();
+  
   process.stdout.write(chalk.green("✓") + " successfully installed inshellisense \n");
 };

src/utils/constants.ts

@@ -9,4 +9,5 @@ export const allResourcesPath = path.join(os.homedir(), inshellisenseFolderName)
 export const loggingResourcesPath = path.join(os.homedir(), inshellisenseFolderName, "log");
 export const nativeResourcesPath = path.join(os.homedir(), inshellisenseFolderName, "native");
 export const shellResourcesPath = path.join(os.homedir(), inshellisenseFolderName, "shell");
+export const specResourcesPath = path.join(os.homedir(), inshellisenseFolderName, "spec");
 export const initResourcesPath = path.join(os.homedir(), inshellisenseFolderName, "init");

src/utils/node.ts

@@ -5,17 +5,58 @@ import path from "node:path";
 import sea from "node:sea";
 import fsAsync from "node:fs/promises";
 import fs from "node:fs";
-import { nativeResourcesPath, shellResourcesPath } from "./constants.js";
+import { nativeResourcesPath, shellResourcesPath, specResourcesPath } from "./constants.js";
 
-export const unpackNativeModules = async (): Promise<void> => {
-  if (!sea.isSea()) return;
+const ASSET_PATH_SEP = "____";
+
+type AssetType = "native" | "shell" | "spec";
+
+const getAssetKeys = (assetType: AssetType) => {
+  if (!sea.isSea()) return [];
+
+  const allKeys = sea.getAssetKeys();
+  switch (assetType) {
+    case "native":
+      return allKeys.filter((key) => !key.includes("shellIntegration") && !key.includes("preexec") && !key.endsWith(".js"));
+    case "shell":
+      return allKeys.filter((key) => key.includes("shellIntegration") || key.includes("preexec"));
+    case "spec":
+      return allKeys.filter((key) => key.endsWith(".js"));
+    default:
+      return [];
+  }
+};
+
+const getAssetFolder = (assetType: AssetType) => {
+  switch (assetType) {
+    case "native":
+      return nativeResourcesPath;
+    case "shell":
+      return shellResourcesPath;
+    case "spec":
+      return specResourcesPath;
+    default:
+      return "";
+  }
+};
 
-  const assetKeys = sea.getAssetKeys().filter((key) => !key.includes("shellIntegration") && !key.includes("preexec"));
+const copyFiles = async(assetType: AssetType, files: string[], sourceFolder: string) => {
+  await Promise.all(
+      files.map(async (file) => {
+        const sourcePath = path.join(sourceFolder, file);
+        const destPath = path.join(getAssetFolder(assetType), file);
+        if (fs.existsSync(destPath)) return;
+        await fsAsync.mkdir(path.dirname(destPath), { recursive: true });
+        await fsAsync.copyFile(sourcePath, destPath);
+      }),
+    );
+};
 
+const copyAssets = async (assetType: AssetType) => {
   await Promise.all(
-    assetKeys.map(async (assetKey) => {
-      const assetPath = assetKey == "conpty.dll" || assetKey == "OpenConsole.exe" ? path.join("conpty", assetKey) : assetKey;
-      const outputPath = path.join(nativeResourcesPath, assetPath);
+    getAssetKeys(assetType).map(async (assetKey) => {
+      const assetPath = assetKey.replaceAll(ASSET_PATH_SEP, path.sep);
+      const outputPath = path.join(getAssetFolder(assetType), assetPath);
       if (fs.existsSync(outputPath)) return;
       const assetBlob = sea.getRawAsset(assetKey);
       await fsAsync.mkdir(path.dirname(outputPath), { recursive: true });
@@ -24,7 +65,13 @@ export const unpackNativeModules = async (): Promise<void> => {
   );
 };
 
-export const permissionNativeModules = async (): Promise<void> => {
+const unpackNativeModules = async (): Promise<void> => {
+  if (!sea.isSea()) return;
+
+  await copyAssets("native");
+};
+
+const permissionNativeModules = async (): Promise<void> => {
   if (!sea.isSea()) return;
 
   const spawnHelper = path.join(nativeResourcesPath, "spawn-helper");
@@ -33,31 +80,36 @@ export const permissionNativeModules = async (): Promise<void> => {
   }
 };
 
-export const unpackShellFiles = async (): Promise<void> => {
+const unpackSpecs = async (): Promise<void> => {
+  if (!sea.isSea()) {
+    const autocompleteSpecFolderPath = path.join(process.cwd(), "node_modules", "@withfig", "autocomplete", "build");
+    const entries = await fsAsync.readdir(autocompleteSpecFolderPath, { recursive: true });
+    const files = entries.filter((f) => {
+      const fullPath = path.join(autocompleteSpecFolderPath, f.toString());
+      return fs.statSync(fullPath).isFile();
+    }).map((f) => f.toString());
+
+    await copyFiles("spec", files, autocompleteSpecFolderPath);
+  }
+  else {
+    await copyAssets("spec");
+  }
+};
+
+const unpackShellFiles = async (): Promise<void> => {
   if (!sea.isSea()) {
     const shellFolderPath = path.join(process.cwd(), "shell");
     const files = (await fsAsync.readdir(shellFolderPath)).map((f) => path.basename(f));
 
-    await Promise.all(
-      files.map(async (file) => {
-        const sourcePath = path.join(shellFolderPath, file);
-        const destPath = path.join(shellResourcesPath, file);
-        if (fs.existsSync(destPath)) return;
-        await fsAsync.mkdir(path.dirname(destPath), { recursive: true });
-        await fsAsync.copyFile(sourcePath, destPath);
-      }),
-    );
+    await copyFiles("shell", files, shellFolderPath);
   } else {
-    const assetKeys = sea.getAssetKeys().filter((key) => key.includes("shellIntegration") || key.includes("preexec"));
-
-    await Promise.all(
-      assetKeys.map(async (assetKey) => {
-        const outputPath = path.join(shellResourcesPath, assetKey);
-        if (fs.existsSync(outputPath)) return;
-        const assetBlob = sea.getRawAsset(assetKey);
-        await fsAsync.mkdir(path.dirname(outputPath), { recursive: true });
-        await fsAsync.writeFile(outputPath, Buffer.from(assetBlob));
-      }),
-    );
+    await copyAssets("shell");
   }
 };
+
+export const unpackResources = async (): Promise<void> => {
+  await unpackNativeModules();
+  await permissionNativeModules();
+  await unpackShellFiles();
+  await unpackSpecs();
+}