fix: complex paths (#291)

* feat: update parser to handle backtick quoted strings, strings without ending quotes, and strings with quotes with content immediately following Signed-off-by: Chapman Pendery <[email protected]> * fix: unwrap strings + escapes during parsing & support wide chars Signed-off-by: Chapman Pendery <[email protected]> * fix: path completions Signed-off-by: Chapman Pendery <[email protected]> --------- Signed-off-by: Chapman Pendery <[email protected]>
microsoft · Oct 24, 2024 · 1815d0a · 1815d0a
1 parent e021a9b
commit 1815d0a
Show file tree

Hide file tree

Showing 12 changed files with 1,123 additions and 79 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -18,12 +18,12 @@
   ],
   "scripts": {
     "build": "tsc",
-    "dev": "node --loader ts-node/esm src/index.ts -V",
+    "dev": "node --import=tsx src/index.ts -V",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "test:e2e": "tui-test",
     "lint": "eslint src/ --ext .ts,.tsx && prettier src/ --check",
     "lint:fix": "eslint src/ --ext .ts,.tsx --fix && prettier src/ --write",
-    "debug": "node --inspect --loader ts-node/esm src/index.ts -V",
+    "debug": "node --inspect --import=tsx src/index.ts -V",
     "pre-commit": "lint-staged"
   },
   "repository": {
@@ -77,7 +77,7 @@
     "lint-staged": "^15.2.2",
     "prettier": "3.0.3",
     "ts-jest": "^29.1.1",
-    "ts-node": "^10.9.2",
+    "tsx": "^4.19.1",
     "typescript": "^5.2.2"
   },
   "lint-staged": {

diff --git a/src/runtime/alias.ts b/src/runtime/alias.ts
@@ -24,7 +24,7 @@ const loadBashAliases = async () => {
     .split("\n")
     .forEach((line) => {
       const [alias, ...commandSegments] = line.replace("alias ", "").replaceAll("'\\''", "'").split("=");
-      loadedAliases[alias] = parseCommand(commandSegments.join("=").slice(1, -1) + " ");
+      loadedAliases[alias] = parseCommand(commandSegments.join("=").slice(1, -1) + " ", Shell.Bash);
     });
 };
 
@@ -40,7 +40,7 @@ const loadZshAliases = async () => {
     .split("\n")
     .forEach((line) => {
       const [alias, ...commandSegments] = line.replaceAll("'\\''", "'").split("=");
-      loadedAliases[alias] = parseCommand(commandSegments.join("=").slice(1, -1) + " ");
+      loadedAliases[alias] = parseCommand(commandSegments.join("=").slice(1, -1) + " ", Shell.Zsh);
     });
 };
 

diff --git a/src/runtime/parser.ts b/src/runtime/parser.ts
@@ -1,33 +1,68 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import wcwidth from "wcwidth";
+import { Shell } from "../utils/shell.js";
+import { getShellWhitespaceEscapeChar } from "./utils.js";
+
 export type CommandToken = {
   token: string;
+  tokenLength: number;
   complete: boolean;
   isOption: boolean;
   isPersistent?: boolean;
   isPath?: boolean;
   isPathComplete?: boolean;
-  isQuoted?: boolean;
+  isQuoted?: boolean; // used for any token starting with quotes
+  isQuoteContinued?: boolean; // used for strings that are fully wrapped in quotes with content after the quotes
 };
 
 const cmdDelim = /(\|\|)|(&&)|(;)|(\|)/;
 const spaceRegex = /\s/;
 
-export const parseCommand = (command: string): CommandToken[] => {
+export const parseCommand = (command: string, shell: Shell): CommandToken[] => {
   const lastCommand = command.split(cmdDelim).at(-1)?.trimStart();
-  return lastCommand ? lex(lastCommand) : [];
+  const tokens = lastCommand ? lex(lastCommand, shell) : [];
+  return sanitizeTokens(tokens, shell);
+};
+
+const sanitizeTokens = (cmdTokens: CommandToken[], shell: Shell): CommandToken[] => unwrapQuotedTokens(unescapeSpaceTokens(cmdTokens, shell));
+
+// remove escapes around spaces
+const unescapeSpaceTokens = (cmdTokens: CommandToken[], shell: Shell): CommandToken[] => {
+  const escapeChar = getShellWhitespaceEscapeChar(shell);
+  return cmdTokens.map((cmdToken) => {
+    const { token, isQuoted } = cmdToken;
+    if (!isQuoted && token.includes(`${escapeChar} `)) {
+      return { ...cmdToken, token: token.replaceAll(`${escapeChar} `, " ") };
+    }
+    return cmdToken;
+  });
 };
 
-const lex = (command: string): CommandToken[] => {
+// need to unwrap tokens that are quoted with content after the quotes like `"hello"world`
+const unwrapQuotedTokens = (cmdTokens: CommandToken[]): CommandToken[] => {
+  return cmdTokens.map((cmdToken) => {
+    const { token, isQuoteContinued } = cmdToken;
+    if (isQuoteContinued) {
+      const quoteChar = token[0];
+      const unquotedToken = token.replaceAll(`\\${quoteChar}`, "\u001B").replaceAll(quoteChar, "").replaceAll("\u001B", quoteChar);
+      return { ...cmdToken, token: unquotedToken };
+    }
+    return cmdToken;
+  });
+};
+
+const lex = (command: string, shell: Shell): CommandToken[] => {
   const tokens: CommandToken[] = [];
-  let [readingQuotedString, readingFlag, readingCmd] = [false, false, false];
+  const escapeChar = getShellWhitespaceEscapeChar(shell);
+  let [readingQuotedString, readingQuoteContinuedString, readingFlag, readingCmd] = [false, false, false, false];
   let readingIdx = 0;
   let readingQuoteChar = "";
 
   [...command].forEach((char, idx) => {
-    const reading = readingQuotedString || readingFlag || readingCmd;
-    if (!reading && (char === `'` || char === `"`)) {
+    const reading = readingQuotedString || readingQuoteContinuedString || readingFlag || readingCmd;
+    if (!reading && (char === `'` || char === `"` || char == "`")) {
       [readingQuotedString, readingIdx, readingQuoteChar] = [true, idx, char];
       return;
     } else if (!reading && char === `-`) {
@@ -38,44 +73,71 @@ const lex = (command: string): CommandToken[] => {
       return;
     }
 
-    if (readingQuotedString && char === readingQuoteChar && command.at(idx - 1) !== "\\") {
+    if (readingQuotedString && char === readingQuoteChar && command.at(idx - 1) !== escapeChar && !spaceRegex.test(command.at(idx + 1) ?? " ")) {
+      readingQuotedString = false;
+      readingQuoteContinuedString = true;
+    } else if (readingQuotedString && char === readingQuoteChar && command.at(idx - 1) !== escapeChar) {
       readingQuotedString = false;
       const complete = idx + 1 < command.length && spaceRegex.test(command[idx + 1]);
       tokens.push({
         token: command.slice(readingIdx + 1, idx),
+        tokenLength: wcwidth(command.slice(readingIdx + 1, idx)) + 2, // +2 for both quotes
         complete,
         isOption: false,
         isQuoted: true,
       });
+    } else if (readingQuoteContinuedString && spaceRegex.test(char) && command.at(idx - 1) !== escapeChar) {
+      readingQuoteContinuedString = false;
+      tokens.push({
+        token: command.slice(readingIdx, idx),
+        tokenLength: wcwidth(command.slice(readingIdx, idx)),
+        complete: true,
+        isOption: false,
+        isQuoted: true,
+        isQuoteContinued: true,
+      });
     } else if ((readingFlag && spaceRegex.test(char)) || char === "=") {
       readingFlag = false;
       tokens.push({
         token: command.slice(readingIdx, idx),
+        tokenLength: wcwidth(command.slice(readingIdx, idx)),
         complete: true,
         isOption: true,
       });
-    } else if (readingCmd && spaceRegex.test(char) && command.at(idx - 1) !== "\\") {
+    } else if (readingCmd && spaceRegex.test(char) && command.at(idx - 1) !== escapeChar) {
       readingCmd = false;
       tokens.push({
         token: command.slice(readingIdx, idx),
+        tokenLength: wcwidth(command.slice(readingIdx, idx)),
         complete: true,
         isOption: false,
       });
     }
   });
 
-  const reading = readingQuotedString || readingFlag || readingCmd;
+  const reading = readingQuotedString || readingQuoteContinuedString || readingFlag || readingCmd;
   if (reading) {
     if (readingQuotedString) {
       tokens.push({
         token: command.slice(readingIdx + 1),
+        tokenLength: wcwidth(command.slice(readingIdx + 1)) + 1, // +1 for the leading quote
+        complete: false,
+        isOption: false,
+        isQuoted: true,
+      });
+    } else if (readingQuoteContinuedString) {
+      tokens.push({
+        token: command.slice(readingIdx),
+        tokenLength: wcwidth(command.slice(readingIdx)),
         complete: false,
         isOption: false,
         isQuoted: true,
+        isQuoteContinued: true,
       });
     } else {
       tokens.push({
         token: command.slice(readingIdx),
+        tokenLength: wcwidth(command.slice(readingIdx)),
         complete: false,
         isOption: readingFlag,
       });

diff --git a/src/runtime/runtime.ts b/src/runtime/runtime.ts
@@ -92,7 +92,7 @@ export const loadLocalSpecsSet = async () => {
 };
 
 export const getSuggestions = async (cmd: string, cwd: string, shell: Shell): Promise<SuggestionBlob | undefined> => {
-  let activeCmd = parseCommand(cmd);
+  let activeCmd = parseCommand(cmd, shell);
   const rootToken = activeCmd.at(0);
   if (activeCmd.length === 0 || !rootToken?.complete) {
     return;
@@ -110,13 +110,10 @@ export const getSuggestions = async (cmd: string, cwd: string, shell: Shell): Pr
     lastCommand.isPath = true;
     lastCommand.isPathComplete = pathyComplete;
   }
-  const result = await runSubcommand(activeCmd.slice(1), subcommand, resolvedCwd);
+  const result = await runSubcommand(activeCmd.slice(1), subcommand, resolvedCwd, shell);
   if (result == null) return;
 
-  let charactersToDrop = lastCommand?.complete ? 0 : lastCommand?.token.length ?? 0;
-  if (pathy) {
-    charactersToDrop = pathyComplete ? 0 : path.basename(lastCommand?.token ?? "").length;
-  }
+  const charactersToDrop = lastCommand?.complete ? 0 : lastCommand?.tokenLength;
   return { ...result, charactersToDrop };
 };
 
@@ -227,6 +224,7 @@ const runOption = async (
   option: Fig.Option,
   subcommand: Fig.Subcommand,
   cwd: string,
+  shell: Shell,
   persistentOptions: Fig.Option[],
   acceptedTokens: CommandToken[],
 ): Promise<SuggestionBlob | undefined> => {
@@ -237,12 +235,13 @@ const runOption = async (
   const isPersistent = persistentOptions.some((o) => (typeof o.name === "string" ? o.name === activeToken.token : o.name.includes(activeToken.token)));
   if ((option.args instanceof Array && option.args.length > 0) || option.args != null) {
     const args = option.args instanceof Array ? option.args : [option.args];
-    return runArg(tokens.slice(1), args, subcommand, cwd, persistentOptions, acceptedTokens.concat(activeToken), true, false);
+    return runArg(tokens.slice(1), args, subcommand, cwd, shell, persistentOptions, acceptedTokens.concat(activeToken), true, false);
   }
   return runSubcommand(
     tokens.slice(1),
     subcommand,
     cwd,
+    shell,
     persistentOptions,
     acceptedTokens.concat({
       ...activeToken,
@@ -256,38 +255,39 @@ const runArg = async (
   args: Fig.Arg[],
   subcommand: Fig.Subcommand,
   cwd: string,
+  shell: Shell,
   persistentOptions: Fig.Option[],
   acceptedTokens: CommandToken[],
   fromOption: boolean,
   fromVariadic: boolean,
 ): Promise<SuggestionBlob | undefined> => {
   if (args.length === 0) {
-    return runSubcommand(tokens, subcommand, cwd, persistentOptions, acceptedTokens, true, !fromOption);
+    return runSubcommand(tokens, subcommand, cwd, shell, persistentOptions, acceptedTokens, true, !fromOption);
   } else if (tokens.length === 0) {
-    return await getArgDrivenRecommendation(args, subcommand, persistentOptions, undefined, acceptedTokens, fromVariadic, cwd);
+    return await getArgDrivenRecommendation(args, subcommand, persistentOptions, undefined, acceptedTokens, fromVariadic, cwd, shell);
   } else if (!tokens.at(0)?.complete) {
-    return await getArgDrivenRecommendation(args, subcommand, persistentOptions, tokens[0], acceptedTokens, fromVariadic, cwd);
+    return await getArgDrivenRecommendation(args, subcommand, persistentOptions, tokens[0], acceptedTokens, fromVariadic, cwd, shell);
   }
 
   const activeToken = tokens[0];
   if (args.every((a) => a.isOptional)) {
     if (activeToken.isOption) {
       const option = getOption(activeToken, persistentOptions.concat(subcommand.options ?? []));
       if (option != null) {
-        return runOption(tokens, option, subcommand, cwd, persistentOptions, acceptedTokens);
+        return runOption(tokens, option, subcommand, cwd, shell, persistentOptions, acceptedTokens);
       }
       return;
     }
 
     const nextSubcommand = await genSubcommand(activeToken.token, subcommand);
     if (nextSubcommand != null) {
-      return runSubcommand(tokens.slice(1), nextSubcommand, cwd, persistentOptions, getPersistentTokens(acceptedTokens.concat(activeToken)));
+      return runSubcommand(tokens.slice(1), nextSubcommand, cwd, shell, persistentOptions, getPersistentTokens(acceptedTokens.concat(activeToken)));
     }
   }
 
   const activeArg = args[0];
   if (activeArg.isVariadic) {
-    return runArg(tokens.slice(1), args, subcommand, cwd, persistentOptions, acceptedTokens.concat(activeToken), fromOption, true);
+    return runArg(tokens.slice(1), args, subcommand, cwd, shell, persistentOptions, acceptedTokens.concat(activeToken), fromOption, true);
   } else if (activeArg.isCommand) {
     if (tokens.length <= 0) {
       return;
@@ -296,24 +296,25 @@ const runArg = async (
     if (spec == null) return;
     const subcommand = getSubcommand(spec);
     if (subcommand == null) return;
-    return runSubcommand(tokens.slice(1), subcommand, cwd);
+    return runSubcommand(tokens.slice(1), subcommand, cwd, shell);
   }
-  return runArg(tokens.slice(1), args.slice(1), subcommand, cwd, persistentOptions, acceptedTokens.concat(activeToken), fromOption, false);
+  return runArg(tokens.slice(1), args.slice(1), subcommand, cwd, shell, persistentOptions, acceptedTokens.concat(activeToken), fromOption, false);
 };
 
 const runSubcommand = async (
   tokens: CommandToken[],
   subcommand: Fig.Subcommand,
   cwd: string,
+  shell: Shell,
   persistentOptions: Fig.Option[] = [],
   acceptedTokens: CommandToken[] = [],
   argsDepleted = false,
   argsUsed = false,
 ): Promise<SuggestionBlob | undefined> => {
   if (tokens.length === 0) {
-    return getSubcommandDrivenRecommendation(subcommand, persistentOptions, undefined, argsDepleted, argsUsed, acceptedTokens, cwd);
+    return getSubcommandDrivenRecommendation(subcommand, persistentOptions, undefined, argsDepleted, argsUsed, acceptedTokens, cwd, shell);
   } else if (!tokens.at(0)?.complete) {
-    return getSubcommandDrivenRecommendation(subcommand, persistentOptions, tokens[0], argsDepleted, argsUsed, acceptedTokens, cwd);
+    return getSubcommandDrivenRecommendation(subcommand, persistentOptions, tokens[0], argsDepleted, argsUsed, acceptedTokens, cwd, shell);
   }
 
   const activeToken = tokens[0];
@@ -323,7 +324,7 @@ const runSubcommand = async (
   if (activeToken.isOption) {
     const option = getOption(activeToken, allOptions);
     if (option != null) {
-      return runOption(tokens, option, subcommand, cwd, persistentOptions, acceptedTokens);
+      return runOption(tokens, option, subcommand, cwd, shell, persistentOptions, acceptedTokens);
     }
     return;
   }
@@ -334,6 +335,7 @@ const runSubcommand = async (
       tokens.slice(1),
       nextSubcommand,
       cwd,
+      shell,
       getPersistentOptions(persistentOptions, subcommand.options),
       getPersistentTokens(acceptedTokens.concat(activeToken)),
     );
@@ -345,8 +347,8 @@ const runSubcommand = async (
 
   const args = getArgs(subcommand.args);
   if (args.length != 0) {
-    return runArg(tokens, args, subcommand, cwd, allOptions, acceptedTokens, false, false);
+    return runArg(tokens, args, subcommand, cwd, shell, allOptions, acceptedTokens, false, false);
   }
   // if the subcommand has no args specified, fallback to the subcommand and ignore this item
-  return runSubcommand(tokens.slice(1), subcommand, cwd, persistentOptions, acceptedTokens.concat(activeToken));
+  return runSubcommand(tokens.slice(1), subcommand, cwd, shell, persistentOptions, acceptedTokens.concat(activeToken));
 };