allow shortening arguments

verhovsky · verhovsky · commit 1d3d1fff2520 · 2021-08-01T02:56:32.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 .DS_Store
 Thumbs.db
 _gitignore
+
+curl_params.pickle
diff --git a/extract_curl_args.py b/extract_curl_args.py
@@ -0,0 +1,314 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import warnings
+import pickle
+import sys
+import subprocess
+import itertools
+from operator import itemgetter
+
+# TODO: make this and the location of the repo relative to script location
+# TODO: and command line args
+OUTPUT_FILE = Path("./resources/js/curl-to-go.js")
+if not OUTPUT_FILE.is_file():
+    sys.exit(
+        f"{OUTPUT_FILE} doesn't exist. You should run this script from curl-to-go/"
+    )
+
+PATH_TO_CURL_REPO = Path("../curl")
+if not PATH_TO_CURL_REPO.is_dir():
+    sys.exit(
+        f"{PATH_TO_CURL_REPO} needs to be a git repo with cURL's source code. "
+        "You can clone it with\n\n"
+        "git clone https://github.com/curl/curl ../curl"
+        # or modify the PATH_TO_CURL_REPO variable
+    )
+
+
+PARAMS_CACHE = Path("curl_params.pickle")
+SHOULD_CACHE = False
+
+JS_PARAMS_START = "BEGIN GENERATED CURL OPTIONS"
+JS_PARAMS_END = "END GENERATED CURL OPTIONS"
+
+# Noteworthy commits:
+#
+# The first commit in cURL's git repo (from 1999)
+# ae1912cb0d494b48d514d937826c9fe83ec96c4d
+# has args defined in main.c, then in
+# 49b79b76316248d5233d08006234933913faaa3b
+# the arg definitions were moved to ./src/tool_getparam.c
+#
+# Originally there were only two arg "types": TRUE/FALSE which signified
+# whether the option expected a value or was a boolean (respectively).
+# Then in
+IMPLICIT_NO_COMMIT = "5abfdc0140df0977b02506d16796f616158bfe88"
+# all boolean (i.e. FALSE "type") options got an implicit --no-OPTION.
+# Then TRUE/FALSE was changed to ARG_STRING/ARG_BOOL.
+# Then it was realized that not all options should have a --no-OPTION
+# counterpart, so a new ARG_NONE type was added for those in
+# 913c3c8f5476bd7bc4d8d00509396bd4b525b8fc
+
+# Other notes:
+#
+# cURL lets you not type the entire argument as long as it's unambiguous,
+# for example you can type --inse instead of --insecure
+#
+# cURL lets you omit the space after a short option that takes an arg
+# for example you can do "-dmydatahere" instead of "-o mydatahere"
+# it even interprets -aqomydata the same as -a -q -o mydatahere
+
+OPTS_START = "struct LongShort aliases[]= {"
+OPTS_END = "};"
+
+BOOL_ARG_TYPES = ["FALSE", "ARG_BOOL", "ARG_NONE"]
+STR_ARG_TYPES = ["TRUE", "ARG_STRING", "ARG_FILENAME"]
+ARG_TYPES = BOOL_ARG_TYPES + STR_ARG_TYPES
+
+OLD_ARG_TYPES_TO_NEW = {"TRUE": "ARG_STRING", "FALSE": "ARG_BOOL"}
+
+
+def flatten(l):
+    return list(itertools.chain.from_iterable(l))
+
+
+def commits_that_changed(filename):
+    lines = subprocess.run(
+        [
+            "git",
+            "log",
+            "--diff-filter=d",
+            "--date-order",
+            "--reverse",
+            "--format=%H %at",  # full commit hash and author date time stamp
+            "--date=iso-strict",
+            "--",
+            filename,
+        ],
+        cwd=PATH_TO_CURL_REPO,
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout
+    for line in lines.splitlines():
+        commit_hash, timestamp = line.strip().split()
+        yield commit_hash, int(timestamp)
+
+
+def extract_params(file_contents):
+    lines = iter(file_contents.splitlines())
+    params = {}
+    for line in lines:
+        if OPTS_START in line:
+            break
+    for line in lines:
+        line = line.strip()
+        if line.endswith(OPTS_END):
+            break
+        if not line.strip().startswith("{"):
+            continue
+
+        # main.c has comments on the same line
+        short, param, arg_type = line.split("/*")[0].strip().strip("{},").split(",")
+
+        short = short.strip().strip('"')
+        param = param.strip().strip('"')
+        arg_type = arg_type.strip()
+
+        if len(short) == 0:
+            raise ValueError(f"short form of {param} is the empty string")
+        if len(short) > 1:  # it's a placeholder value, not a real option
+            short = None
+
+        value = (short, arg_type)
+
+        if param in params and params[param] != value:
+            # if param not in ["login-options"]:  # I know about this one already.
+            warnings.warn(
+                f"{param!r} repeated with different values: {params[param]} vs. {value} "
+            )
+
+        if arg_type not in ARG_TYPES:
+            raise ValueError(f"unknown arg type: {arg_type}")
+
+        params[param] = value
+    return [(l, s, t) for l, (s, t) in params.items()]
+
+
+def explicit_params_over_time():
+    """yields the command line arguments that appear in the source code over time"""
+    for filename in ["./src/main.c", "./src/tool_getparam.c"]:
+        for commit_hash, timestamp in commits_that_changed(filename):
+            contents = subprocess.run(
+                ["git", "cat-file", "-p", f"{commit_hash}:{filename}"],
+                cwd=PATH_TO_CURL_REPO,
+                capture_output=True,
+                check=True,
+            ).stdout
+            try:
+                contents = contents.decode("utf-8")
+            except UnicodeDecodeError:
+                contents = contents.decode("latin1")
+            params = extract_params(contents)
+            if not params:
+                raise ValueError(
+                    f"Failed to extract params from {commit_hash}:{filename}"
+                )
+            yield commit_hash, timestamp, params
+
+
+def consecutive_runs(seq):
+    for k, g in itertools.groupby(enumerate(seq), lambda i_x: i_x[0] - i_x[1]):
+        result = list(map(itemgetter(1), g))
+        yield result[0], result[-1]
+
+
+# TODO: de-spaghettify
+def params_over_time(params_all_time):
+    hashes = [h for h, *_ in params_all_time]
+    to_idx = {c: i for i, c in enumerate(hashes)}
+    to_hash = {i: c for i, c in enumerate(hashes)}
+
+    params_all_time = [p for _, _, p in params_all_time]
+
+    long_args = {}
+    short_args = {}
+
+    # --metalink became a boolean
+    # 'metalink': [('string', [(14917, 15129)]), ('bool', [(15129, None)])],
+
+    for commit_idx, params in enumerate(params_all_time):
+        after_implicit_no = commit_idx >= to_idx[IMPLICIT_NO_COMMIT]
+        for long, short, arg_type in params:
+            if arg_type == "FALSE":
+                arg_type = "ARG_BOOL" if after_implicit_no else "ARG_NONE"
+
+            arg_type = {"TRUE": "ARG_STRING", "ARG_FILENAME": "ARG_STRING"}.get(
+                arg_type, arg_type
+            )
+
+            arg_type = arg_type.removeprefix("ARG_").lower()
+            long_args.setdefault((long, arg_type), []).append(commit_idx)
+
+            if short is not None:
+                short_args.setdefault((short, long), []).append(commit_idx)
+
+    # this option was removed more than once
+    # ('sasl-authzid', 'ARG_STRING', None)
+    # This one just had the short option changed and put back I think
+    # ('http1.0', '0')
+    new_long_args = {}
+    new_short_args = {}
+    for (long, arg_type), commits in long_args.items():
+        lifetimes = [
+            (
+                start if start > 0 else None,
+                (end + 1) if ((end + 1) < len(params_all_time)) else None,
+            )
+            for start, end in consecutive_runs(commits)
+        ]
+
+        arg_data = {"type": arg_type}
+        # one arg had a trailing space
+        name = long.removeprefix("disable-").strip()
+        if name != long:
+            arg_data["name"] = name
+        ends = [l[1] for l in lifetimes]
+        if None not in ends:
+            arg_data["deleted"] = to_hash[max(ends)]
+
+        new_long_args[long] = arg_data
+        if arg_type == "bool":
+            new_long_args["no-" + long] = {**arg_data, "name": name, "expand": False}
+        elif arg_type == "none":
+            new_long_args[long]["type"] = "bool"
+
+    for (short, long), commits in short_args.items():
+        lifetimes = [
+            (
+                start if start > 0 else None,
+                (end + 1) if ((end + 1) < len(params_all_time)) else None,
+            )
+            for start, end in consecutive_runs(commits)
+        ]
+
+        # -N is short for --no-buffer
+        if short == "N":
+            long = "no-" + long
+
+        arg_data = {"long": long}
+        ends = [l[1] for l in lifetimes]
+        deleted = None not in ends
+        if deleted:
+            arg_data["deleted"] = to_hash[max(ends)]
+
+        if short in new_short_args:
+            if new_short_args[short].get("deleted"):
+                new_short_args[short] = arg_data
+        else:
+            new_short_args[short] = arg_data
+
+    def as_js(d, var_name):
+        yield f"\tvar {var_name} = {{"
+        for top_key, opt_dict in d.items():
+            # TODO: rough
+            def quote(key):
+                return key if key.isalpha() else repr(key)
+
+            def val_to_js(val):
+                if isinstance(val, str):
+                    return repr(val)
+                if isinstance(val, bool):
+                    return str(val).lower()
+                raise TypeError(f"can't convert values of type {type(val)} to JS")
+
+            vals = [f"{quote(k)}: {val_to_js(v)}" for k, v in opt_dict.items()]
+
+            yield f"\t\t{top_key!r}: {{{', '.join(vals)}}},"
+        yield "\t}"
+
+    yield from as_js(new_long_args, "longOptions")
+    yield from as_js(new_short_args, "shortOptions")
+
+
+def on_git_master():
+    output = subprocess.run(
+        ["git", "status", "-uno"], cwd=PATH_TO_CURL_REPO, capture_output=True, text=True
+    ).stdout.strip()
+    return output.startswith("On branch master")
+
+
+if __name__ == "__main__":
+    if not on_git_master():
+        sys.exit("not on curl's git master")
+
+    # cache because this takes a few seconds
+    if SHOULD_CACHE:
+        if not PARAMS_CACHE.is_file():
+            params_all_time = list(explicit_params_over_time())
+            with open(PARAMS_CACHE, "wb") as f:
+                params_all_time = pickle.dump(params_all_time, f)
+        else:
+            with open(PARAMS_CACHE, "rb") as f:
+                params_all_time = pickle.load(f)
+    else:
+        params_all_time = list(explicit_params_over_time())
+
+    js_params_lines = params_over_time(params_all_time)
+
+    new_lines = []
+    with open(OUTPUT_FILE) as f:
+        for line in f:
+            new_lines.append(line)
+            if JS_PARAMS_START in line:
+                break
+        new_lines += [l+'\n' for l in js_params_lines]
+        for line in f:
+            if JS_PARAMS_END in line:
+                new_lines.append(line)
+                break
+        for line in f:
+            new_lines.append(line)
+    with open(OUTPUT_FILE, 'w', newline='\n') as f:
+        f.write(''.join(new_lines))
diff --git a/resources/js/common.js b/resources/js/common.js
@@ -92,7 +92,7 @@ $(function()
 
 	// Fill in examples
 	$('#example1').click(function() {
-		$('#input').val('curl canhazip.com').keyup();
+		$('#input').val('curl icanhazip.com').keyup();
 	});
 	$('#example2').click(function() {
 		$('#input').val('curl https://api.example.com/surprise \\\n     -u banana:coconuts \\\n     -d "sample data"').keyup();
diff --git a/resources/js/curl-to-go.js b/resources/js/curl-to-go.js
