feat apilinks.json generator (#153)

* feat apilinks.json generator

Closes #152

Signed-off-by: flakey5 <[email protected]>

Co-authored-by: Claudio W <[email protected]>

* Update getBaseGitHubUrl.mjs

---------

Co-authored-by: Claudio W <[email protected]>

Author: flakey5

README.md

@@ -39,6 +39,6 @@ Options:
   -o, --output <path>        Specify the relative or absolute output directory
   -v, --version <semver>     Specify the target version of Node.js, semver compliant (default: "v22.6.0")
   -c, --changelog <url>      Specify the path (file: or https://) to the CHANGELOG.md file (default: "https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md")
-  -t, --target [mode...]     Set the processing target modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify")
+  -t, --target [mode...]     Set the processing target modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify", "api-links")
   -h, --help                 display help for command
 ```

bin/cli.mjs

@@ -9,8 +9,8 @@ import { coerce } from 'semver';
 import { DOC_NODE_CHANGELOG_URL, DOC_NODE_VERSION } from '../src/constants.mjs';
 import createGenerator from '../src/generators.mjs';
 import generators from '../src/generators/index.mjs';
-import createLoader from '../src/loader.mjs';
-import createParser from '../src/parser.mjs';
+import createMarkdownLoader from '../src/loaders/markdown.mjs';
+import createMarkdownParser from '../src/parsers/markdown.mjs';
 import createNodeReleases from '../src/releases.mjs';
 
 const availableGenerators = Object.keys(generators);
@@ -68,8 +68,8 @@ program
  */
 const { input, output, target = [], version, changelog } = program.opts();
 
-const { loadFiles } = createLoader();
-const { parseApiDocs } = createParser();
+const { loadFiles } = createMarkdownLoader();
+const { parseApiDocs } = createMarkdownParser();
 
 const apiDocFiles = loadFiles(input);
 
@@ -83,6 +83,8 @@ const { getAllMajors } = createNodeReleases(changelog);
 await runGenerators({
   // A list of target modes for the API docs parser
   generators: target,
+  // Resolved `input` to be used
+  input: input,
   // Resolved `output` path to be used
   output: resolve(output),
   // Resolved SemVer of current Node.js version

package-lock.json

@@ -29,7 +29,9 @@
     "prettier": "3.4.2"
   },
   "dependencies": {
+    "acorn": "^8.14.0",
     "commander": "^13.1.0",
+    "estree-util-visit": "^2.0.0",
     "dedent": "^1.5.3",
     "github-slugger": "^2.0.0",
     "glob": "^11.0.1",

package.json

@@ -1,10 +1,20 @@
 'use strict';
 
-import availableGenerators from './generators/index.mjs';
+import publicGenerators from './generators/index.mjs';
+import astJs from './generators/ast-js/index.mjs';
+
+const availableGenerators = {
+  ...publicGenerators,
+  // This one is a little special since we don't want it to run unless we need
+  // it and we also don't want it to be publicly accessible through the CLI.
+  'ast-js': astJs,
+};
 
 /**
  * @typedef {{ ast: import('./generators/types.d.ts').GeneratorMetadata<ApiDocMetadataEntry, ApiDocMetadataEntry>}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
  * @typedef {import('./generators/types.d.ts').AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one
+ * @param markdownInput
+ * @param jsInput
  *
  * This method creates a system that allows you to register generators
  * and then execute them in a specific order, keeping track of the
@@ -18,17 +28,20 @@ import availableGenerators from './generators/index.mjs';
  * Generators can also write to files. These would usually be considered
  * the final generators in the chain.
  *
- * @param {ApiDocMetadataEntry} input The parsed API doc metadata entries
+ * @param {ApiDocMetadataEntry} markdownInput The parsed API doc metadata entries
+ * @param {Array<import('acorn').Program>} parsedJsFiles
  */
-const createGenerator = input => {
+const createGenerator = markdownInput => {
   /**
    * We store all the registered generators to be processed
    * within a Record, so we can access their results at any time whenever needed
    * (we store the Promises of the generator outputs)
    *
    * @type {{ [K in keyof AllGenerators]: ReturnType<AllGenerators[K]['generate']> }}
    */
-  const cachedGenerators = { ast: Promise.resolve(input) };
+  const cachedGenerators = {
+    ast: Promise.resolve(markdownInput),
+  };
 
   /**
    * Runs the Generator engine with the provided top-level input and the given generator options

src/generators.mjs

@@ -0,0 +1,4 @@
+'use strict';
+
+// Checks if a string is a valid name for a constructor in JavaScript
+export const CONSTRUCTOR_EXPRESSION = /^[A-Z]/;

src/generators/api-links/constants.mjs

@@ -0,0 +1,105 @@
+'use strict';
+
+import { basename, dirname, join } from 'node:path';
+import { writeFile } from 'node:fs/promises';
+import {
+  getBaseGitHubUrl,
+  getCurrentGitHash,
+} from './utils/getBaseGitHubUrl.mjs';
+import { extractExports } from './utils/extractExports.mjs';
+import { findDefinitions } from './utils/findDefinitions.mjs';
+import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs';
+
+/**
+ * This generator is responsible for mapping publicly accessible functions in
+ * Node.js to their source locations in the Node.js repository.
+ *
+ * This is a top-level generator. It takes in the raw AST tree of the JavaScript
+ * source files. It outputs a `apilinks.json` file into the specified output
+ * directory.
+ *
+ * @typedef {Array<JsProgram>} Input
+ *
+ * @type {import('../types.d.ts').GeneratorMetadata<Input, Record<string, string>>}
+ */
+export default {
+  name: 'api-links',
+
+  version: '1.0.0',
+
+  description:
+    'Creates a mapping of publicly accessible functions to their source locations in the Node.js repository.',
+
+  // Unlike the rest of the generators, this utilizes Javascript sources being
+  // passed into the input field rather than Markdown.
+  dependsOn: 'ast-js',
+
+  /**
+   * Generates the `apilinks.json` file.
+   *
+   * @param {Input} input
+   * @param {Partial<GeneratorOptions>} options
+   */
+  async generate(input, { output }) {
+    /**
+     * @type Record<string, string>
+     */
+    const definitions = {};
+
+    /**
+     * @type {string}
+     */
+    let baseGithubLink;
+
+    if (input.length > 0) {
+      const repositoryDirectory = dirname(input[0].path);
+
+      const repository = getBaseGitHubUrl(repositoryDirectory);
+
+      const tag = getCurrentGitHash(repositoryDirectory);
+
+      baseGithubLink = `${repository}/blob/${tag}`;
+    }
+
+    input.forEach(program => {
+      /**
+       * Mapping of definitions to their line number
+       * @type {Record<string, number>}
+       * @example { 'someclass.foo': 10 }
+       */
+      const nameToLineNumberMap = {};
+
+      // `http.js` -> `http`
+      const programBasename = basename(program.path, '.js');
+
+      const exports = extractExports(
+        program,
+        programBasename,
+        nameToLineNumberMap
+      );
+
+      findDefinitions(program, programBasename, nameToLineNumberMap, exports);
+
+      checkIndirectReferences(program, exports, nameToLineNumberMap);
+
+      const githubLink =
+        `${baseGithubLink}/lib/${programBasename}.js`.replaceAll('\\', '/');
+
+      // Add the exports we found in this program to our output
+      Object.keys(nameToLineNumberMap).forEach(key => {
+        const lineNumber = nameToLineNumberMap[key];
+
+        definitions[key] = `${githubLink}#L${lineNumber}`;
+      });
+    });
+
+    if (output) {
+      await writeFile(
+        join(output, 'apilinks.json'),
+        JSON.stringify(definitions)
+      );
+    }
+
+    return definitions;
+  },
+};

src/generators/api-links/index.mjs

@@ -0,0 +1,5 @@
+export interface ProgramExports {
+  ctors: Array<string>;
+  identifiers: Array<string>;
+  indirects: Record<string, string>;
+}

src/generators/api-links/types.d.ts

@@ -0,0 +1,24 @@
+import { visit } from 'estree-util-visit';
+
+/**
+ * @param {import('acorn').Program} program
+ * @param {import('../types.d.ts').ProgramExports} exports
+ * @param {Record<string, number>} nameToLineNumberMap
+ */
+export function checkIndirectReferences(program, exports, nameToLineNumberMap) {
+  if (Object.keys(exports.indirects).length === 0) {
+    return;
+  }
+
+  visit(program, node => {
+    if (!node.loc || node.type !== 'FunctionDeclaration') {
+      return;
+    }
+
+    const name = node.id.name;
+
+    if (name in exports.indirects) {
+      nameToLineNumberMap[exports.indirects[name]] = node.loc.start.line;
+    }
+  });
+}