Merge pull request #38 from PolymerLabs/transform-scaffold

Scaffolding for transform mode

Author: aomarks

config.schema.json

@@ -65,6 +65,22 @@
             ],
             "type": "object"
         },
+        "TransformOutputConfig": {
+            "additionalProperties": false,
+            "description": "Configuration specific to the `transform` output mode.",
+            "properties": {
+                "mode": {
+                    "enum": [
+                        "transform"
+                    ],
+                    "type": "string"
+                }
+            },
+            "required": [
+                "mode"
+            ],
+            "type": "object"
+        },
         "XlbConfig": {
             "additionalProperties": false,
             "description": "Parse an XLB XML file. These files contain translations organized using the\nsame message names that we originally requested.\nConfiguration for XLB interchange format.",
@@ -130,7 +146,14 @@
             "description": "Localization interchange format and configuration specific to that format."
         },
         "output": {
-            "$ref": "#/definitions/RuntimeOutputConfig",
+            "anyOf": [
+                {
+                    "$ref": "#/definitions/RuntimeOutputConfig"
+                },
+                {
+                    "$ref": "#/definitions/TransformOutputConfig"
+                }
+            ],
             "description": "Set and configure the output mode."
         },
         "patches": {

package-lock.json

@@ -32,6 +32,7 @@
     "fs-extra": "^9.0.0",
     "glob": "^7.1.6",
     "jsonschema": "^1.2.6",
+    "lit-html": "^1.2.1",
     "minimist": "^1.2.5",
     "parse5": "^6.0.0",
     "source-map-support": "^0.5.19",
@@ -46,6 +47,7 @@
     "@types/minimist": "^1.2.0",
     "@types/node": "^14.0.1",
     "@types/parse5": "^5.0.2",
+    "@types/prettier": "^2.0.1",
     "@types/xmldom": "^0.1.29",
     "@typescript-eslint/eslint-plugin": "^3.3.0",
     "@typescript-eslint/parser": "^3.3.0",
@@ -54,7 +56,6 @@
     "diff": "^4.0.2",
     "dir-compare": "^2.3.0",
     "eslint": "^7.0.0",
-    "lit-html": "^1.2.1",
     "prettier": "^2.0.5",
     "rimraf": "^3.0.2",
     "typescript-json-schema": "^0.42.0"

package.json

@@ -15,6 +15,7 @@ import * as minimist from 'minimist';
 import {programFromTsConfig, printDiagnostics} from './typescript';
 import {extractMessagesFromProgram} from './program-analysis';
 import {runtimeOutput} from './outputters/runtime';
+import {transformOutput} from './outputters/transform';
 import {makeFormatter} from './formatters';
 import {ProgramMessage, Message} from './messages';
 import {KnownError, throwUnreachable} from './error';
@@ -94,9 +95,11 @@ async function runAndThrow(config: Config) {
 
   if (config.output.mode === 'runtime') {
     runtimeOutput(messages, translationMap, config, config.output);
+  } else if (config.output.mode === 'transform') {
+    transformOutput(translationMap, config, program);
   } else {
     throwUnreachable(
-      config.output.mode,
+      config.output,
       `Internal error: unknown output mode ${
         (config.output as typeof config.output).mode
       }`

src/cli.ts

@@ -16,6 +16,7 @@ import {Locale} from './locales';
 import {KnownError} from './error';
 import {FormatConfig} from './formatters';
 import {RuntimeOutputConfig} from './outputters/runtime';
+import {TransformOutputConfig} from './outputters/transform';
 
 interface ConfigFile {
   /**
@@ -49,7 +50,7 @@ interface ConfigFile {
   /**
    * Set and configure the output mode.
    */
-  output: RuntimeOutputConfig;
+  output: RuntimeOutputConfig | TransformOutputConfig;
 
   /**
    * Optional string substitutions to apply to specific locale messages. Useful

src/config.ts

@@ -16,7 +16,13 @@ import {Config} from '../config';
 import {Locale} from '../locales';
 import {Formatter} from './index';
 import {KnownError} from '../error';
-import {Bundle, Message, ProgramMessage, Placeholder} from '../messages';
+import {
+  Bundle,
+  Message,
+  ProgramMessage,
+  Placeholder,
+  makeMessageIdMap,
+} from '../messages';
 import {
   getOneElementByTagNameOrThrow,
   getNonEmptyAttributeOrThrow,
@@ -180,10 +186,7 @@ export class XliffFormatter implements Formatter {
     targetLocale: Locale,
     targetMessages: Message[]
   ): string {
-    const translationsByName = new Map<string, Message>();
-    for (const message of targetMessages) {
-      translationsByName.set(message.name, message);
-    }
+    const translationsByName = makeMessageIdMap(targetMessages);
 
     const doc = new xmldom.DOMImplementation().createDocument('', '', null);
     const indent = (node: Element | Document, level = 0) =>

src/formatters/xliff.ts

@@ -91,3 +91,16 @@ export interface Placeholder {
   // END_BOLD, and also allow a way to define them in the template (e.g. a
   // "placeholder-name" element attribute or similar).
 }
+
+/**
+ * Given an array of messages, return a new map from message ID to message.
+ */
+export function makeMessageIdMap<T extends Message>(
+  messages: T[]
+): Map<string, T> {
+  const map = new Map<string, T>();
+  for (const msg of messages) {
+    map.set(msg.name, msg);
+  }
+  return map;
+}

src/messages.ts

@@ -0,0 +1,91 @@
+/**
+ * @license
+ * Copyright (c) 2020 The Polymer Project Authors. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt The complete set of authors may be found
+ * at http://polymer.github.io/AUTHORS.txt The complete set of contributors may
+ * be found at http://polymer.github.io/CONTRIBUTORS.txt Code distributed by
+ * Google as part of the polymer project is also subject to an additional IP
+ * rights grant found at http://polymer.github.io/PATENTS.txt
+ */
+
+import {Message} from '../messages';
+import {Locale} from '../locales';
+import {Config} from '../config';
+import * as ts from 'typescript';
+import * as pathLib from 'path';
+
+/**
+ * Configuration specific to the `transform` output mode.
+ */
+export interface TransformOutputConfig {
+  mode: 'transform';
+}
+
+/**
+ * Compile and emit the given TypeScript program using the lit-localize
+ * transformer.
+ */
+export function transformOutput(
+  translationsByLocale: Map<Locale, Message[]>,
+  config: Config,
+  program: ts.Program
+) {
+  // TODO(aomarks) It doesn't seem that it's possible for a TypeScript
+  // transformer to emit a new file, so we just have to emit for each locale.
+  // Need to do some more investigation into the best way to integrate this
+  // transformation into a real project so that the user can still use --watch
+  // and other tsc flags. It would also be nice to support the language server,
+  // so that diagnostics will show up immediately in the editor.
+  const opts = program.getCompilerOptions();
+  const outRoot = opts.outDir || '.';
+  for (const locale of [config.sourceLocale, ...config.targetLocales]) {
+    let translations;
+    if (locale !== config.sourceLocale) {
+      translations = new Map<string, Message>();
+      for (const message of translationsByLocale.get(locale) || []) {
+        translations.set(message.name, message);
+      }
+    }
+    opts.outDir = pathLib.join(outRoot, '/', locale);
+    program.emit(undefined, undefined, undefined, undefined, {
+      before: [litLocalizeTransform(translations)],
+    });
+  }
+}
+
+/**
+ * Return a TypeScript TransformerFactory for the lit-localize transformer.
+ */
+export function litLocalizeTransform(
+  translations: Map<string, Message> | undefined
+): ts.TransformerFactory<ts.SourceFile> {
+  return (context) => {
+    const transformer = new Transformer(context, translations);
+    return (file) => ts.visitNode(file, transformer.boundVisitNode);
+  };
+}
+
+/**
+ * Implementation of the lit-localize TypeScript transformer.
+ */
+class Transformer {
+  private context: ts.TransformationContext;
+  boundVisitNode = this.visitNode.bind(this);
+
+  constructor(
+    context: ts.TransformationContext,
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    _translations: Map<string, Message> | undefined
+  ) {
+    this.context = context;
+  }
+
+  /**
+   * Top-level delegating visitor for all nodes.
+   */
+  visitNode(node: ts.Node): ts.VisitResult<ts.Node> {
+    // TODO(aomarks) The transformer!
+    return ts.visitEachChild(node, this.boundVisitNode, this.context);
+  }
+}

src/outputters/transform.ts

@@ -70,7 +70,7 @@ export function e2eGoldensTest(
       '--no-install',
       'prettier',
       '--write',
-      `${outputDir}/**/*.ts`,
+      `${outputDir}/**/*.{ts,js}`,
     ]);
 
     if (process.env.UPDATE_TEST_GOLDENS) {

src/tests/e2e-goldens-test.ts

@@ -0,0 +1,14 @@
+/**
+ * @license
+ * Copyright (c) 2020 The Polymer Project Authors. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt The complete set of authors may be found
+ * at http://polymer.github.io/AUTHORS.txt The complete set of contributors may
+ * be found at http://polymer.github.io/CONTRIBUTORS.txt Code distributed by
+ * Google as part of the polymer project is also subject to an additional IP
+ * rights grant found at http://polymer.github.io/PATENTS.txt
+ */
+
+import {e2eGoldensTest} from './e2e-goldens-test';
+
+e2eGoldensTest('transform', ['--config=lit-localize.json']);

src/tests/transform.test.ts

@@ -0,0 +1,84 @@
+/**
+ * @license
+ * Copyright (c) 2020 The Polymer Project Authors. All rights reserved.
+ * This code may only be used under the BSD style license found at
+ * http://polymer.github.io/LICENSE.txt The complete set of authors may be found
+ * at http://polymer.github.io/AUTHORS.txt The complete set of contributors may
+ * be found at http://polymer.github.io/CONTRIBUTORS.txt Code distributed by
+ * Google as part of the polymer project is also subject to an additional IP
+ * rights grant found at http://polymer.github.io/PATENTS.txt
+ */
+
+import {litLocalizeTransform} from '../outputters/transform';
+import * as ts from 'typescript';
+import {Message, makeMessageIdMap} from '../messages';
+import test, {ExecutionContext} from 'ava';
+import * as prettier from 'prettier';
+import {compileTsFragment, CompilerHostCache} from './compile-ts-fragment';
+
+const cache = new CompilerHostCache();
+const IMPORT_MSG = `import { msg } from "./lib_client/index.js";\n`;
+const IMPORT_LIT_HTML = `import { html } from "lit-html";\n`;
+
+/**
+ * Compile the given fragment of TypeScript source code using the lit-localize
+ * litLocalizeTransformer with the given translations. Check that there are no errors and
+ * that the output matches (prettier-formatted).
+ */
+function checkTransform(
+  t: ExecutionContext,
+  inputTs: string,
+  expectedJs: string,
+  messages: Message[]
+) {
+  // Rather than fuss with imports in all the test cases, this little hack
+  // automatically imports for `msg` and `html` (assuming those strings aren't
+  // used with any other meanings).
+  if (inputTs.includes('msg')) {
+    inputTs = IMPORT_MSG + inputTs;
+    // Note we don't expect to see the `msg` import in the output JS, since it
+    // should be un-used after litLocalizeTransformation.
+  }
+  if (inputTs.includes('html')) {
+    inputTs = IMPORT_LIT_HTML + inputTs;
+    expectedJs = IMPORT_LIT_HTML + expectedJs;
+  }
+  const options = ts.getDefaultCompilerOptions();
+  options.target = ts.ScriptTarget.ES2015;
+  options.module = ts.ModuleKind.ESNext;
+  options.moduleResolution = ts.ModuleResolutionKind.NodeJs;
+  // Don't automatically load typings from nodes_modules/@types, we're not using
+  // them here, so it's a waste of time.
+  options.typeRoots = [];
+  const result = compileTsFragment(inputTs, options, cache, {
+    before: [litLocalizeTransform(makeMessageIdMap(messages))],
+  });
+
+  let formattedExpected = prettier.format(expectedJs, {parser: 'typescript'});
+  let formattedActual;
+  try {
+    formattedActual = prettier.format(result.code || '', {
+      parser: 'typescript',
+    });
+  } catch {
+    // We might emit invalid TypeScript in a failing test. Rather than fail with
+    // a Prettier parse exception, it's more useful to see a diff.
+    formattedExpected = expectedJs;
+    formattedActual = result.code;
+  }
+  t.is(formattedActual, formattedExpected);
+  t.deepEqual(result.diagnostics, []);
+}
+
+test('unchanged const', (t) => {
+  const src = 'const foo = "foo";';
+  checkTransform(t, src, src, []);
+});
+
+test('unchanged html', (t) => {
+  const src =
+    'const foo = "foo"; const bar = "bar"; html`Hello ${foo} and ${bar}!`;';
+  checkTransform(t, src, src, []);
+});
+
+// TODO(aomarks) The actual tests!

src/tests/transform.unit.test.ts

@@ -0,0 +1,16 @@
+import {html} from 'lit-html';
+import {msg} from '../../../lib_client/index.js';
+
+msg('string', 'Hello World!');
+
+msg('lit', html`Hello <b><i>World!</i></b>`);
+
+msg('variables_1', (name: string) => `Hello ${name}!`, 'World');
+
+msg(
+  'lit_variables_1',
+  (url: string, name: string) =>
+    html`Hello ${name}, click <a href="${url}">here</a>!`,
+  'https://www.example.com/',
+  'World'
+);

testdata/transform/goldens/foo.ts

@@ -0,0 +1,13 @@
+{
+  "$schema": "../../../config.schema.json",
+  "sourceLocale": "en",
+  "targetLocales": ["es-419", "zh_CN"],
+  "tsConfig": "tsconfig.json",
+  "output": {
+    "mode": "transform"
+  },
+  "interchange": {
+    "format": "xliff",
+    "xliffDir": "xliff/"
+  }
+}