feat(jsii-rosetta transliterate): transliterate a jsii assembly (#2869)

The new `jsii-rosetta transliterate` command can be used to
transliterate one or more jsii assemblies (the `.jsii` file) to one or
more target languages. The output assembly has all code examples
transliterated into the correct target language.

The current feature is experimental. It does not include renaming API
elements within the assembly (only code examples are touched) - this
feature may be added in the future.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

Author: RomainMuller

packages/jsii-pacmak/lib/targets/dotnet/dotnetdocgenerator.ts

@@ -2,6 +2,7 @@ import * as spec from '@jsii/spec';
 import { CodeMaker } from 'codemaker';
 import {
   Rosetta,
+  TargetLanguage,
   Translation,
   enforcesStrictMode,
   typeScriptSnippetFromSource,
@@ -166,7 +167,10 @@ export class DotNetDocGenerator {
       'example',
       enforcesStrictMode(this.assembly),
     );
-    const translated = this.rosetta.translateSnippet(snippet, 'csharp');
+    const translated = this.rosetta.translateSnippet(
+      snippet,
+      TargetLanguage.CSHARP,
+    );
     if (!translated) {
       return example;
     }
@@ -176,7 +180,7 @@ export class DotNetDocGenerator {
   private convertSamplesInMarkdown(markdown: string): string {
     return this.rosetta.translateSnippetsInMarkdown(
       markdown,
-      'csharp',
+      TargetLanguage.CSHARP,
       enforcesStrictMode(this.assembly),
       (trans) => ({
         language: trans.language,

packages/jsii-pacmak/lib/targets/java.ts

@@ -5,6 +5,7 @@ import * as fs from 'fs-extra';
 import * as reflect from 'jsii-reflect';
 import {
   Rosetta,
+  TargetLanguage,
   typeScriptSnippetFromSource,
   Translation,
   enforcesStrictMode,
@@ -2916,7 +2917,10 @@ class JavaGenerator extends Generator {
       'example',
       enforcesStrictMode(this.assembly),
     );
-    const translated = this.rosetta.translateSnippet(snippet, 'java');
+    const translated = this.rosetta.translateSnippet(
+      snippet,
+      TargetLanguage.JAVA,
+    );
     if (!translated) {
       return example;
     }
@@ -2926,7 +2930,7 @@ class JavaGenerator extends Generator {
   private convertSamplesInMarkdown(markdown: string): string {
     return this.rosetta.translateSnippetsInMarkdown(
       markdown,
-      'java',
+      TargetLanguage.JAVA,
       enforcesStrictMode(this.assembly),
       (trans) => ({
         language: trans.language,

packages/jsii-pacmak/lib/targets/python.ts

@@ -4,6 +4,7 @@ import * as escapeStringRegexp from 'escape-string-regexp';
 import * as fs from 'fs-extra';
 import * as reflect from 'jsii-reflect';
 import {
+  TargetLanguage,
   Translation,
   Rosetta,
   enforcesStrictMode,
@@ -2290,7 +2291,10 @@ class PythonGenerator extends Generator {
       'example',
       enforcesStrictMode(this.assembly),
     );
-    const translated = this.rosetta.translateSnippet(snippet, 'python');
+    const translated = this.rosetta.translateSnippet(
+      snippet,
+      TargetLanguage.PYTHON,
+    );
     if (!translated) {
       return example;
     }
@@ -2300,7 +2304,7 @@ class PythonGenerator extends Generator {
   public convertMarkdown(markdown: string): string {
     return this.rosetta.translateSnippetsInMarkdown(
       markdown,
-      'python',
+      TargetLanguage.PYTHON,
       enforcesStrictMode(this.assembly),
       (trans) => ({
         language: trans.language,

packages/jsii-rosetta/bin/jsii-rosetta.ts

@@ -10,6 +10,8 @@ import {
 import { translateMarkdown } from '../lib/commands/convert';
 import { extractSnippets } from '../lib/commands/extract';
 import { readTablet } from '../lib/commands/read';
+import { transliterateAssembly } from '../lib/commands/transliterate';
+import { TargetLanguage } from '../lib/languages';
 import { PythonVisitor } from '../lib/languages/python';
 import { VisualizeAstVisitor } from '../lib/languages/visualize';
 import * as logging from '../lib/logging';
@@ -166,6 +168,60 @@ function main() {
         }
       }),
     )
+    .command(
+      'transliterate [ASSEMBLY..]',
+      '(EXPERIMENTAL) Transliterates the designated assemblies',
+      (command) =>
+        command
+          .positional('ASSEMBLY', {
+            type: 'string',
+            string: true,
+            default: new Array<string>(),
+            required: true,
+            describe: 'Assembly to transliterate',
+          })
+          .option('language', {
+            alias: 'l',
+            type: 'string',
+            string: true,
+            default: new Array<string>(),
+            describe: 'Language ID to transliterate to',
+          })
+          .options('strict', {
+            alias: 's',
+            type: 'boolean',
+            describe:
+              'Fail if an example that needs live transliteration fails to compile (which could cause incorrect transpilation results)',
+          })
+          .option('tablet', {
+            alias: 't',
+            type: 'string',
+            describe:
+              'Language tablet containing pre-translated code examples to use (these are generated by the `extract` command)',
+          }),
+      wrapHandler((args) => {
+        const assemblies = (
+          args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']
+        ).map((dir) => path.resolve(process.cwd(), dir));
+        const languages =
+          args.language.length > 0
+            ? args.language.map((lang) => {
+                const target = Object.entries(TargetLanguage).find(
+                  ([k]) => k === lang,
+                )?.[1];
+                if (target == null) {
+                  throw new Error(
+                    `Unknown target language: ${lang}. Expected one of ${Object.keys(
+                      TargetLanguage,
+                    ).join(', ')}`,
+                  );
+                }
+                return target;
+              })
+            : Object.values(TargetLanguage);
+        return transliterateAssembly(assemblies, languages, args);
+      }),
+    )
     .command(
       'read <TABLET> [KEY] [LANGUAGE]',
       'Display snippets in a language tablet file',

packages/jsii-rosetta/jest.config.ts

@@ -1,3 +1,7 @@
-import config from '../../jest.config';
+import { join } from 'path';
 
-export default config;
+import { overriddenConfig } from '../../jest.config';
+
+export default overriddenConfig({
+  setupFiles: [join(__dirname, 'jestsetup.js')],
+});

packages/jsii-rosetta/jestsetup.js

@@ -0,0 +1,4 @@
+// Require `mock-fs` before `jest` initializes, as `mock-fs`  relies on
+// hijacking the `fs` module, which `jest` also hijacks (and that needs to
+// happen last).
+require('mock-fs');

packages/jsii-rosetta/lib/commands/transliterate.ts

@@ -0,0 +1,203 @@
+import { Assembly, Docs, SPEC_FILE_NAME, Type, TypeKind } from '@jsii/spec';
+import { readJson, writeJson } from 'fs-extra';
+import { resolve } from 'path';
+
+import { fixturize } from '../fixtures';
+import { TargetLanguage } from '../languages';
+import { debug } from '../logging';
+import { Rosetta } from '../rosetta';
+import { SnippetParameters, typeScriptSnippetFromSource } from '../snippet';
+import { Translation } from '../tablets/tablets';
+
+export interface TransliterateAssemblyOptions {
+  /**
+   * Whather transliteration should fail upon failing to compile an example that
+   * required live transliteration.
+   *
+   * @default false
+   */
+  readonly strict?: boolean;
+
+  /**
+   * A pre-build translation tablet (as produced by `jsii-rosetta extract`).
+   *
+   * @default - Only the default tablet (`.jsii.tabl.json`) files will be used.
+   */
+  readonly tablet?: string;
+}
+
+/**
+ * Prepares transliterated versions of the designated assemblies into the
+ * selected taregt languages.
+ *
+ * @param assemblyLocations the directories which contain assemblies to
+ *                          transliterate.
+ * @param targetLanguages   the languages into which to transliterate.
+ * @param tabletLocation    an optional Rosetta tablet file to source
+ *                          pre-transliterated snippets from.
+ *
+ * @experimental
+ */
+export async function transliterateAssembly(
+  assemblyLocations: readonly string[],
+  targetLanguages: readonly TargetLanguage[],
+  options: TransliterateAssemblyOptions = {},
+): Promise<void> {
+  const rosetta = new Rosetta({
+    includeCompilerDiagnostics: true,
+    liveConversion: true,
+    targetLanguages,
+  });
+  if (options.tablet) {
+    await rosetta.loadTabletFromFile(options.tablet);
+  }
+  const assemblies = await loadAssemblies(assemblyLocations, rosetta);
+
+  for (const [location, loadAssembly] of assemblies.entries()) {
+    for (const language of targetLanguages) {
+      const now = new Date().getTime();
+      // eslint-disable-next-line no-await-in-loop
+      const result = await loadAssembly();
+      if (result.readme?.markdown) {
+        result.readme.markdown = rosetta.translateSnippetsInMarkdown(
+          result.readme.markdown,
+          language,
+          true /* strict */,
+          (translation) => ({
+            language: translation.language,
+            source: prefixDisclaimer(translation),
+          }),
+          location,
+        );
+      }
+      for (const type of Object.values(result.types ?? {})) {
+        transliterateType(type, rosetta, language, location);
+      }
+      // eslint-disable-next-line no-await-in-loop
+      await writeJson(
+        resolve(location, `${SPEC_FILE_NAME}.${language}`),
+        result,
+        { spaces: 2 },
+      );
+      const then = new Date().getTime();
+      debug(
+        `Done transliterating ${result.name}@${
+          result.version
+        } to ${language} after ${then - now} milliseconds`,
+      );
+    }
+  }
+
+  rosetta.printDiagnostics(process.stderr);
+  if (rosetta.hasErrors && options.strict) {
+    throw new Error(
+      'Strict mode is enabled and some examples failed compilation!',
+    );
+  }
+}
+
+/**
+ * Given a set of directories containing `.jsii` assemblies, load all the
+ * assemblies into the provided `Rosetta` instance and return a map of
+ * directories to assembly-loading functions (the function re-loads the original
+ * assembly from disk on each invocation).
+ *
+ * @param directories the assembly-containing directories to traverse.
+ * @param rosetta     the `Rosetta` instance in which to load assemblies.
+ *
+ * @returns a map of directories to a function that loads the `.jsii` assembly
+ *          contained therein from disk.
+ */
+async function loadAssemblies(
+  directories: readonly string[],
+  rosetta: Rosetta,
+): Promise<ReadonlyMap<string, AssemblyLoader>> {
+  const result = new Map<string, AssemblyLoader>();
+
+  for (const directory of directories) {
+    const loader = () => readJson(resolve(directory, SPEC_FILE_NAME));
+    // eslint-disable-next-line no-await-in-loop
+    await rosetta.addAssembly(await loader(), directory);
+    result.set(directory, loader);
+  }
+
+  return result;
+}
+
+type Mutable<T> = { -readonly [K in keyof T]: Mutable<T[K]> };
+type AssemblyLoader = () => Promise<Mutable<Assembly>>;
+
+function prefixDisclaimer(translation: Translation): string {
+  const message = translation.didCompile
+    ? 'Example automatically generated. See https://github.com/aws/jsii/issues/826'
+    : 'Example automatically generated without compilation. See https://github.com/aws/jsii/issues/826';
+  return `${commentToken()} ${message}\n${translation.source}`;
+
+  function commentToken() {
+    // This is future-proofed a bit, but don't read too much in this...
+    switch (translation.language) {
+      case 'python':
+      case 'ruby':
+        return '#';
+      case 'csharp':
+      case 'java':
+      case 'go':
+      default:
+        return '//';
+    }
+  }
+}
+
+function transliterateType(
+  type: Type,
+  rosetta: Rosetta,
+  language: TargetLanguage,
+  workingDirectory: string,
+): void {
+  transliterateDocs(type.docs);
+  switch (type.kind) {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore 7029
+    case TypeKind.Class:
+      transliterateDocs(type?.initializer?.docs);
+
+    // fallthrough
+    case TypeKind.Interface:
+      for (const method of type.methods ?? []) {
+        transliterateDocs(method.docs);
+        for (const parameter of method.parameters ?? []) {
+          transliterateDocs(parameter.docs);
+        }
+      }
+      for (const property of type.properties ?? []) {
+        transliterateDocs(property.docs);
+      }
+      break;
+
+    case TypeKind.Enum:
+      for (const member of type.members) {
+        transliterateDocs(member.docs);
+      }
+      break;
+
+    default:
+      throw new Error(`Unsupported type kind: ${(type as any).kind}`);
+  }
+
+  function transliterateDocs(docs: Docs | undefined) {
+    if (docs?.example) {
+      const snippet = fixturize(
+        typeScriptSnippetFromSource(
+          docs.example,
+          'example',
+          true /* strict */,
+          { [SnippetParameters.$PROJECT_DIRECTORY]: workingDirectory },
+        ),
+      );
+      const translation = rosetta.translateSnippet(snippet, language);
+      if (translation != null) {
+        docs.example = prefixDisclaimer(translation);
+      }
+    }
+  }
+}

packages/jsii-rosetta/lib/index.ts

@@ -1,5 +1,6 @@
 export * from './translate';
 export { renderTree } from './o-tree';
+export { TargetLanguage } from './languages/target-language';
 export { CSharpVisitor } from './languages/csharp';
 export { JavaVisitor } from './languages/java';
 export { PythonVisitor } from './languages/python';

packages/jsii-rosetta/lib/languages/csharp.ts

@@ -23,6 +23,7 @@ import {
 } from '../typescript/types';
 import { flat, partition, setExtend } from '../util';
 import { DefaultVisitor } from './default';
+import { TargetLanguage } from './target-language';
 
 interface CSharpLanguageContext {
   /**
@@ -74,7 +75,7 @@ interface CSharpLanguageContext {
 type CSharpRenderer = AstRenderer<CSharpLanguageContext>;
 
 export class CSharpVisitor extends DefaultVisitor<CSharpLanguageContext> {
-  public readonly language = 'csharp';
+  public readonly language = TargetLanguage.CSHARP;
 
   public readonly defaultContext = {
     propertyOrMethod: false,