feat: prepare support for intersection types (#4898)

This is a preparatory PR for adding support for intersection types (`A & B`) to jsii.

That feature might land or might not, but to figure it out we first need to update `@jsii/spec` to allow modeling intersection types to begin with, as other jsii packages live outside this repo so we can't do an atomic commit adding the feature everywhere; we have to release an updated version of the spec first.

To this end, we're doing the following:

- Add the necessary fields to the spec to express the new feature.
- Add a `usedFeatures` field to assemblies to indicate whether or not a "new feature" is being used, so that tools can reject an assembly if it's using a feature they don't support.
- Wherever we call Rosetta directly from `jsii-pacmak`, we have to do runtime type assertions to make sure the types conversion is valid.

The `usedFeatures` field is chosen rather than a schema version field, to maximize compatibility: we want to gate loading an assembly on whether or not the assembly actually *uses* the new feature, not on whether or not the producing tool *knows about* the new feature.  JSON Schema validation would do the same thing (albeit with a worse error message), but for example the kernel doesn't do JSON Schema validation because it is a slow process.

Proper support for type intersections is added to `@jsii/kernel` already (because it's trivial), but `jsii-pacmak` only has pro-forma type support; it will fail at runtime.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

Author: rix0rrr

packages/@jsii/kernel/src/kernel.test.ts

@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
+import { Assembly, SchemaVersion } from '@jsii/spec';
 import * as childProcess from 'child_process';
 import * as fs from 'fs-extra';
 import * as os from 'os';
@@ -1336,6 +1337,36 @@ defineTest('loading a module twice idepotently succeeds', async (sandbox) => {
   });
 });
 
+defineTest.skipIf(!!recordingOutput)(
+  'loading an assembly with unsupported features fails',
+  async (sandbox) => {
+    const testAssembly: Assembly = {
+      author: {
+        name: 'Me',
+        roles: ['owner'],
+      },
+      description: 'Test',
+      fingerprint: 'asdf',
+      homepage: 'www.example.com',
+      jsiiVersion: '1.20.3',
+      license: 'MIT',
+      name: 'test',
+      repository: { type: 'other', url: 'www.example.com' },
+      schema: SchemaVersion.LATEST,
+      version: '1.2.3',
+      usedFeatures: ['will-never-be-used' as any],
+    };
+
+    await expect(async () =>
+      sandbox.load({
+        tarball: await makeJsiiTarball(testAssembly),
+        name: testAssembly.name,
+        version: testAssembly.version,
+      }),
+    ).rejects.toThrow(/will-never-be-used/);
+  },
+);
+
 defineTest(
   'fails if trying to load two different versions of the same module',
   async (sandbox) => {
@@ -2275,27 +2306,57 @@ async function preparePackage(module: string, useCache = true) {
   }
 
   const packageRoot = findPackageRoot(module);
-  await new Promise<void>((ok, ko) => {
-    const child = childProcess.spawn('npm', ['pack', packageRoot], {
-      cwd: staging,
+
+  await runNpm(['pack', packageRoot], staging);
+
+  const dir = path.join(staging, (await fs.readdir(staging))[0]);
+  cache.set(module, dir);
+  return dir;
+}
+
+async function makeJsiiTarball(assembly: Assembly) {
+  const staging = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'jsii-kernel-tests-'),
+  );
+
+  // clean up only if we are not recording, so playback can refer to these
+  if (!recordingOutput) {
+    process.on('exit', () => fs.removeSync(staging)); // cleanup
+  }
+
+  await fs.writeJson(path.join(staging, 'package.json'), {
+    name: assembly.name,
+    version: assembly.version,
+  });
+
+  await fs.writeJson(path.join(staging, '.jsii'), assembly);
+
+  await runNpm(['pack'], staging);
+
+  const tarballs = (await fs.readdir(staging)).filter((x) => x.endsWith('gz'));
+
+  return path.join(staging, tarballs[0]);
+}
+
+async function runNpm(args: string[], cwd: string): Promise<string> {
+  return new Promise<string>((ok, ko) => {
+    const child = childProcess.spawn('npm', args, {
+      cwd,
       shell: true,
       stdio: ['ignore', 'pipe', 'ignore'],
     });
     const stdout = new Array<Buffer>();
     child.stdout.on('data', (chunk) => stdout.push(Buffer.from(chunk)));
     child.once('close', (code, signal) => {
       if (code === 0) {
-        return ok();
+        return ok(Buffer.concat(stdout).toString());
       }
       if (code != null) {
         return ko(`'npm pack' exited with code ${code}`);
       }
       return ko(`'npm pack' killed by signal ${signal}`);
     });
   });
-  const dir = path.join(staging, (await fs.readdir(staging))[0]);
-  cache.set(module, dir);
-  return dir;
 }
 
 function findPackageRoot(pkg: string) {

packages/@jsii/kernel/src/kernel.ts

@@ -34,6 +34,7 @@ export class RuntimeError extends Error implements JsiiError {
     super(message);
   }
 }
+
 export class Kernel {
   /**
    * Set to true for verbose debugging.

packages/@jsii/kernel/src/serialization.ts

@@ -941,6 +941,10 @@ export function serializationType(
     );
   }
 
+  if (spec.isIntersectionTypeReference(typeRef.type)) {
+    throw new Error(`Intersection types cannot be serialized`);
+  }
+
   // The next part of the conversion is lookup-dependent
   const type = lookup(typeRef.type.fqn);
 

packages/@jsii/spec/src/assembly-utils.ts

@@ -3,6 +3,7 @@ import * as path from 'path';
 import * as zlib from 'zlib';
 
 import {
+  ALL_TYPESYSTEM_ENFORCED_FEATURES,
   Assembly,
   SPEC_FILE_NAME,
   SPEC_FILE_NAME_COMPRESSED,
@@ -122,11 +123,13 @@ const failNoReadfileProvided = (filename: string) => {
  * @param assemblyBuffer buffer containing SPEC_FILE_NAME contents
  * @param readFile a callback to use for reading additional support files
  * @param validate whether or not to validate the assembly
+ * @param supportedFeatures the set of supported features (default: all features enforced by the type system)
  */
 export function loadAssemblyFromBuffer(
   assemblyBuffer: Buffer,
   readFile: (filename: string) => Buffer = failNoReadfileProvided,
   validate = true,
+  supportedFeatures: string[] = ALL_TYPESYSTEM_ENFORCED_FEATURES,
 ): Assembly {
   let contents = JSON.parse(assemblyBuffer.toString('utf-8'));
 
@@ -135,6 +138,20 @@ export function loadAssemblyFromBuffer(
     contents = followRedirect(contents, readFile);
   }
 
+  // Do feature checking *before* validating using JSONSchema, and do it always.
+  // - In case validation is enabled, feature checking will produce a
+  //   more useful error message.
+  // - In case validation is disabled, feature checking is cheap and will catch
+  //   common problems.
+  const unsupported = ((contents as Assembly).usedFeatures ?? []).filter(
+    (feat) => !supportedFeatures.includes(feat),
+  );
+  if (unsupported.length > 0) {
+    throw new Error(
+      `This jsii tool cannot load the given assembly; using unsupported feature(s): ${unsupported.join(', ')}`,
+    );
+  }
+
   return validate ? validateAssembly(contents) : contents;
 }
 
@@ -149,9 +166,10 @@ export function loadAssemblyFromBuffer(
 export function loadAssemblyFromPath(
   directory: string,
   validate = true,
+  supportedFeatures?: string[],
 ): Assembly {
   const assemblyFile = findAssemblyFile(directory);
-  return loadAssemblyFromFile(assemblyFile, validate);
+  return loadAssemblyFromFile(assemblyFile, validate, supportedFeatures);
 }
 
 /**
@@ -165,13 +183,15 @@ export function loadAssemblyFromPath(
 export function loadAssemblyFromFile(
   pathToFile: string,
   validate = true,
+  supportedFeatures?: string[],
 ): Assembly {
   const data = fs.readFileSync(pathToFile);
   try {
     return loadAssemblyFromBuffer(
       data,
       (filename) => fs.readFileSync(path.resolve(pathToFile, '..', filename)),
       validate,
+      supportedFeatures,
     );
   } catch (e: any) {
     throw new Error(`Error loading assembly from file ${pathToFile}:\n${e}`);

packages/@jsii/spec/src/assembly.ts

@@ -17,6 +17,16 @@ export interface Assembly
     ReadMeContainer {
   /**
    * The version of the spec schema
+   *
+   * NOTE: we cannot ever change this value anymore! If we do, new instances of
+   * assmblies that would have been backwards compatible with the old schema
+   * still cannot be loaded, because the schema versions don't match exactly (as
+   * validated by JSON Schema validators on loading).
+   *
+   * We *must* always emit this value as 'jsii/0.10.0'.
+   *
+   * Instead, see `usedFeatures` for a dynamic way to do feature validation
+   * without relying on a fixed schema version.
    */
   schema: SchemaVersion.LATEST;
 
@@ -154,6 +164,18 @@ export interface Assembly
    * @default none
    */
   bin?: { readonly [script: string]: string };
+
+  /**
+   * List of features used in this assembly
+   *
+   * If a modern jsii feature is used in the assembly, a descriptive string
+   * should be added here. This field will be used to selectively reject the
+   * loading of assemblies that requires features not currently supported by the
+   * jsii kernel, or downstream jsii tools.
+   *
+   * @default - Only original jsii features.
+   */
+  usedFeatures?: JsiiFeature[];
 }
 
 /**
@@ -545,7 +567,8 @@ export type TypeReference =
   | NamedTypeReference
   | PrimitiveTypeReference
   | CollectionTypeReference
-  | UnionTypeReference;
+  | UnionTypeReference
+  | IntersectionTypeReference;
 
 /**
  * The standard representation of the `any` type (includes optionality marker).
@@ -626,12 +649,36 @@ export interface UnionTypeReference {
     types: TypeReference[];
   };
 }
+
 export function isUnionTypeReference(
   ref: TypeReference | undefined,
 ): ref is UnionTypeReference {
   return !!(ref as UnionTypeReference)?.union;
 }
 
+/**
+ * Reference to an intersection type.
+ */
+export interface IntersectionTypeReference {
+  /**
+   * Indicates that this is an intersection type, which means it must satisfy all of the given types.
+   */
+  intersection: {
+    /**
+     * All the possible types (including the primary type).
+     *
+     * @minItems 2
+     */
+    types: TypeReference[];
+  };
+}
+
+export function isIntersectionTypeReference(
+  ref: TypeReference | undefined,
+): ref is IntersectionTypeReference {
+  return !!(ref as IntersectionTypeReference)?.intersection;
+}
+
 /**
  * Methods and properties can be overridden from parent classes or implemented
  * from interfaces.
@@ -1028,6 +1075,39 @@ export function describeTypeReference(type?: TypeReference): string {
   throw new Error('Unrecognized type reference');
 }
 
+/**
+ * Predefined constants for a set of jsii extension features
+ */
+export type JsiiFeature = 'intersection-types';
+
+/**
+ * For every feature, is it enforced by the type system?
+ *
+ * Effectively: if a jsii tools links against the most recent version of the
+ * spec, is the TypeScript type system going to ensure that they must have
+ * support for a given new feature, through exhaustiveness checking?
+ *
+ * (This map also forces completeness, so we are guaranteed to have a string
+ * value for every possible `JsiiFeature` type branch).
+ */
+const IS_FEATURE_TYPESYSTEM_ENFORCED: Record<JsiiFeature, boolean> = {
+  'intersection-types': true,
+};
+
+/**
+ * A list of all jsii extension features
+ */
+export const ALL_FEATURES = Object.keys(IS_FEATURE_TYPESYSTEM_ENFORCED);
+
+/**
+ * A list of all jsii extension features
+ */
+export const ALL_TYPESYSTEM_ENFORCED_FEATURES = Object.entries(
+  IS_FEATURE_TYPESYSTEM_ENFORCED,
+)
+  .filter(([_, v]) => v)
+  .map(([k, _]) => k);
+
 /**
  * Determines whether an entity is deprecated.
  *

packages/jsii-pacmak/lib/index.ts

@@ -6,6 +6,7 @@ import { cwd } from 'process';
 import * as logging from './logging';
 import { findJsiiModules, updateAllNpmIgnores } from './npm-modules';
 import { JsiiModule } from './packaging';
+import { assertSpecIsRosettaCompatible } from './rosetta-assembly';
 import { ALL_BUILDERS, TargetName } from './targets';
 import { Timers } from './timer';
 import { Toposorted } from './toposort';
@@ -84,6 +85,9 @@ export async function pacmak({
     return Promise.all(
       modulesToPackageFlat.map(async (m) => {
         await m.load(system, validateAssemblies);
+
+        assertSpecIsRosettaCompatible(m.assembly.spec);
+
         return rosetta.addAssembly(m.assembly.spec, m.moduleDirectory);
       }),
     );

packages/jsii-pacmak/lib/rosetta-assembly.ts

@@ -0,0 +1,30 @@
+import * as spec from '@jsii/spec';
+import { JsiiFeature } from '@jsii/spec';
+import { RosettaTabletReader } from 'jsii-rosetta';
+
+/**
+ * Assert that the given spec is safe to give to Rosetta
+ *
+ * We have to do it like this, because Rosetta has its own internal copy of the
+ * spec and new schema additions may make it technically illegal to assign our
+ * Assembly instance to Rosetta's Assembly type.
+ *
+ * We do runtime validation here to make sure that assignment is safe,
+ * and then assert it in the type system.
+ *
+ * The check should be cheap, this gets called quite a lot.
+ */
+export function assertSpecIsRosettaCompatible(
+  x: spec.Assembly,
+): asserts x is Parameters<RosettaTabletReader['addAssembly']>[0] {
+  const rosettaSupportedFeatures: JsiiFeature[] = [];
+
+  const unsupported = (x.usedFeatures ?? []).filter(
+    (f) => !rosettaSupportedFeatures.includes(f),
+  );
+  if (unsupported.length > 0) {
+    throw new Error(
+      `This assembly uses features Rosetta doesn't support yet: ${unsupported.join(', ')}`,
+    );
+  }
+}

packages/jsii-pacmak/lib/targets/dotnet/dotnetdocgenerator.ts

@@ -11,6 +11,7 @@ import * as xmlbuilder from 'xmlbuilder';
 
 import { renderSummary } from '../_utils';
 import { DotNetNameUtils } from './nameutils';
+import { assertSpecIsRosettaCompatible } from '../../rosetta-assembly';
 
 /**
  * Generates the Jsii attributes and calls for the .NET runtime
@@ -160,6 +161,7 @@ export class DotNetDocGenerator {
   }
 
   private convertExample(example: string, apiLocation: ApiLocation): string {
+    assertSpecIsRosettaCompatible(this.assembly);
     const translated = this.rosetta.translateExample(
       apiLocation,
       example,
@@ -170,6 +172,7 @@ export class DotNetDocGenerator {
   }
 
   private convertSamplesInMarkdown(markdown: string, api: ApiLocation): string {
+    assertSpecIsRosettaCompatible(this.assembly);
     return this.rosetta.translateSnippetsInMarkdown(
       api,
       markdown,