fix(aws-cdk-lib): toolkit is unaware of CDK app errors (#37294)

We want to know about the synth errors that happened in the CDK app in the Toolkit. In order to do that, we are doing the following things:

We print the error code between special markers. We have chosen the guillemet to print the error codes: `«InvalidBucketName»`. It nicely braces the error, look visually appealing, and is uncommon enough that it is unlikely to appear in the output of a CDK app already accidentally.

The CLI will scan `stdout` and `stderr` for text between these markers. That approach is chosen because it will work regardless of whether the CDK app is executing via jsii or not (`uncaughtException` handlers will only work for direct Node programs, not jsii programs).

In order for this masterful scheme not to be foiled by a customer putting the following into their program:

```ts
console.log('«IveTrickedYouIntoCollectingCustomerContent»');
```

Whenever an error with an error code is constructed, we write the code to special file that is indicated by the `CDK_ERROR_FILE` environment variable (which will be set by the Toolkit). Only codes that appear in that file are eligible for scanning from `stderr`/`stdout`, so that we are not tricked into collecting customer content.

Why don't we just take the error code in that file as gospel? Because the exception could be caught and the program continued. That an Error object is produced doesn't mean it terminates the program.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/aws-cdk-lib/core/lib/errors.ts

@@ -1,3 +1,4 @@
+import * as fs from 'fs';
 import type { IConstruct } from 'constructs';
 import { constructInfoFromConstruct } from './private/runtime-info';
 import { captureCallStack, renderCallStackJustMyCode } from './stack-trace';
@@ -141,11 +142,13 @@ abstract class ConstructError extends Error {
 
     // The "stack" field in Node.js includes the error description. If it doesn't, Node will fall back to an
     // ugly way of rendering the error.
-    this.stack = `${this.name}: ${msg}\n${renderCallStackJustMyCode(captureCallStack(ctr)).join('\n')}`;
+    this.stack = `«${this.name}» ${msg}\n${renderCallStackJustMyCode(captureCallStack(ctr)).join('\n')}`;
 
     if (scope) {
       this.stack += `\nRelates to construct:\n${renderConstructRootPath(scope)}`;
     }
+
+    maybeWriteErrorCode(this.name);
   }
 }
 
@@ -260,3 +263,39 @@ export function renderConstructRootPath(construct: IConstruct) {
 
   return ret.join('\n');
 }
+
+const THROWN_ERRORS = new Set<string>();
+
+/**
+ * If the appropriate environment variable is set, write this error code to a list of error codes in the given file.
+ *
+ * The reason we do this is so that the CLI can scan `stderr` for one of the
+ * error codes between markers, and be confident that when it finds something
+ * that it's not user content but an actual error we threw.
+ *
+ * - Why not just scan `stderr`? Because customers could put customer content
+ *   between those markers, and we would capture user content as an error code (we
+ *   explicitly don't want to do that!)
+ *
+ * - Why not take the error code immediately? Because the error could have been
+ *   caught; but we only want to capture the error that terminated the program.
+ *
+ * So we're doing a double whammy of writing potential error codes to a file, then
+ * make sure that we find that error code in `stderr`.
+ */
+function maybeWriteErrorCode(errorCode: string) {
+  const file = process.env.CDK_ERROR_FILE;
+  if (!file) {
+    return;
+  }
+
+  // Only if this error is new
+  const oldSize = THROWN_ERRORS.size;
+  THROWN_ERRORS.add(errorCode);
+  if (THROWN_ERRORS.size === oldSize) {
+    return;
+  }
+
+  // Update the indicated file
+  fs.writeFileSync(file, Array.from(THROWN_ERRORS).sort().join('\n'), 'utf-8');
+}

packages/aws-cdk-lib/core/test/errors.test.ts

@@ -1,3 +1,6 @@
+import { rm, readFile } from 'fs/promises';
+import * as os from 'os';
+import * as path from 'path';
 import { inspect } from 'util';
 import { Bucket } from '../../aws-s3';
 import { App, Stack } from '../lib';
@@ -30,7 +33,7 @@ describe('ValidationError', () => {
       version: expect.stringMatching(/^\d+\.\d+\.\d+$/),
     });
     expect(error.message).toBe('this is an error');
-    expect(error.stack).toContain('ValidationError: this is an error');
+    expect(error.stack).toContain('«ValidationError» this is an error');
     expect(error.stack).toContain('└─ MyStack');
   });
 
@@ -40,7 +43,7 @@ describe('ValidationError', () => {
     expect(Errors.isConstructError(error)).toBe(true);
     expect(Errors.isValidationError(error)).toBe(true);
     expect(error.name).toBe('ValidationError');
-    expect(error.stack).toContain('ValidationError: this is an error');
+    expect(error.stack).toContain('«ValidationError» this is an error');
   });
 
   test('presentation of a ValidationError', () => {
@@ -55,7 +58,7 @@ describe('ValidationError', () => {
       // NodeJS will render an uncaught error using inspect()
       const errorString = inspect(e);
       expect(anonymizeBetweenParens(errorString)).toMatchInlineSnapshot(`
-"ErrorCode: There is an error here
+"«ErrorCode» There is an error here
     at <anonymous> (...)
     ...Promise.then.completed in jest-circus...
     at new Promise (...)
@@ -75,14 +78,37 @@ Relates to construct:
       // NodeJS will render an uncaught error using inspect()
       const errorString = inspect(e);
       expect(anonymizeBetweenParens(errorString)).toMatchInlineSnapshot(`
-"ErrorCode: There is an error here
+"«ErrorCode» There is an error here
     at <anonymous> (...)
     ...Promise.then.completed in jest-circus...
     at new Promise (...)
     ...jest-circus, node internals, jest-circus, jest-runner..."
 `);
     }
   });
+
+  test('writing error codes to disk', async () => {
+    const file = path.join(os.tmpdir(), 'errors.txt');
+    await rm(file, { force: true });
+    try {
+      process.env.CDK_ERROR_FILE = file;
+
+      try {
+        throw new UnscopedValidationError('Error1', 'bla');
+      } catch { }
+      const contents1 = await readFile(file, 'utf-8');
+      expect(contents1).toEqual('Error1');
+
+      try {
+        throw new UnscopedValidationError('Error2', 'bla');
+      } catch { }
+      const contents2 = await readFile(file, 'utf-8');
+      expect(contents2).toEqual('Error1\nError2');
+    } finally {
+      delete process.env.CDK_ERROR_FILE;
+      await rm(file, { force: true });
+    }
+  });
 });
 
 /**