Detect malformed UTF-8 files and refuse to engage further (#53667)

Author: RyanCavanaugh

src/compiler/diagnosticMessages.json

@@ -1605,6 +1605,10 @@
         "category": "Error",
         "code": 1489
     },
+    "File appears to be binary.": {
+        "category": "Error",
+        "code": 1490
+    },
 
     "The types of '{0}' are incompatible between these types.": {
         "category": "Error",

src/compiler/program.ts

@@ -779,7 +779,7 @@ export function formatDiagnosticsWithColorAndContext(diagnostics: readonly Diagn
         output += formatColorAndReset(` TS${diagnostic.code}: `, ForegroundColorEscapeSequences.Grey);
         output += flattenDiagnosticMessageText(diagnostic.messageText, host.getNewLine());
 
-        if (diagnostic.file) {
+        if (diagnostic.file && diagnostic.code !== Diagnostics.File_appears_to_be_binary.code) {
             output += host.getNewLine();
             output += formatCodeSpan(diagnostic.file, diagnostic.start!, diagnostic.length!, "", getCategoryFormat(diagnostic.category), host); // TODO: GH#18217
         }

src/compiler/scanner.ts

@@ -1782,16 +1782,28 @@ export function createScanner(languageVersion: ScriptTarget,
             if (pos >= end) {
                 return token = SyntaxKind.EndOfFileToken;
             }
-            const ch = codePointAt(text, pos);
 
-            // Special handling for shebang
-            if (ch === CharacterCodes.hash && pos === 0 && isShebangTrivia(text, pos)) {
-                pos = scanShebangTrivia(text, pos);
-                if (skipTrivia) {
-                    continue;
+            const ch = codePointAt(text, pos);
+            if (pos === 0) {
+                // If a file wasn't valid text at all, it will usually be apparent at
+                // position 0 because UTF-8 decode will fail and produce U+FFFD.
+                // If that happens, just issue one error and refuse to try to scan further;
+                // this is likely a binary file that cannot be parsed
+                if (ch === CharacterCodes.replacementCharacter) {
+                    // Jump to the end of the file and fail.
+                    error(Diagnostics.File_appears_to_be_binary);
+                    pos = end;
+                    return token = SyntaxKind.NonTextFileMarkerTrivia;
                 }
-                else {
-                    return token = SyntaxKind.ShebangTrivia;
+                // Special handling for shebang
+                if (ch === CharacterCodes.hash && isShebangTrivia(text, pos)) {
+                    pos = scanShebangTrivia(text, pos);
+                    if (skipTrivia) {
+                        continue;
+                    }
+                    else {
+                        return token = SyntaxKind.ShebangTrivia;
+                    }
                 }
             }
 

src/compiler/types.ts

@@ -49,6 +49,9 @@ export const enum SyntaxKind {
     // We detect and provide better error recovery when we encounter a git merge marker.  This
     // allows us to edit files with git-conflict markers in them in a much more pleasant manner.
     ConflictMarkerTrivia,
+    // If a file is actually binary, with any luck, we'll get U+FFFD REPLACEMENT CHARACTER
+    // in position zero and can just skip what is surely a doomed parse.
+    NonTextFileMarkerTrivia,
     // Literals
     NumericLiteral,
     BigIntLiteral,
@@ -7469,6 +7472,9 @@ export const enum CharacterCodes {
     mathematicalSpace = 0x205F,
     ogham = 0x1680,
 
+    // Unicode replacement character produced when a byte sequence is invalid
+    replacementCharacter = 0xFFFD,
+
     _ = 0x5F,
     $ = 0x24,