Merge pull request #28644 from Microsoft/cacheHostResults

[release-3.2] Cache results for readFile, fileExists, directory exists, sourceFiles for .d.ts files across the build (only first time)

Author: sheetalkamat

src/compiler/program.ts

@@ -73,7 +73,6 @@ namespace ts {
     // TODO(shkamat): update this after reworking ts build API
     export function createCompilerHostWorker(options: CompilerOptions, setParentNodes?: boolean, system = sys): CompilerHost {
         const existingDirectories = createMap<boolean>();
-
         function getCanonicalFileName(fileName: string): string {
             // if underlying system can distinguish between two files whose names differs only in cases then file name already in canonical form.
             // otherwise use toLowerCase as a canonical form.
@@ -84,7 +83,7 @@ namespace ts {
             let text: string | undefined;
             try {
                 performance.mark("beforeIORead");
-                text = system.readFile(fileName, options.charset);
+                text = compilerHost.readFile(fileName);
                 performance.mark("afterIORead");
                 performance.measure("I/O Read", "beforeIORead", "afterIORead");
             }
@@ -113,7 +112,12 @@ namespace ts {
             if (directoryPath.length > getRootLength(directoryPath) && !directoryExists(directoryPath)) {
                 const parentDirectory = getDirectoryPath(directoryPath);
                 ensureDirectoriesExist(parentDirectory);
-                system.createDirectory(directoryPath);
+                if (compilerHost.createDirectory) {
+                    compilerHost.createDirectory(directoryPath);
+                }
+                else {
+                    system.createDirectory(directoryPath);
+                }
             }
         }
 
@@ -177,8 +181,7 @@ namespace ts {
 
         const newLine = getNewLineCharacter(options, () => system.newLine);
         const realpath = system.realpath && ((path: string) => system.realpath!(path));
-
-        return {
+        const compilerHost: CompilerHost = {
             getSourceFile,
             getDefaultLibLocation,
             getDefaultLibFileName: options => combinePaths(getDefaultLibLocation(), getDefaultLibFileName(options)),
@@ -194,7 +197,117 @@ namespace ts {
             getEnvironmentVariable: name => system.getEnvironmentVariable ? system.getEnvironmentVariable(name) : "",
             getDirectories: (path: string) => system.getDirectories(path),
             realpath,
-            readDirectory: (path, extensions, include, exclude, depth) => system.readDirectory(path, extensions, include, exclude, depth)
+            readDirectory: (path, extensions, include, exclude, depth) => system.readDirectory(path, extensions, include, exclude, depth),
+            createDirectory: d => system.createDirectory(d)
+        };
+        return compilerHost;
+    }
+
+    /*@internal*/
+    export function changeCompilerHostToUseCache(
+        host: CompilerHost,
+        toPath: (fileName: string) => Path,
+        useCacheForSourceFile: boolean
+    ) {
+        const originalReadFile = host.readFile;
+        const originalFileExists = host.fileExists;
+        const originalDirectoryExists = host.directoryExists;
+        const originalCreateDirectory = host.createDirectory;
+        const originalWriteFile = host.writeFile;
+        const originalGetSourceFile = host.getSourceFile;
+        const readFileCache = createMap<string | false>();
+        const fileExistsCache = createMap<boolean>();
+        const directoryExistsCache = createMap<boolean>();
+        const sourceFileCache = createMap<SourceFile>();
+
+        const readFileWithCache = (fileName: string): string | undefined => {
+            const key = toPath(fileName);
+            const value = readFileCache.get(key);
+            if (value !== undefined) return value || undefined;
+            return setReadFileCache(key, fileName);
+        };
+        const setReadFileCache = (key: Path, fileName: string) => {
+            const newValue = originalReadFile.call(host, fileName);
+            readFileCache.set(key, newValue || false);
+            return newValue;
+        };
+        host.readFile = fileName => {
+            const key = toPath(fileName);
+            const value = readFileCache.get(key);
+            if (value !== undefined) return value; // could be .d.ts from output
+            if (!fileExtensionIs(fileName, Extension.Json)) {
+                return originalReadFile.call(host, fileName);
+            }
+
+            return setReadFileCache(key, fileName);
+        };
+
+        if (useCacheForSourceFile) {
+            host.getSourceFile = (fileName, languageVersion, onError, shouldCreateNewSourceFile) => {
+                const key = toPath(fileName);
+                const value = sourceFileCache.get(key);
+                if (value) return value;
+
+                const sourceFile = originalGetSourceFile.call(host, fileName, languageVersion, onError, shouldCreateNewSourceFile);
+                if (sourceFile && (isDeclarationFileName(fileName) || fileExtensionIs(fileName, Extension.Json))) {
+                    sourceFileCache.set(key, sourceFile);
+                }
+                return sourceFile;
+            };
+        }
+
+        // fileExists for any kind of extension
+        host.fileExists = fileName => {
+            const key = toPath(fileName);
+            const value = fileExistsCache.get(key);
+            if (value !== undefined) return value;
+            const newValue = originalFileExists.call(host, fileName);
+            fileExistsCache.set(key, !!newValue);
+            return newValue;
+        };
+        host.writeFile = (fileName, data, writeByteOrderMark, onError, sourceFiles) => {
+            const key = toPath(fileName);
+            fileExistsCache.delete(key);
+
+            const value = readFileCache.get(key);
+            if (value && value !== data) {
+                readFileCache.delete(key);
+                sourceFileCache.delete(key);
+            }
+            else if (useCacheForSourceFile) {
+                const sourceFile = sourceFileCache.get(key);
+                if (sourceFile && sourceFile.text !== data) {
+                    sourceFileCache.delete(key);
+                }
+            }
+            originalWriteFile.call(host, fileName, data, writeByteOrderMark, onError, sourceFiles);
+        };
+
+        // directoryExists
+        if (originalDirectoryExists && originalCreateDirectory) {
+            host.directoryExists = directory => {
+                const key = toPath(directory);
+                const value = directoryExistsCache.get(key);
+                if (value !== undefined) return value;
+                const newValue = originalDirectoryExists.call(host, directory);
+                directoryExistsCache.set(key, !!newValue);
+                return newValue;
+            };
+            host.createDirectory = directory => {
+                const key = toPath(directory);
+                directoryExistsCache.delete(key);
+                originalCreateDirectory.call(host, directory);
+            };
+        }
+
+        return {
+            originalReadFile,
+            originalFileExists,
+            originalDirectoryExists,
+            originalCreateDirectory,
+            originalWriteFile,
+            originalGetSourceFile,
+            readFileWithCache
         };
     }
 

src/compiler/tsbuild.ts

@@ -433,6 +433,7 @@ namespace ts {
         const missingRoots = createMap<true>();
         let globalDependencyGraph: DependencyGraph | undefined;
         const writeFileName = (s: string) => host.trace && host.trace(s);
+        let readFileWithCache = (f: string) => host.readFile(f);
 
         // Watch state
         const diagnostics = createFileMap<ReadonlyArray<Diagnostic>>(toPath);
@@ -1073,7 +1074,7 @@ namespace ts {
                 let priorChangeTime: Date | undefined;
                 if (!anyDtsChanged && isDeclarationFile(name)) {
                     // Check for unchanged .d.ts files
-                    if (host.fileExists(name) && host.readFile(name) === text) {
+                    if (host.fileExists(name) && readFileWithCache(name) === text) {
                         priorChangeTime = host.getModifiedTime(name);
                     }
                     else {
@@ -1184,6 +1185,15 @@ namespace ts {
 
         function buildAllProjects(): ExitStatus {
             if (options.watch) { reportWatchStatus(Diagnostics.Starting_compilation_in_watch_mode); }
+            // TODO:: In watch mode as well to use caches for incremental build once we can invalidate caches correctly and have right api
+            // Override readFile for json files and output .d.ts to cache the text
+            const { originalReadFile, originalFileExists, originalDirectoryExists,
+                originalCreateDirectory, originalWriteFile, originalGetSourceFile,
+                readFileWithCache: newReadFileWithCache
+            } = changeCompilerHostToUseCache(host, toPath, /*useCacheForSourceFile*/ true);
+            const savedReadFileWithCache = readFileWithCache;
+            readFileWithCache = newReadFileWithCache;
+
             const graph = getGlobalDependencyGraph();
             reportBuildQueue(graph);
             let anyFailed = false;
@@ -1234,6 +1244,13 @@ namespace ts {
                 anyFailed = anyFailed || !!(buildResult & BuildResultFlags.AnyErrors);
             }
             reportErrorSummary();
+            host.readFile = originalReadFile;
+            host.fileExists = originalFileExists;
+            host.directoryExists = originalDirectoryExists;
+            host.createDirectory = originalCreateDirectory;
+            host.writeFile = originalWriteFile;
+            readFileWithCache = savedReadFileWithCache;
+            host.getSourceFile = originalGetSourceFile;
             return anyFailed ? ExitStatus.DiagnosticsPresent_OutputsSkipped : ExitStatus.Success;
         }
 

src/compiler/types.ts

@@ -5014,6 +5014,9 @@ namespace ts {
         /* @internal */ hasInvalidatedResolution?: HasInvalidatedResolution;
         /* @internal */ hasChangedAutomaticTypeDirectiveNames?: boolean;
         createHash?(data: string): string;
+
+        // TODO: later handle this in better way in builder host instead once the api for tsbuild finalizes and doesnt use compilerHost as base
+        /*@internal*/createDirectory?(directory: string): void;
     }
 
     /* @internal */

src/tsc/tsc.ts

@@ -227,6 +227,9 @@ namespace ts {
 
     function performCompilation(rootNames: string[], projectReferences: ReadonlyArray<ProjectReference> | undefined, options: CompilerOptions, configFileParsingDiagnostics?: ReadonlyArray<Diagnostic>) {
         const host = createCompilerHost(options);
+        const currentDirectory = host.getCurrentDirectory();
+        const getCanonicalFileName = createGetCanonicalFileName(host.useCaseSensitiveFileNames());
+        changeCompilerHostToUseCache(host, fileName => toPath(fileName, currentDirectory, getCanonicalFileName), /*useCacheForSourceFile*/ false);
         enableStatistics(options);
 
         const programOptions: CreateProgramOptions = {