From 6a37fd46fe23ce8bfeb7f5f5a598b4ea916197d7 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Tue, 20 Nov 2018 13:26:14 -0800 Subject: [PATCH 1/3] Cache results for readFile, fileExists, directory exists, sourceFiles for .d.ts files across the build (only first time) --- src/compiler/program.ts | 17 ++++--- src/compiler/tsbuild.ts | 101 +++++++++++++++++++++++++++++++++++++++- src/compiler/types.ts | 3 ++ 3 files changed, 114 insertions(+), 7 deletions(-) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 968ad2d073e5e..24eeff36c6cbb 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -73,7 +73,6 @@ namespace ts { // TODO(shkamat): update this after reworking ts build API export function createCompilerHostWorker(options: CompilerOptions, setParentNodes?: boolean, system = sys): CompilerHost { const existingDirectories = createMap(); - function getCanonicalFileName(fileName: string): string { // if underlying system can distinguish between two files whose names differs only in cases then file name already in canonical form. // otherwise use toLowerCase as a canonical form. @@ -84,7 +83,7 @@ namespace ts { let text: string | undefined; try { performance.mark("beforeIORead"); - text = system.readFile(fileName, options.charset); + text = host.readFile(fileName); performance.mark("afterIORead"); performance.measure("I/O Read", "beforeIORead", "afterIORead"); } @@ -113,7 +112,12 @@ namespace ts { if (directoryPath.length > getRootLength(directoryPath) && !directoryExists(directoryPath)) { const parentDirectory = getDirectoryPath(directoryPath); ensureDirectoriesExist(parentDirectory); - system.createDirectory(directoryPath); + if (host.createDirectory) { + host.createDirectory(directoryPath); + } + else { + system.createDirectory(directoryPath); + } } } @@ -177,8 +181,7 @@ namespace ts { const newLine = getNewLineCharacter(options, () => system.newLine); const realpath = system.realpath && ((path: string) => system.realpath!(path)); - - return { + const host: CompilerHost = { getSourceFile, getDefaultLibLocation, getDefaultLibFileName: options => combinePaths(getDefaultLibLocation(), getDefaultLibFileName(options)), @@ -194,8 +197,10 @@ namespace ts { getEnvironmentVariable: name => system.getEnvironmentVariable ? system.getEnvironmentVariable(name) : "", getDirectories: (path: string) => system.getDirectories(path), realpath, - readDirectory: (path, extensions, include, exclude, depth) => system.readDirectory(path, extensions, include, exclude, depth) + readDirectory: (path, extensions, include, exclude, depth) => system.readDirectory(path, extensions, include, exclude, depth), + createDirectory: d => system.createDirectory(d) }; + return host; } export function getPreEmitDiagnostics(program: Program, sourceFile?: SourceFile, cancellationToken?: CancellationToken): ReadonlyArray { diff --git a/src/compiler/tsbuild.ts b/src/compiler/tsbuild.ts index e17057b9140ec..cd3ed158995e2 100644 --- a/src/compiler/tsbuild.ts +++ b/src/compiler/tsbuild.ts @@ -433,6 +433,7 @@ namespace ts { const missingRoots = createMap(); let globalDependencyGraph: DependencyGraph | undefined; const writeFileName = (s: string) => host.trace && host.trace(s); + let readFileWithCache = (f: string) => host.readFile(f); // Watch state const diagnostics = createFileMap>(toPath); @@ -1072,7 +1073,7 @@ namespace ts { let priorChangeTime: Date | undefined; if (!anyDtsChanged && isDeclarationFile(fileName)) { // Check for unchanged .d.ts files - if (host.fileExists(fileName) && host.readFile(fileName) === content) { + if (host.fileExists(fileName) && readFileWithCache(fileName) === content) { priorChangeTime = host.getModifiedTime(fileName); } else { @@ -1182,6 +1183,97 @@ namespace ts { function buildAllProjects(): ExitStatus { if (options.watch) { reportWatchStatus(Diagnostics.Starting_compilation_in_watch_mode); } + const originalReadFile = host.readFile; + const originalFileExists = host.fileExists; + const originalDirectoryExists = host.directoryExists; + const originalCreateDirectory = host.createDirectory; + const originalWriteFile = host.writeFile; + const originalGetSourceFile = host.getSourceFile; + const readFileCache = createMap(); + const fileExistsCache = createMap(); + const directoryExistsCache = createMap(); + const sourceFileCache = createMap(); + const savedReadFileWithCache = readFileWithCache; + + // TODO:: In watch mode as well to use caches for incremental build once we can invalidate caches correctly and have right api + // Override readFile for json files and output .d.ts to cache the text + readFileWithCache = fileName => { + const key = toPath(fileName); + const value = readFileCache.get(key); + if (value !== undefined) return value || undefined; + return setReadFileCache(key, fileName); + }; + const setReadFileCache = (key: Path, fileName: string) => { + const newValue = originalReadFile.call(host, fileName); + readFileCache.set(key, newValue || false); + return newValue; + }; + host.readFile = fileName => { + const key = toPath(fileName); + const value = readFileCache.get(key); + if (value !== undefined) return value; // could be .d.ts from output + if (!fileExtensionIs(fileName, Extension.Json)) { + return originalReadFile.call(host, fileName); + } + + return setReadFileCache(key, fileName); + }; + host.getSourceFile = (fileName, languageVersion, onError, shouldCreateNewSourceFile) => { + const key = toPath(fileName); + const value = sourceFileCache.get(key); + if (value) return value; + + const sourceFile = originalGetSourceFile.call(host, fileName, languageVersion, onError, shouldCreateNewSourceFile); + if (sourceFile && (isDeclarationFileName(fileName) || fileExtensionIs(fileName, Extension.Json))) { + sourceFileCache.set(key, sourceFile); + } + return sourceFile; + }; + + // fileExits for any kind of extension + host.fileExists = fileName => { + const key = toPath(fileName); + const value = fileExistsCache.get(key); + if (value !== undefined) return value; + const newValue = originalFileExists.call(host, fileName); + fileExistsCache.set(key, !!newValue); + return newValue; + }; + host.writeFile = (fileName, data, writeByteOrderMark, onError, sourceFiles) => { + const key = toPath(fileName); + fileExistsCache.delete(key); + + const value = readFileCache.get(key); + if (value && value !== data) { + readFileCache.delete(key); + sourceFileCache.delete(key); + } + else { + const sourceFile = sourceFileCache.get(key); + if (sourceFile && sourceFile.text !== data) { + sourceFileCache.delete(key); + } + } + originalWriteFile.call(host, fileName, data, writeByteOrderMark, onError, sourceFiles); + }; + + // directoryExists + if (originalDirectoryExists && originalCreateDirectory) { + host.directoryExists = directory => { + const key = toPath(directory); + const value = directoryExistsCache.get(key); + if (value !== undefined) return value; + const newValue = originalDirectoryExists.call(host, directory); + directoryExistsCache.set(key, !!newValue); + return newValue; + }; + host.createDirectory = directory => { + const key = toPath(directory); + directoryExistsCache.delete(key); + originalCreateDirectory.call(host, directory); + }; + } + const graph = getGlobalDependencyGraph(); reportBuildQueue(graph); let anyFailed = false; @@ -1232,6 +1324,13 @@ namespace ts { anyFailed = anyFailed || !!(buildResult & BuildResultFlags.AnyErrors); } reportErrorSummary(); + host.readFile = originalReadFile; + host.fileExists = originalFileExists; + host.directoryExists = originalDirectoryExists; + host.createDirectory = originalCreateDirectory; + host.writeFile = originalWriteFile; + readFileWithCache = savedReadFileWithCache; + host.getSourceFile = originalGetSourceFile; return anyFailed ? ExitStatus.DiagnosticsPresent_OutputsSkipped : ExitStatus.Success; } diff --git a/src/compiler/types.ts b/src/compiler/types.ts index b4c405c31742a..17444ed97040f 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -5014,6 +5014,9 @@ namespace ts { /* @internal */ hasInvalidatedResolution?: HasInvalidatedResolution; /* @internal */ hasChangedAutomaticTypeDirectiveNames?: boolean; createHash?(data: string): string; + + // TODO: later handle this in better way in builder host instead once the ap + /*@internal*/createDirectory?(directory: string): void; } /* @internal */ From c7f8959034f245d00579ffff5e2b86b4d3c9aed3 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Wed, 21 Nov 2018 09:03:26 -0800 Subject: [PATCH 2/3] PR feedback. --- src/compiler/program.ts | 10 +++++----- src/compiler/tsbuild.ts | 2 +- src/compiler/types.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 24eeff36c6cbb..118850ab73747 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -83,7 +83,7 @@ namespace ts { let text: string | undefined; try { performance.mark("beforeIORead"); - text = host.readFile(fileName); + text = compilerHost.readFile(fileName); performance.mark("afterIORead"); performance.measure("I/O Read", "beforeIORead", "afterIORead"); } @@ -112,8 +112,8 @@ namespace ts { if (directoryPath.length > getRootLength(directoryPath) && !directoryExists(directoryPath)) { const parentDirectory = getDirectoryPath(directoryPath); ensureDirectoriesExist(parentDirectory); - if (host.createDirectory) { - host.createDirectory(directoryPath); + if (compilerHost.createDirectory) { + compilerHost.createDirectory(directoryPath); } else { system.createDirectory(directoryPath); @@ -181,7 +181,7 @@ namespace ts { const newLine = getNewLineCharacter(options, () => system.newLine); const realpath = system.realpath && ((path: string) => system.realpath!(path)); - const host: CompilerHost = { + const compilerHost: CompilerHost = { getSourceFile, getDefaultLibLocation, getDefaultLibFileName: options => combinePaths(getDefaultLibLocation(), getDefaultLibFileName(options)), @@ -200,7 +200,7 @@ namespace ts { readDirectory: (path, extensions, include, exclude, depth) => system.readDirectory(path, extensions, include, exclude, depth), createDirectory: d => system.createDirectory(d) }; - return host; + return compilerHost; } export function getPreEmitDiagnostics(program: Program, sourceFile?: SourceFile, cancellationToken?: CancellationToken): ReadonlyArray { diff --git a/src/compiler/tsbuild.ts b/src/compiler/tsbuild.ts index cd3ed158995e2..b0f7dd7ac3c5d 100644 --- a/src/compiler/tsbuild.ts +++ b/src/compiler/tsbuild.ts @@ -1230,7 +1230,7 @@ namespace ts { return sourceFile; }; - // fileExits for any kind of extension + // fileExists for any kind of extension host.fileExists = fileName => { const key = toPath(fileName); const value = fileExistsCache.get(key); diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 17444ed97040f..495465f70326c 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -5015,7 +5015,7 @@ namespace ts { /* @internal */ hasChangedAutomaticTypeDirectiveNames?: boolean; createHash?(data: string): string; - // TODO: later handle this in better way in builder host instead once the ap + // TODO: later handle this in better way in builder host instead once the api for tsbuild finalizes and doesnt use compilerHost as base /*@internal*/createDirectory?(directory: string): void; } From 8d5d9005679901d3e68e63b970fe44ac2ab8c0f6 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Wed, 21 Nov 2018 09:18:03 -0800 Subject: [PATCH 3/3] Factor out caching logic so tsc (without watch can use it and --watch has its own cache logic). --- src/compiler/program.ts | 108 ++++++++++++++++++++++++++++++++++++++++ src/compiler/tsbuild.ts | 94 +++------------------------------- src/tsc/tsc.ts | 3 ++ 3 files changed, 117 insertions(+), 88 deletions(-) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 118850ab73747..1789b161e39d9 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -203,6 +203,114 @@ namespace ts { return compilerHost; } + /*@internal*/ + export function changeCompilerHostToUseCache( + host: CompilerHost, + toPath: (fileName: string) => Path, + useCacheForSourceFile: boolean + ) { + const originalReadFile = host.readFile; + const originalFileExists = host.fileExists; + const originalDirectoryExists = host.directoryExists; + const originalCreateDirectory = host.createDirectory; + const originalWriteFile = host.writeFile; + const originalGetSourceFile = host.getSourceFile; + const readFileCache = createMap(); + const fileExistsCache = createMap(); + const directoryExistsCache = createMap(); + const sourceFileCache = createMap(); + + const readFileWithCache = (fileName: string): string | undefined => { + const key = toPath(fileName); + const value = readFileCache.get(key); + if (value !== undefined) return value || undefined; + return setReadFileCache(key, fileName); + }; + const setReadFileCache = (key: Path, fileName: string) => { + const newValue = originalReadFile.call(host, fileName); + readFileCache.set(key, newValue || false); + return newValue; + }; + host.readFile = fileName => { + const key = toPath(fileName); + const value = readFileCache.get(key); + if (value !== undefined) return value; // could be .d.ts from output + if (!fileExtensionIs(fileName, Extension.Json)) { + return originalReadFile.call(host, fileName); + } + + return setReadFileCache(key, fileName); + }; + + if (useCacheForSourceFile) { + host.getSourceFile = (fileName, languageVersion, onError, shouldCreateNewSourceFile) => { + const key = toPath(fileName); + const value = sourceFileCache.get(key); + if (value) return value; + + const sourceFile = originalGetSourceFile.call(host, fileName, languageVersion, onError, shouldCreateNewSourceFile); + if (sourceFile && (isDeclarationFileName(fileName) || fileExtensionIs(fileName, Extension.Json))) { + sourceFileCache.set(key, sourceFile); + } + return sourceFile; + }; + } + + // fileExists for any kind of extension + host.fileExists = fileName => { + const key = toPath(fileName); + const value = fileExistsCache.get(key); + if (value !== undefined) return value; + const newValue = originalFileExists.call(host, fileName); + fileExistsCache.set(key, !!newValue); + return newValue; + }; + host.writeFile = (fileName, data, writeByteOrderMark, onError, sourceFiles) => { + const key = toPath(fileName); + fileExistsCache.delete(key); + + const value = readFileCache.get(key); + if (value && value !== data) { + readFileCache.delete(key); + sourceFileCache.delete(key); + } + else if (useCacheForSourceFile) { + const sourceFile = sourceFileCache.get(key); + if (sourceFile && sourceFile.text !== data) { + sourceFileCache.delete(key); + } + } + originalWriteFile.call(host, fileName, data, writeByteOrderMark, onError, sourceFiles); + }; + + // directoryExists + if (originalDirectoryExists && originalCreateDirectory) { + host.directoryExists = directory => { + const key = toPath(directory); + const value = directoryExistsCache.get(key); + if (value !== undefined) return value; + const newValue = originalDirectoryExists.call(host, directory); + directoryExistsCache.set(key, !!newValue); + return newValue; + }; + host.createDirectory = directory => { + const key = toPath(directory); + directoryExistsCache.delete(key); + originalCreateDirectory.call(host, directory); + }; + } + + return { + originalReadFile, + originalFileExists, + originalDirectoryExists, + originalCreateDirectory, + originalWriteFile, + originalGetSourceFile, + readFileWithCache + }; + } + export function getPreEmitDiagnostics(program: Program, sourceFile?: SourceFile, cancellationToken?: CancellationToken): ReadonlyArray { const diagnostics = [ ...program.getConfigFileParsingDiagnostics(), diff --git a/src/compiler/tsbuild.ts b/src/compiler/tsbuild.ts index b0f7dd7ac3c5d..2b8cac3ea57d2 100644 --- a/src/compiler/tsbuild.ts +++ b/src/compiler/tsbuild.ts @@ -1183,96 +1183,14 @@ namespace ts { function buildAllProjects(): ExitStatus { if (options.watch) { reportWatchStatus(Diagnostics.Starting_compilation_in_watch_mode); } - const originalReadFile = host.readFile; - const originalFileExists = host.fileExists; - const originalDirectoryExists = host.directoryExists; - const originalCreateDirectory = host.createDirectory; - const originalWriteFile = host.writeFile; - const originalGetSourceFile = host.getSourceFile; - const readFileCache = createMap(); - const fileExistsCache = createMap(); - const directoryExistsCache = createMap(); - const sourceFileCache = createMap(); - const savedReadFileWithCache = readFileWithCache; - // TODO:: In watch mode as well to use caches for incremental build once we can invalidate caches correctly and have right api // Override readFile for json files and output .d.ts to cache the text - readFileWithCache = fileName => { - const key = toPath(fileName); - const value = readFileCache.get(key); - if (value !== undefined) return value || undefined; - return setReadFileCache(key, fileName); - }; - const setReadFileCache = (key: Path, fileName: string) => { - const newValue = originalReadFile.call(host, fileName); - readFileCache.set(key, newValue || false); - return newValue; - }; - host.readFile = fileName => { - const key = toPath(fileName); - const value = readFileCache.get(key); - if (value !== undefined) return value; // could be .d.ts from output - if (!fileExtensionIs(fileName, Extension.Json)) { - return originalReadFile.call(host, fileName); - } - - return setReadFileCache(key, fileName); - }; - host.getSourceFile = (fileName, languageVersion, onError, shouldCreateNewSourceFile) => { - const key = toPath(fileName); - const value = sourceFileCache.get(key); - if (value) return value; - - const sourceFile = originalGetSourceFile.call(host, fileName, languageVersion, onError, shouldCreateNewSourceFile); - if (sourceFile && (isDeclarationFileName(fileName) || fileExtensionIs(fileName, Extension.Json))) { - sourceFileCache.set(key, sourceFile); - } - return sourceFile; - }; - - // fileExists for any kind of extension - host.fileExists = fileName => { - const key = toPath(fileName); - const value = fileExistsCache.get(key); - if (value !== undefined) return value; - const newValue = originalFileExists.call(host, fileName); - fileExistsCache.set(key, !!newValue); - return newValue; - }; - host.writeFile = (fileName, data, writeByteOrderMark, onError, sourceFiles) => { - const key = toPath(fileName); - fileExistsCache.delete(key); - - const value = readFileCache.get(key); - if (value && value !== data) { - readFileCache.delete(key); - sourceFileCache.delete(key); - } - else { - const sourceFile = sourceFileCache.get(key); - if (sourceFile && sourceFile.text !== data) { - sourceFileCache.delete(key); - } - } - originalWriteFile.call(host, fileName, data, writeByteOrderMark, onError, sourceFiles); - }; - - // directoryExists - if (originalDirectoryExists && originalCreateDirectory) { - host.directoryExists = directory => { - const key = toPath(directory); - const value = directoryExistsCache.get(key); - if (value !== undefined) return value; - const newValue = originalDirectoryExists.call(host, directory); - directoryExistsCache.set(key, !!newValue); - return newValue; - }; - host.createDirectory = directory => { - const key = toPath(directory); - directoryExistsCache.delete(key); - originalCreateDirectory.call(host, directory); - }; - } + const { originalReadFile, originalFileExists, originalDirectoryExists, + originalCreateDirectory, originalWriteFile, originalGetSourceFile, + readFileWithCache: newReadFileWithCache + } = changeCompilerHostToUseCache(host, toPath, /*useCacheForSourceFile*/ true); + const savedReadFileWithCache = readFileWithCache; + readFileWithCache = newReadFileWithCache; const graph = getGlobalDependencyGraph(); reportBuildQueue(graph); diff --git a/src/tsc/tsc.ts b/src/tsc/tsc.ts index 8d9b2f1afa7cd..ec3da718ef4a3 100644 --- a/src/tsc/tsc.ts +++ b/src/tsc/tsc.ts @@ -227,6 +227,9 @@ namespace ts { function performCompilation(rootNames: string[], projectReferences: ReadonlyArray | undefined, options: CompilerOptions, configFileParsingDiagnostics?: ReadonlyArray) { const host = createCompilerHost(options); + const currentDirectory = host.getCurrentDirectory(); + const getCanonicalFileName = createGetCanonicalFileName(host.useCaseSensitiveFileNames()); + changeCompilerHostToUseCache(host, fileName => toPath(fileName, currentDirectory, getCanonicalFileName), /*useCacheForSourceFile*/ false); enableStatistics(options); const programOptions: CreateProgramOptions = {