From 5c931c9ca42baf2564f62ddacf2fd6b92e020086 Mon Sep 17 00:00:00 2001 From: Brad Zacher Date: Fri, 18 Oct 2019 10:50:33 -0700 Subject: [PATCH 1/2] feat: use createProgram backed by a local source file cache --- packages/typescript-estree/jest.config.js | 6 +- .../src/host/ProgramCache.ts | 138 ++++++++++++++ .../src/host/SourceFileCache.ts | 168 ++++++++++++++++++ packages/typescript-estree/src/host/common.ts | 39 ++++ .../src/host/createProgramWithProject.ts | 38 ++++ .../src/host/parseConfigFile.ts | 31 ++++ .../fixtures/multiFileProject/src/file1.ts | 1 + .../fixtures/multiFileProject/src/file2.ts | 1 + .../fixtures/multiFileProject/src/file3.ts | 1 + .../fixtures/multiFileProject/tsconfig.json | 3 + .../tests/host/createProgramWithProject.ts | 80 +++++++++ 11 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 packages/typescript-estree/src/host/ProgramCache.ts create mode 100644 packages/typescript-estree/src/host/SourceFileCache.ts create mode 100644 packages/typescript-estree/src/host/common.ts create mode 100644 packages/typescript-estree/src/host/createProgramWithProject.ts create mode 100644 packages/typescript-estree/src/host/parseConfigFile.ts create mode 100644 packages/typescript-estree/tests/fixtures/multiFileProject/src/file1.ts create mode 100644 packages/typescript-estree/tests/fixtures/multiFileProject/src/file2.ts create mode 100644 packages/typescript-estree/tests/fixtures/multiFileProject/src/file3.ts create mode 100644 packages/typescript-estree/tests/fixtures/multiFileProject/tsconfig.json create mode 100644 packages/typescript-estree/tests/host/createProgramWithProject.ts diff --git a/packages/typescript-estree/jest.config.js b/packages/typescript-estree/jest.config.js index e01f6ed07751..b8051744d7bc 100644 --- a/packages/typescript-estree/jest.config.js +++ b/packages/typescript-estree/jest.config.js @@ -5,7 +5,11 @@ module.exports = { transform: { '^.+\\.tsx?$': 'ts-jest', }, - testRegex: './tests/(lib/.*\\.(jsx?|tsx?)|ast-alignment/spec\\.ts)$', + testRegex: [ + './tests/host/.*\\.(jsx?|tsx?)$', + './tests/lib/.*\\.(jsx?|tsx?)$', + './tests/ast-alignment/spec\\.ts$', + ], collectCoverage: false, collectCoverageFrom: ['src/**/*.{js,jsx,ts,tsx}'], moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], diff --git a/packages/typescript-estree/src/host/ProgramCache.ts b/packages/typescript-estree/src/host/ProgramCache.ts new file mode 100644 index 000000000000..febfc5b5268d --- /dev/null +++ b/packages/typescript-estree/src/host/ProgramCache.ts @@ -0,0 +1,138 @@ +import path from 'path'; +import ts from 'typescript'; +import { + getCanonicalFileName, + noop, + NormalisedTSConfigPath, + normaliseTSConfigPath, + TSConfigPath, +} from './common'; +import { parseConfigFile } from './parseConfigFile'; +import { SourceFileCache } from './SourceFileCache'; + +interface CachedProgram { + invalidated: boolean; + program?: ts.Program; +} + +class ProgramCache { + private readonly programCache = new Map< + NormalisedTSConfigPath, + CachedProgram + >(); + private readonly sourceFileCache = new SourceFileCache(); + + /** + * Gets the current program for the given tsconfig path + */ + public getProgram(tsconfigPath: TSConfigPath): CachedProgram { + const normalisedTSConfigPath = normaliseTSConfigPath(tsconfigPath); + return this.getProgramNormalised(normalisedTSConfigPath); + } + private getProgramNormalised( + tsconfigPath: NormalisedTSConfigPath, + ): CachedProgram { + const entry = this.programCache.get(tsconfigPath); + if (!entry) { + return { invalidated: true }; + } + + return entry; + } + + public createProgram( + tsconfigPath: TSConfigPath, + oldProgram?: ts.Program, + ): ts.Program { + const normalisedTSConfigPath = normaliseTSConfigPath(tsconfigPath); + const tsconfig = parseConfigFile(normalisedTSConfigPath); + + this.sourceFileCache.registerTSConfig(tsconfig, normalisedTSConfigPath); + + const createProgramOptions = this.createProgramOptions( + tsconfig, + normalisedTSConfigPath, + ); + createProgramOptions.oldProgram = oldProgram; + const program = ts.createProgram(createProgramOptions); + + const entry: CachedProgram = { + program, + invalidated: false, + }; + this.programCache.set(normalisedTSConfigPath, entry); + + return program; + } + + private createProgramOptions( + tsconfig: ts.ParsedCommandLine, + tsconfigPath: NormalisedTSConfigPath, + ): ts.CreateProgramOptions { + const compilerHost: ts.CompilerHost = { + getSourceFile: fileName => + this.sourceFileCache.getSourceFile(fileName, tsconfigPath), + getSourceFileByPath: (fileName, filePath) => + this.sourceFileCache.getSourceFileByPath( + fileName, + filePath, + tsconfigPath, + ), + getCanonicalFileName, + useCaseSensitiveFileNames: () => ts.sys.useCaseSensitiveFileNames, + getNewLine: () => ts.sys.newLine, + getDefaultLibFileName: options => { + return path.join( + path.dirname(path.normalize(ts.sys.getExecutingFilePath())), + ts.getDefaultLibFileName(options), + ); + }, + writeFile: noop, + getCurrentDirectory: ts.sys.getCurrentDirectory, + fileExists: filePath => + this.sourceFileCache.fileExists(filePath, tsconfigPath), + readFile: filePath => + this.sourceFileCache.readFile(filePath, tsconfigPath), + realpath: ts.sys.realpath, + directoryExists: ts.sys.directoryExists, + getDirectories: ts.sys.getDirectories, + readDirectory: ts.sys.readDirectory, + // eslint-disable-next-line @typescript-eslint/ban-ts-ignore + // @ts-ignore - interal TS api - https://github.com/microsoft/TypeScript/blob/dd58bfc51417d0a037ea75fe8e8cd690ee9950eb/src/compiler/types.ts#L5469 + onReleaseOldSourceFile: this.sourceFileCache.onReleaseSourceFile, + }; + + const createProgramOptions: ts.CreateProgramOptions = { + host: compilerHost, + options: tsconfig.options, + rootNames: tsconfig.fileNames, + }; + + return createProgramOptions; + } + + public createOrUpdateSourceFile( + contents: string, + filePath: string, + tsconfigPath: string, + ): void { + const normalisedTSConfigPath = normaliseTSConfigPath(tsconfigPath); + if ( + this.sourceFileCache.createOrUpdateSourceFile( + filePath, + normalisedTSConfigPath, + contents, + ) + ) { + // invalidate the program because the source code changed + this.getProgramNormalised(normalisedTSConfigPath).invalidated = true; + } + } + + public clear(): void { + this.programCache.clear(); + this.sourceFileCache.clear(); + } +} + +export { ProgramCache }; diff --git a/packages/typescript-estree/src/host/SourceFileCache.ts b/packages/typescript-estree/src/host/SourceFileCache.ts new file mode 100644 index 000000000000..6e04432262e5 --- /dev/null +++ b/packages/typescript-estree/src/host/SourceFileCache.ts @@ -0,0 +1,168 @@ +import path from 'path'; +import ts from 'typescript'; +import { normaliseFilePath, NormalisedTSConfigPath, toPath } from './common'; + +interface CachedSourceFile { + sourceFile: ts.SourceFile; + refProjects: Set; +} + +class SourceFileCache { + private readonly sourceFileCache = new Map(); + + public registerTSConfig( + tsconfig: ts.ParsedCommandLine, + tsconfigPath: NormalisedTSConfigPath, + ): void { + tsconfig.fileNames.forEach(fileName => { + if (!this.getSourceFile(fileName, tsconfigPath)) { + this.createSourceFile(fileName, tsconfigPath); + } + }); + } + + /** + * @returns true if the file was created/updated, false if there were no changes + */ + public createOrUpdateSourceFile( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, + contents: string, + ): boolean { + const filePath = normaliseFilePath(fileName, tsconfigPath); + if (this.sourceFileCache.has(filePath)) { + return this.updateSourceFile(fileName, tsconfigPath, contents); + } + + this.createSourceFile(fileName, tsconfigPath, contents); + return true; + } + + private createSourceFile( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, + contents?: string, + ): void { + const filePath = normaliseFilePath(fileName, tsconfigPath); + contents = contents || ts.sys.readFile(filePath); + if (!contents) { + throw new Error(`Unable to find file ${filePath}`); + } + + const sourceFile = ts.createSourceFile( + fileName, + contents, + ts.ScriptTarget.ESNext, + ); + + this.sourceFileCache.set(filePath, { + sourceFile, + refProjects: new Set([tsconfigPath]), + }); + } + + /** + * @returns true if the file was updated, false otherwise + */ + private updateSourceFile( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, + contents: string, + ): boolean { + const filePath = normaliseFilePath(fileName, tsconfigPath); + const entry = this.sourceFileCache.get(filePath); + if (!entry) { + throw new Error( + `Attempting to update file that is not in the cache: ${filePath}`, + ); + } + + if (entry.sourceFile.getText() === contents) { + return false; + } + + const sourceFile = ts.createSourceFile( + fileName, + contents, + ts.ScriptTarget.ESNext, + ); + + this.sourceFileCache.set(filePath, { + sourceFile, + refProjects: new Set([...entry.refProjects, tsconfigPath]), + }); + return true; + } + + public fileExists( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, + ): boolean { + return this.sourceFileCache.has(normaliseFilePath(fileName, tsconfigPath)); + } + + public readFile( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, + ): string | undefined { + const filePath = normaliseFilePath(fileName, tsconfigPath); + const cache = this.sourceFileCache.get(filePath); + if (cache) { + if (typeof cache === 'string') { + return cache; + } + return cache.sourceFile.getText(); + } + + return ts.sys.readFile(filePath); + } + + public getSourceFile( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, + ): ts.SourceFile | undefined { + return this.getSourceFileByPath( + fileName, + toPath(fileName, path.dirname(tsconfigPath)), + tsconfigPath, + ); + } + + public getSourceFileByPath( + _fileName: string, + filePath: ts.Path, + tsconfigPath: NormalisedTSConfigPath, + ): ts.SourceFile | undefined { + const entry = this.sourceFileCache.get(filePath); + if (!entry) { + return undefined; + } + + entry.refProjects.add(tsconfigPath); + + return entry.sourceFile; + } + + public onReleaseSourceFile( + oldSourceFile: ts.SourceFile, + tsconfigPath: NormalisedTSConfigPath, + ): void { + const filePath = normaliseFilePath(oldSourceFile.fileName, tsconfigPath); + const entry = this.sourceFileCache.get(filePath); + + if (!entry) { + return; + } + + entry.refProjects.delete(tsconfigPath); + if (entry.refProjects.size <= 0) { + this.sourceFileCache.delete(filePath); + } + } + + public clear(): void { + this.sourceFileCache.clear(); + } +} + +export { SourceFileCache }; diff --git a/packages/typescript-estree/src/host/common.ts b/packages/typescript-estree/src/host/common.ts new file mode 100644 index 000000000000..3563526fefc5 --- /dev/null +++ b/packages/typescript-estree/src/host/common.ts @@ -0,0 +1,39 @@ +import path from 'path'; +import ts from 'typescript'; + +const getCanonicalFileName = ts.sys.useCaseSensitiveFileNames + ? (path: string): string => path + : (path: string): string => path.toLowerCase(); + +const noop = (): void => {}; + +type TSConfigPath = string; +type NormalisedTSConfigPath = string & { __pathBrand: never }; +function normaliseTSConfigPath( + tsconfigPath: TSConfigPath, +): NormalisedTSConfigPath { + return path.normalize(tsconfigPath) as NormalisedTSConfigPath; +} + +function normaliseFilePath( + fileName: string, + tsconfigPath: NormalisedTSConfigPath, +): ts.Path { + return toPath(fileName, path.dirname(tsconfigPath)); +} + +function toPath(fileName: string, basePath: string | undefined): ts.Path { + // eslint-disable-next-line @typescript-eslint/ban-ts-ignore + // @ts-ignore - internal typescript API - https://github.com/microsoft/TypeScript/blob/dd58bfc51417d0a037ea75fe8e8cd690ee9950eb/src/compiler/path.ts#L554-L559 + return ts.toPath(fileName, basePath, getCanonicalFileName); +} + +export { + getCanonicalFileName, + noop, + toPath, + normaliseFilePath, + NormalisedTSConfigPath, + normaliseTSConfigPath, + TSConfigPath, +}; diff --git a/packages/typescript-estree/src/host/createProgramWithProject.ts b/packages/typescript-estree/src/host/createProgramWithProject.ts new file mode 100644 index 000000000000..f6818b69f6cc --- /dev/null +++ b/packages/typescript-estree/src/host/createProgramWithProject.ts @@ -0,0 +1,38 @@ +import ts from 'typescript'; +import { ProgramCache } from './ProgramCache'; + +const programCache = new ProgramCache(); + +/** + * @param filePath the path to the file to get the program for. Must either be an absolute path, or a path relative to tsconfigPath. + * @param tsconfigPath the path to the tsconfig file. Must either be an absolute path, or a path relative to the CWD. + */ +function createProgramWithProject( + code: string, + filePath: string, + tsconfigPath: string, +): ts.Program { + programCache.createOrUpdateSourceFile(code, filePath, tsconfigPath); + + // we have a cached program that can be used + const cachedProgram = programCache.getProgram(tsconfigPath); + if (cachedProgram.program && !cachedProgram.invalidated) { + return cachedProgram.program; + } + + const program = programCache.createProgram( + tsconfigPath, + cachedProgram.program, + ); + + return program; +} + +/** + * Provided for testing use cases only + */ +function clearCache(): void { + programCache.clear(); +} + +export { createProgramWithProject, clearCache }; diff --git a/packages/typescript-estree/src/host/parseConfigFile.ts b/packages/typescript-estree/src/host/parseConfigFile.ts new file mode 100644 index 000000000000..2002ccd14c9e --- /dev/null +++ b/packages/typescript-estree/src/host/parseConfigFile.ts @@ -0,0 +1,31 @@ +import path from 'path'; +import ts from 'typescript'; + +type RawJsonConfig = + | ts.TsConfigSourceFile + // internal typescript API - https://github.com/microsoft/TypeScript/blob/1bfc47252fa2c02416dec8417723e247faae466d/src/compiler/types.ts#L2893 + | { parseDiagnostics: { messageText: string }[] }; + +function parseConfigFile(tsconfigPath: string): ts.ParsedCommandLine { + const rawConfigSource = ts.readJsonConfigFile(tsconfigPath, path => + ts.sys.readFile(path), + ) as RawJsonConfig; + if ( + 'parseDiagnostics' in rawConfigSource && + rawConfigSource.parseDiagnostics && + rawConfigSource.parseDiagnostics.length > 0 + ) { + throw new Error(rawConfigSource.parseDiagnostics[0].messageText); + } + + const configFile = ts.parseJsonSourceFileConfigFileContent( + rawConfigSource as ts.TsConfigSourceFile, + ts.sys, + path.dirname(tsconfigPath), + {}, // TODO - extended options + ); + + return configFile; +} + +export { parseConfigFile }; diff --git a/packages/typescript-estree/tests/fixtures/multiFileProject/src/file1.ts b/packages/typescript-estree/tests/fixtures/multiFileProject/src/file1.ts new file mode 100644 index 000000000000..8b95882897a9 --- /dev/null +++ b/packages/typescript-estree/tests/fixtures/multiFileProject/src/file1.ts @@ -0,0 +1 @@ +const file1 = true; diff --git a/packages/typescript-estree/tests/fixtures/multiFileProject/src/file2.ts b/packages/typescript-estree/tests/fixtures/multiFileProject/src/file2.ts new file mode 100644 index 000000000000..fb8b56e9cd6e --- /dev/null +++ b/packages/typescript-estree/tests/fixtures/multiFileProject/src/file2.ts @@ -0,0 +1 @@ +const file2 = true; diff --git a/packages/typescript-estree/tests/fixtures/multiFileProject/src/file3.ts b/packages/typescript-estree/tests/fixtures/multiFileProject/src/file3.ts new file mode 100644 index 000000000000..9459bc9ba765 --- /dev/null +++ b/packages/typescript-estree/tests/fixtures/multiFileProject/src/file3.ts @@ -0,0 +1 @@ +const file3 = true; diff --git a/packages/typescript-estree/tests/fixtures/multiFileProject/tsconfig.json b/packages/typescript-estree/tests/fixtures/multiFileProject/tsconfig.json new file mode 100644 index 000000000000..bcd8df0e91cb --- /dev/null +++ b/packages/typescript-estree/tests/fixtures/multiFileProject/tsconfig.json @@ -0,0 +1,3 @@ +{ + "include": ["src"] +} diff --git a/packages/typescript-estree/tests/host/createProgramWithProject.ts b/packages/typescript-estree/tests/host/createProgramWithProject.ts new file mode 100644 index 000000000000..812c4a85ce33 --- /dev/null +++ b/packages/typescript-estree/tests/host/createProgramWithProject.ts @@ -0,0 +1,80 @@ +import path from 'path'; +import { + clearCache, + createProgramWithProject, +} from '../../src/host/createProgramWithProject'; +import { Program } from 'typescript'; + +afterEach(() => { + clearCache(); +}); + +describe('createProgramWithProject', () => { + describe('simple project', () => { + const FIXTURES_DIR = `${process.cwd()}/tests/fixtures/simpleProject`; + const FILE_PATH = './file.ts'; + const tsconfigPath = `${FIXTURES_DIR}/tsconfig.json`; + + function parse(code = `const x = ${Math.random()};`): Program { + const program = createProgramWithProject(code, FILE_PATH, tsconfigPath); + expect(program).not.toBeUndefined(); + const sourceFile = program.getSourceFile( + path.resolve(FIXTURES_DIR, FILE_PATH), + ); + expect(sourceFile).not.toBeUndefined(); + expect(sourceFile!.getText()).toEqual(code); + + return program; + } + it('should parse a file', () => { + parse(); + }); + + it('should handle local updates to an existing file', () => { + parse(); + parse(); + }); + + it('should not generate a new project if the file contents do not change', () => { + const oldProg = parse('const x = 1;'); + const newProg = parse('const x = 1;'); + + expect(oldProg).toBe(newProg); + }); + }); + + describe('multi-file project', () => { + const FIXTURES_DIR = `${process.cwd()}/tests/fixtures/multiFileProject`; + const tsconfigPath = `${FIXTURES_DIR}/tsconfig.json`; + + function parse( + filePath: string, + code = `const x = ${Math.random()};`, + ): void { + const program = createProgramWithProject(code, filePath, tsconfigPath); + expect(program).not.toBeUndefined(); + const sourceFile = program.getSourceFile( + path.resolve(FIXTURES_DIR, filePath), + ); + expect(sourceFile).not.toBeUndefined(); + expect(sourceFile!.getText()).toEqual(code); + } + it('should parse a file', () => { + parse('./src/file1.ts'); + }); + + it('should handle local updates to the file', () => { + parse('./src/file1.ts'); + parse('./src/file1.ts'); + }); + + it('should handle multiple files', () => { + const file1 = `./src/file1.ts`; + parse(file1); + const file2 = `./src/file2.ts`; + parse(file2); + const file3 = `./src/file2.ts`; + parse(file3); + }); + }); +}); From 77dbf562204464a656acb136674836865bf23d4e Mon Sep 17 00:00:00 2001 From: Brad Zacher Date: Fri, 18 Oct 2019 11:04:19 -0700 Subject: [PATCH 2/2] chore: schedule automatic invalidation --- .../src/host/ProgramCache.ts | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/typescript-estree/src/host/ProgramCache.ts b/packages/typescript-estree/src/host/ProgramCache.ts index febfc5b5268d..ad4602af4cc9 100644 --- a/packages/typescript-estree/src/host/ProgramCache.ts +++ b/packages/typescript-estree/src/host/ProgramCache.ts @@ -13,8 +13,11 @@ import { SourceFileCache } from './SourceFileCache'; interface CachedProgram { invalidated: boolean; program?: ts.Program; + timeoutId?: NodeJS.Timeout; } +const INVALIDATION_TIMEOUT_MS = 10; + class ProgramCache { private readonly programCache = new Map< NormalisedTSConfigPath, @@ -45,6 +48,9 @@ class ProgramCache { oldProgram?: ts.Program, ): ts.Program { const normalisedTSConfigPath = normaliseTSConfigPath(tsconfigPath); + // ensure the old program is invalidated if it exists + this.invalidateProgram(this.getProgramNormalised(normalisedTSConfigPath)); + const tsconfig = parseConfigFile(normalisedTSConfigPath); this.sourceFileCache.registerTSConfig(tsconfig, normalisedTSConfigPath); @@ -61,6 +67,8 @@ class ProgramCache { invalidated: false, }; this.programCache.set(normalisedTSConfigPath, entry); + // schedule the invalidation of the program so we can handle newly created files + this.scheduleInvalidateProgram(entry); return program; } @@ -125,7 +133,7 @@ class ProgramCache { ) ) { // invalidate the program because the source code changed - this.getProgramNormalised(normalisedTSConfigPath).invalidated = true; + this.invalidateProgram(this.getProgramNormalised(normalisedTSConfigPath)); } } @@ -133,6 +141,32 @@ class ProgramCache { this.programCache.clear(); this.sourceFileCache.clear(); } + + private scheduleInvalidateProgram(entry: CachedProgram): void { + if (entry.invalidated) { + return; + } + + if (entry.timeoutId !== undefined) { + clearTimeout(entry.timeoutId); + } + + entry.timeoutId = setTimeout(() => { + this.invalidateProgram(entry); + }, INVALIDATION_TIMEOUT_MS); + } + private invalidateProgram(entry: CachedProgram): void { + if (entry.invalidated) { + return; + } + + if (entry.timeoutId !== undefined) { + clearTimeout(entry.timeoutId); + } + + entry.invalidated = true; + delete entry.timeoutId; + } } export { ProgramCache };