diff --git a/.eslintignore b/.eslintignore index 431bc8030293..734c0a7cab86 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,6 +6,7 @@ coverage __snapshots__ .docusaurus build +packages/performance/generated # Files copied as part of the build packages/types/src/generated/**/*.ts diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000000..5dc93b5c5171 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,23 @@ +name: 'Performance Testing' + +on: workflow_dispatch + +jobs: + comparison: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/prepare-install + - run: brew install hyperfine + - run: PERFORMANCE_FILE_COUNT=${{ matrix.file-count }} yarn workspace @typescript-eslint/performance run generate + - run: hyperfine yarn workspace @typescript-eslint/performance run test:${{ matrix.test}} + strategy: + matrix: + file-count: [50, 100, 150, 200] + test: ['project', 'service'] + summary: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/prepare-install + - run: yarn workspace @typescript-eslint/performance run test:summary diff --git a/.gitignore b/.gitignore index d3dbc7cdd6aa..7793b3f87897 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ npm-debug.log* yarn-debug.log* yarn-error.log* +# Performance testing +packages/performance/generated + # Website packages/website/.docusaurus packages/website/.cache-loader @@ -95,4 +98,4 @@ packages/types/src/generated/**/*.ts !.yarn/sdks !.yarn/versions -.nx/cache \ No newline at end of file +.nx/cache diff --git a/packages/performance/.eslintrc.cjs b/packages/performance/.eslintrc.cjs new file mode 100644 index 000000000000..d2b3123e6c8f --- /dev/null +++ b/packages/performance/.eslintrc.cjs @@ -0,0 +1,15 @@ +/* eslint-env node */ +module.exports = { + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/strict-type-checked', + 'plugin:@typescript-eslint/stylistic-type-checked', + ], + plugins: ['@typescript-eslint'], + parser: '@typescript-eslint/parser', + parserOptions: { + project: true, + tsconfigRootDir: __dirname, + }, + root: true, +}; diff --git a/packages/performance/README.md b/packages/performance/README.md new file mode 100644 index 000000000000..9daed88615d8 --- /dev/null +++ b/packages/performance/README.md @@ -0,0 +1,27 @@ +# Performance Testing + +A quick internal utility for testing lint performance. + +## Generating a Summary + +`test:summary` runs the three performance scenarios under different file counts, then prints a summary table: + +```shell +yarn test:summary +``` + +For each of those file counts, the generated code is 50% `.js` and 50% `.ts`. + +## Running Tests Manually + +First generate the number of files you'd like to test: + +```shell +PERFORMANCE_FILE_COUNT=123 yarn generate +``` + +Then run one or more of the test types: + +- `yarn test:project`: Traditional `project: true` +- `yarn test:service` `EXPERIMENTAL_useProjectService` +- `yarn test:service:seeded`: `EXPERIMENTAL_useProjectService`, with `allowDefaultProjectFallbackFilesGlobs` diff --git a/packages/performance/generate.ts b/packages/performance/generate.ts new file mode 100644 index 000000000000..787f4ed78576 --- /dev/null +++ b/packages/performance/generate.ts @@ -0,0 +1,56 @@ +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import { + generateESLintConfig, + generateJsFile, + generateTSConfig, + generateTsFile, +} from './generating.js'; +import { directory, writeFile } from './writing.js'; + +const fileCount = Number(process.env.PERFORMANCE_FILE_COUNT); +if (!fileCount) { + throw new Error( + `Invalid process.env.PERFORMANCE_FILE_COUNT: ${process.env.PERFORMANCE_FILE_COUNT}`, + ); +} + +await fs.rm(directory, { force: true, recursive: true }); +await fs.mkdir(path.join(directory, 'src'), { recursive: true }); + +// The root-level TSConfig is used for the project service, which intentionally +// uses the default/inferred project for non-included JS files. +await writeFile(`tsconfig.json`, generateTSConfig(false)); + +// The explicit project allows JS, akin to the common tsconfig.eslint.json. +await writeFile(`tsconfig.project.json`, generateTSConfig(true)); + +for (const [alias, parserOptions] of [ + ['project', `project: "./tsconfig.project.json"`], + ['service', `EXPERIMENTAL_useProjectService: true`], + [ + 'service.seeded', + `allowDefaultProjectFallbackFilesGlobs: ["./src/*.js"],\n EXPERIMENTAL_useProjectService: true`, + ], +] as const) { + await writeFile( + `.eslintrc.${alias}.cjs`, + generateESLintConfig(parserOptions), + ); +} + +for (let i = 0; i < fileCount; i += 1) { + const [extension, generator] = + i % 2 ? ['js', generateJsFile] : ['ts', generateTsFile]; + + await writeFile(`src/example${i}.${extension}`, generator(i)); +} + +await writeFile( + 'src/index.ts', + new Array(fileCount) + .fill(undefined) + .map((_, i) => `export { example${i} } from "./example${i}.js";`) + .join('\n'), +); diff --git a/packages/performance/generating.ts b/packages/performance/generating.ts new file mode 100644 index 000000000000..3f109c9ebb25 --- /dev/null +++ b/packages/performance/generating.ts @@ -0,0 +1,48 @@ +export function generateESLintConfig(parserOptions: string) { + return ` +/* eslint-env node */ +module.exports = { + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/strict-type-checked', + 'plugin:@typescript-eslint/stylistic-type-checked', + ], + plugins: ['@typescript-eslint'], + parser: '@typescript-eslint/parser', + parserOptions: { + ${parserOptions}, + tsconfigRootDir: __dirname + }, + root: true, +}; +`.trimStart(); +} + +export function generateJsFile(i: number) { + return ` +export async function example${i}(input) { + return typeof input === 'number' ? input.toPrecision(1) : input.toUpperCase(); +} +`.trimStart(); +} + +export function generateTsFile(i: number) { + return ` +export async function example${i}(input: T) { + return typeof input === 'number' ? input.toPrecision(1) : input.toUpperCase(); +} +`.trimStart(); +} + +export function generateTSConfig(allowJs: boolean) { + return ` +{ + "compilerOptions": { + "allowJs": ${allowJs}, + "module": "ESNext", + "strict": true, + "target": "ESNext" + } +} +`.trimStart(); +} diff --git a/packages/performance/package.json b/packages/performance/package.json new file mode 100644 index 000000000000..53b1a5eabc6a --- /dev/null +++ b/packages/performance/package.json @@ -0,0 +1,28 @@ +{ + "name": "@typescript-eslint/performance", + "version": "6.9.1", + "private": true, + "type": "module", + "repository": { + "type": "git", + "url": "https://github.com/typescript-eslint/typescript-eslint.git", + "directory": "packages/performance" + }, + "scripts": { + "generate": "tsx ./generate.ts", + "generate:watch": "tsx --watch ./generate.ts", + "test:project": "time eslint --config ./generated/.eslintrc.project.cjs ./generated || true", + "test:service": "time eslint --config ./generated/.eslintrc.service.cjs ./generated || true", + "test:service:seeded": "time eslint --config ./generated/.eslintrc.service.seeded.cjs ./generated || true", + "test:summary": "tsx ./summary.ts" + }, + "dependencies": { + "@typescript-eslint/eslint-plugin": "6.9.1", + "@typescript-eslint/parser": "6.9.1" + }, + "devDependencies": { + "eslint": "*", + "execa": "*", + "tsx": "*" + } +} diff --git a/packages/performance/summary.ts b/packages/performance/summary.ts new file mode 100644 index 000000000000..3af1d03648d7 --- /dev/null +++ b/packages/performance/summary.ts @@ -0,0 +1,17 @@ +import { $ } from 'execa'; + +const getTiming = ({ stderr }: { stderr: string }) => + stderr.trim().replace(/ +/g, ' '); + +const counts: Record = {}; + +for (const fileCount of [5, 10, 25, 50, 75, 100, 125, 150, 175, 200]) { + await $({ env: { PERFORMANCE_FILE_COUNT: `${fileCount}` } })`yarn generate`; + const project = getTiming(await $`yarn test:project`); + const service = getTiming(await $`yarn test:service`); + const serviceSeeded = getTiming(await $`yarn test:service:seeded`); + + counts[fileCount] = { project, service, 'service (seeded)': serviceSeeded }; +} + +console.table(counts); diff --git a/packages/performance/tsconfig.json b/packages/performance/tsconfig.json new file mode 100644 index 000000000000..236a4968f88f --- /dev/null +++ b/packages/performance/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "target": "ESNext" + } +} diff --git a/packages/performance/writing.ts b/packages/performance/writing.ts new file mode 100644 index 000000000000..ba908e17766a --- /dev/null +++ b/packages/performance/writing.ts @@ -0,0 +1,8 @@ +import * as fs from 'fs/promises'; +import * as path from 'path'; + +export const directory = 'generated'; + +export async function writeFile(fileName: string, data: string) { + await fs.writeFile(path.join(directory, fileName), data); +} diff --git a/packages/typescript-estree/src/create-program/createProjectService.ts b/packages/typescript-estree/src/create-program/createProjectService.ts index 333d221f85b1..3aed2c400841 100644 --- a/packages/typescript-estree/src/create-program/createProjectService.ts +++ b/packages/typescript-estree/src/create-program/createProjectService.ts @@ -1,4 +1,6 @@ /* eslint-disable @typescript-eslint/no-empty-function -- for TypeScript APIs*/ +import * as fs from 'fs'; +import { sync as globSync } from 'globby'; import type * as ts from 'typescript/lib/tsserverlibrary'; const doNothing = (): void => {}; @@ -9,7 +11,9 @@ const createStubFileWatcher = (): ts.FileWatcher => ({ export type TypeScriptProjectService = ts.server.ProjectService; -export function createProjectService(): TypeScriptProjectService { +export function createProjectService( + allowDefaultProjectFallbackFilesGlobs: string[] = [], +): TypeScriptProjectService { // We import this lazily to avoid its cost for users who don't use the service const tsserver = require('typescript/lib/tsserverlibrary') as typeof ts; @@ -27,7 +31,7 @@ export function createProjectService(): TypeScriptProjectService { watchFile: createStubFileWatcher, }; - return new tsserver.server.ProjectService({ + const service = new tsserver.server.ProjectService({ host: system, cancellationToken: { isCancellationRequested: (): boolean => false }, useSingleInferredProject: false, @@ -45,5 +49,13 @@ export function createProjectService(): TypeScriptProjectService { }, session: undefined, }); + + for (const filesGlob of allowDefaultProjectFallbackFilesGlobs) { + for (const fileName of globSync(filesGlob, { ignore: ['node_modules/'] })) { + service.openClientFile(fileName, fs.readFileSync(fileName).toString()); + } + } + + return service; } /* eslint-enable @typescript-eslint/no-empty-function */ diff --git a/packages/typescript-estree/src/parseSettings/createParseSettings.ts b/packages/typescript-estree/src/parseSettings/createParseSettings.ts index 3062a6164b32..de564a7be9e7 100644 --- a/packages/typescript-estree/src/parseSettings/createParseSettings.ts +++ b/packages/typescript-estree/src/parseSettings/createParseSettings.ts @@ -55,7 +55,9 @@ export function createParseSettings( process.env.TYPESCRIPT_ESLINT_EXPERIMENTAL_TSSERVER !== 'false') || (process.env.TYPESCRIPT_ESLINT_EXPERIMENTAL_TSSERVER === 'true' && options.EXPERIMENTAL_useProjectService !== false) - ? (TSSERVER_PROJECT_SERVICE ??= createProjectService()) + ? (TSSERVER_PROJECT_SERVICE ??= createProjectService( + options.allowDefaultProjectFallbackFilesGlobs, + )) : undefined, EXPERIMENTAL_useSourceOfProjectReferenceRedirect: options.EXPERIMENTAL_useSourceOfProjectReferenceRedirect === true, diff --git a/packages/typescript-estree/src/parser-options.ts b/packages/typescript-estree/src/parser-options.ts index 711ef4bca34b..7d510e4068ff 100644 --- a/packages/typescript-estree/src/parser-options.ts +++ b/packages/typescript-estree/src/parser-options.ts @@ -103,6 +103,16 @@ interface ParseAndGenerateServicesOptions extends ParseOptions { */ EXPERIMENTAL_useProjectService?: boolean; + /** + * ***EXPERIMENTAL FLAG*** - Use this at your own risk. + * + * If `EXPERIMENTAL_useProjectService` is true, which files to load into the service immediately. + * Intentionally an annoyingly long name for now. We'll think of a better one if we need. + * + * @see https://github.com/typescript-eslint/typescript-eslint/pull/7870 + */ + allowDefaultProjectFallbackFilesGlobs?: string[]; + /** * ***EXPERIMENTAL FLAG*** - Use this at your own risk. * diff --git a/packages/typescript-estree/src/useProgramFromProjectService.ts b/packages/typescript-estree/src/useProgramFromProjectService.ts index 16a7933a671c..6a233e63e319 100644 --- a/packages/typescript-estree/src/useProgramFromProjectService.ts +++ b/packages/typescript-estree/src/useProgramFromProjectService.ts @@ -10,15 +10,12 @@ export function useProgramFromProjectService( projectService: server.ProjectService, parseSettings: Readonly, ): ASTAndDefiniteProgram | undefined { - const opened = projectService.openClientFile( + projectService.openClientFile( absolutify(parseSettings.filePath), parseSettings.codeFullText, /* scriptKind */ undefined, parseSettings.tsconfigRootDir, ); - if (!opened.configFileName) { - return undefined; - } const scriptInfo = projectService.getScriptInfo(parseSettings.filePath); const program = projectService diff --git a/packages/typescript-estree/tests/lib/semanticInfo.test.ts b/packages/typescript-estree/tests/lib/semanticInfo.test.ts index 83569479085c..6ba1361b3733 100644 --- a/packages/typescript-estree/tests/lib/semanticInfo.test.ts +++ b/packages/typescript-estree/tests/lib/semanticInfo.test.ts @@ -335,33 +335,76 @@ describe('semanticInfo', () => { const parseResult = parseAndGenerateServices(code, optionsProjectString); expect(parseResult.services.program).toBe(program1); }); - } - it('file not in provided program instance(s)', () => { - const filename = 'non-existent-file.ts'; - const program1 = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); - const options = createOptions(filename); - const optionsWithSingleProgram = { - ...options, - programs: [program1], - }; - expect(() => - parseAndGenerateServices('const foo = 5;', optionsWithSingleProgram), - ).toThrow( - `The file was not found in any of the provided program instance(s): ${filename}`, - ); + it('file not in single provided program instance should throw', () => { + const filename = 'non-existent-file.ts'; + const program = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); + const options = createOptions(filename); + const optionsWithSingleProgram = { + ...options, + programs: [program], + }; + expect(() => + parseAndGenerateServices('const foo = 5;', optionsWithSingleProgram), + ).toThrow( + `The file was not found in any of the provided program instance(s): ${filename}`, + ); + }); - const program2 = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); - const optionsWithMultiplePrograms = { - ...options, - programs: [program1, program2], - }; - expect(() => - parseAndGenerateServices('const foo = 5;', optionsWithMultiplePrograms), - ).toThrow( - `The file was not found in any of the provided program instance(s): ${filename}`, - ); - }); + it('file not in multiple provided program instances should throw', () => { + const filename = 'non-existent-file.ts'; + const program1 = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); + const options = createOptions(filename); + const optionsWithSingleProgram = { + ...options, + programs: [program1], + }; + expect(() => + parseAndGenerateServices('const foo = 5;', optionsWithSingleProgram), + ).toThrow( + `The file was not found in any of the provided program instance(s): ${filename}`, + ); + + const program2 = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); + const optionsWithMultiplePrograms = { + ...options, + programs: [program1, program2], + }; + expect(() => + parseAndGenerateServices('const foo = 5;', optionsWithMultiplePrograms), + ).toThrow( + `The file was not found in any of the provided program instance(s): ${filename}`, + ); + }); + } else { + it('file not in single provided program instance should not throw', () => { + const filename = 'non-existent-file.ts'; + const program = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); + const options = createOptions(filename); + const optionsWithSingleProgram = { + ...options, + programs: [program], + }; + expect(() => + parseAndGenerateServices('const foo = 5;', optionsWithSingleProgram), + ).not.toThrow(); + }); + + it('file not in multiple provided program instances should not throw', () => { + const filename = 'non-existent-file.ts'; + const program1 = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); + const options = createOptions(filename); + + const program2 = createProgram(path.join(FIXTURES_DIR, 'tsconfig.json')); + const optionsWithMultiplePrograms = { + ...options, + programs: [program1, program2], + }; + expect(() => + parseAndGenerateServices('const foo = 5;', optionsWithMultiplePrograms), + ).not.toThrow(); + }); + } it('createProgram fails on non-existent file', () => { expect(() => createProgram('tsconfig.non-existent.json')).toThrow(); diff --git a/yarn.lock b/yarn.lock index 6d7cc6f99e96..65caa0396ebc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5904,6 +5904,18 @@ __metadata: languageName: unknown linkType: soft +"@typescript-eslint/performance@workspace:packages/performance": + version: 0.0.0-use.local + resolution: "@typescript-eslint/performance@workspace:packages/performance" + dependencies: + "@typescript-eslint/eslint-plugin": 6.9.1 + "@typescript-eslint/parser": 6.9.1 + eslint: "*" + execa: "*" + tsx: "*" + languageName: unknown + linkType: soft + "@typescript-eslint/repo-tools@workspace:packages/repo-tools": version: 0.0.0-use.local resolution: "@typescript-eslint/repo-tools@workspace:packages/repo-tools"