typescript-eslint · JoshuaKGoldberg · Nov 2, 2023 · Nov 2, 2023 · Nov 2, 2023 · Nov 5, 2023
diff --git a/.eslintignore b/.eslintignore
@@ -6,6 +6,7 @@ coverage
 __snapshots__
 .docusaurus
 build
+packages/performance/generated
 
 # Files copied as part of the build
 packages/types/src/generated/**/*.ts

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -0,0 +1,23 @@
+name: 'Performance Testing'
+
+on: workflow_dispatch
+
+jobs:
+  comparison:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/prepare-install
+      - run: brew install hyperfine
+      - run: PERFORMANCE_FILE_COUNT=${{ matrix.file-count }} yarn workspace @typescript-eslint/performance run generate
+      - run: hyperfine yarn workspace @typescript-eslint/performance run test:${{ matrix.test}}
+    strategy:
+      matrix:
+        file-count: [50, 100, 150, 200]
+        test: ['project', 'service']
+  summary:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/prepare-install
+      - run: yarn workspace @typescript-eslint/performance run test:summary
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,9 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
+# Performance testing
+packages/performance/generated
+
 # Website
 packages/website/.docusaurus
 packages/website/.cache-loader
@@ -95,4 +98,4 @@ packages/types/src/generated/**/*.ts
 !.yarn/sdks
 !.yarn/versions
 
-.nx/cache
+.nx/cache
diff --git a/packages/performance/.eslintrc.cjs b/packages/performance/.eslintrc.cjs
@@ -0,0 +1,15 @@
+/* eslint-env node */
+module.exports = {
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/strict-type-checked',
+    'plugin:@typescript-eslint/stylistic-type-checked',
+  ],
+  plugins: ['@typescript-eslint'],
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: true,
+    tsconfigRootDir: __dirname,
+  },
+  root: true,
+};
diff --git a/packages/performance/README.md b/packages/performance/README.md
@@ -0,0 +1,27 @@
+# Performance Testing
+
+A quick internal utility for testing lint performance.
+
+## Generating a Summary
+
+`test:summary` runs the three performance scenarios under different file counts, then prints a summary table:
+
+```shell
+yarn test:summary
+```
+
+For each of those file counts, the generated code is 50% `.js` and 50% `.ts`.
+
+## Running Tests Manually
+
+First generate the number of files you'd like to test:
+
+```shell
+PERFORMANCE_FILE_COUNT=123 yarn generate
+```
+
+Then run one or more of the test types:
+
+- `yarn test:project`: Traditional `project: true`
+- `yarn test:service` `EXPERIMENTAL_useProjectService`
+- `yarn test:service:seeded`: `EXPERIMENTAL_useProjectService`, with `allowDefaultProjectFallbackFilesGlobs`
diff --git a/packages/performance/generate.ts b/packages/performance/generate.ts
@@ -0,0 +1,56 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+import {
+  generateESLintConfig,
+  generateJsFile,
+  generateTSConfig,
+  generateTsFile,
+} from './generating.js';
+import { directory, writeFile } from './writing.js';
+
+const fileCount = Number(process.env.PERFORMANCE_FILE_COUNT);
+if (!fileCount) {
+  throw new Error(
+    `Invalid process.env.PERFORMANCE_FILE_COUNT: ${process.env.PERFORMANCE_FILE_COUNT}`,
+  );
+}
+
+await fs.rm(directory, { force: true, recursive: true });
+await fs.mkdir(path.join(directory, 'src'), { recursive: true });
+
+// The root-level TSConfig is used for the project service, which intentionally
+// uses the default/inferred project for non-included JS files.
+await writeFile(`tsconfig.json`, generateTSConfig(false));
+
+// The explicit project allows JS, akin to the common tsconfig.eslint.json.
+await writeFile(`tsconfig.project.json`, generateTSConfig(true));
+
+for (const [alias, parserOptions] of [
+  ['project', `project: "./tsconfig.project.json"`],
+  ['service', `EXPERIMENTAL_useProjectService: true`],
+  [
+    'service.seeded',
+    `allowDefaultProjectFallbackFilesGlobs: ["./src/*.js"],\n    EXPERIMENTAL_useProjectService: true`,
+  ],
+] as const) {
+  await writeFile(
+    `.eslintrc.${alias}.cjs`,
+    generateESLintConfig(parserOptions),
+  );
+}
+
+for (let i = 0; i < fileCount; i += 1) {
+  const [extension, generator] =
+    i % 2 ? ['js', generateJsFile] : ['ts', generateTsFile];
+
+  await writeFile(`src/example${i}.${extension}`, generator(i));
+}
+
+await writeFile(
+  'src/index.ts',
+  new Array(fileCount)
+    .fill(undefined)
+    .map((_, i) => `export { example${i} } from "./example${i}.js";`)
+    .join('\n'),
+);
diff --git a/packages/performance/generating.ts b/packages/performance/generating.ts
@@ -0,0 +1,48 @@
+export function generateESLintConfig(parserOptions: string) {
+  return `
+/* eslint-env node */
+module.exports = {
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/strict-type-checked',
+    'plugin:@typescript-eslint/stylistic-type-checked',
+  ],
+  plugins: ['@typescript-eslint'],
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    ${parserOptions},
+    tsconfigRootDir: __dirname
+  },
+  root: true,
+};
+`.trimStart();
+}
+
+export function generateJsFile(i: number) {
+  return `
+export async function example${i}(input) {
+  return typeof input === 'number' ? input.toPrecision(1) : input.toUpperCase();
+}
+`.trimStart();
+}
+
+export function generateTsFile(i: number) {
+  return `
+export async function example${i}<T extends number | string>(input: T) {
+  return typeof input === 'number' ? input.toPrecision(1) : input.toUpperCase();
+}
+`.trimStart();
+}
+
+export function generateTSConfig(allowJs: boolean) {
+  return `
+{
+  "compilerOptions": {
+    "allowJs": ${allowJs},
+    "module": "ESNext",
+    "strict": true,
+    "target": "ESNext"
+  }
+}
+`.trimStart();
+}
diff --git a/packages/performance/package.json b/packages/performance/package.json
@@ -0,0 +1,28 @@
+{
+  "name": "@typescript-eslint/performance",
+  "version": "6.9.1",
+  "private": true,
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/typescript-eslint/typescript-eslint.git",
+    "directory": "packages/performance"
+  },
+  "scripts": {
+    "generate": "tsx ./generate.ts",
+    "generate:watch": "tsx --watch ./generate.ts",
+    "test:project": "time eslint --config ./generated/.eslintrc.project.cjs ./generated || true",
+    "test:service": "time eslint --config ./generated/.eslintrc.service.cjs ./generated || true",
+    "test:service:seeded": "time eslint --config ./generated/.eslintrc.service.seeded.cjs ./generated || true",
+    "test:summary": "tsx ./summary.ts"
+  },
+  "dependencies": {
+    "@typescript-eslint/eslint-plugin": "6.9.1",
+    "@typescript-eslint/parser": "6.9.1"
+  },
+  "devDependencies": {
+    "eslint": "*",
+    "execa": "*",
+    "tsx": "*"
+  }
+}
diff --git a/packages/performance/summary.ts b/packages/performance/summary.ts
@@ -0,0 +1,17 @@
+import { $ } from 'execa';
+
+const getTiming = ({ stderr }: { stderr: string }) =>
+  stderr.trim().replace(/ +/g, ' ');
+
+const counts: Record<string, unknown> = {};
+
+for (const fileCount of [5, 10, 25, 50, 75, 100, 125, 150, 175, 200]) {
+  await $({ env: { PERFORMANCE_FILE_COUNT: `${fileCount}` } })`yarn generate`;
+  const project = getTiming(await $`yarn test:project`);
+  const service = getTiming(await $`yarn test:service`);
+  const serviceSeeded = getTiming(await $`yarn test:service:seeded`);
+
+  counts[fileCount] = { project, service, 'service (seeded)': serviceSeeded };
+}
+
+console.table(counts);
diff --git a/packages/performance/tsconfig.json b/packages/performance/tsconfig.json
@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "strict": true,
+    "target": "ESNext"
+  }
+}
diff --git a/packages/performance/writing.ts b/packages/performance/writing.ts
@@ -0,0 +1,8 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+export const directory = 'generated';
+
+export async function writeFile(fileName: string, data: string) {
+  await fs.writeFile(path.join(directory, fileName), data);
+}
diff --git a/packages/typescript-estree/src/create-program/createProjectService.ts b/packages/typescript-estree/src/create-program/createProjectService.ts
@@ -1,4 +1,6 @@
 /* eslint-disable @typescript-eslint/no-empty-function -- for TypeScript APIs*/
+import * as fs from 'fs';
+import { sync as globSync } from 'globby';
 import type * as ts from 'typescript/lib/tsserverlibrary';
 
 const doNothing = (): void => {};
@@ -9,7 +11,9 @@ const createStubFileWatcher = (): ts.FileWatcher => ({
 
 export type TypeScriptProjectService = ts.server.ProjectService;
 
-export function createProjectService(): TypeScriptProjectService {
+export function createProjectService(
+  allowDefaultProjectFallbackFilesGlobs: string[] = [],
+): TypeScriptProjectService {
   // We import this lazily to avoid its cost for users who don't use the service
   const tsserver = require('typescript/lib/tsserverlibrary') as typeof ts;
 
@@ -27,7 +31,7 @@ export function createProjectService(): TypeScriptProjectService {
     watchFile: createStubFileWatcher,
   };
 
-  return new tsserver.server.ProjectService({
+  const service = new tsserver.server.ProjectService({
     host: system,
     cancellationToken: { isCancellationRequested: (): boolean => false },
     useSingleInferredProject: false,
@@ -45,5 +49,13 @@ export function createProjectService(): TypeScriptProjectService {
     },
     session: undefined,
   });
+
+  for (const filesGlob of allowDefaultProjectFallbackFilesGlobs) {
+    for (const fileName of globSync(filesGlob, { ignore: ['node_modules/'] })) {
+      service.openClientFile(fileName, fs.readFileSync(fileName).toString());
+    }
+  }
+
+  return service;
 }
 /* eslint-enable @typescript-eslint/no-empty-function */
diff --git a/packages/typescript-estree/src/parseSettings/createParseSettings.ts b/packages/typescript-estree/src/parseSettings/createParseSettings.ts
@@ -55,7 +55,9 @@ export function createParseSettings(
         process.env.TYPESCRIPT_ESLINT_EXPERIMENTAL_TSSERVER !== 'false') ||
       (process.env.TYPESCRIPT_ESLINT_EXPERIMENTAL_TSSERVER === 'true' &&
         options.EXPERIMENTAL_useProjectService !== false)
-        ? (TSSERVER_PROJECT_SERVICE ??= createProjectService())
+        ? (TSSERVER_PROJECT_SERVICE ??= createProjectService(
+            options.allowDefaultProjectFallbackFilesGlobs,
+          ))
         : undefined,
     EXPERIMENTAL_useSourceOfProjectReferenceRedirect:
       options.EXPERIMENTAL_useSourceOfProjectReferenceRedirect === true,

diff --git a/packages/typescript-estree/src/parser-options.ts b/packages/typescript-estree/src/parser-options.ts
@@ -103,6 +103,16 @@ interface ParseAndGenerateServicesOptions extends ParseOptions {
    */
   EXPERIMENTAL_useProjectService?: boolean;
 
+  /**
+   * ***EXPERIMENTAL FLAG*** - Use this at your own risk.
+   *
+   * If `EXPERIMENTAL_useProjectService` is true, which files to load into the service immediately.
+   * Intentionally an annoyingly long name for now. We'll think of a better one if we need.
+   *
+   * @see https://github.com/typescript-eslint/typescript-eslint/pull/7870
+   */
+  allowDefaultProjectFallbackFilesGlobs?: string[];
+
   /**
    * ***EXPERIMENTAL FLAG*** - Use this at your own risk.
    *

diff --git a/packages/typescript-estree/src/useProgramFromProjectService.ts b/packages/typescript-estree/src/useProgramFromProjectService.ts
@@ -10,15 +10,12 @@ export function useProgramFromProjectService(
   projectService: server.ProjectService,
   parseSettings: Readonly<MutableParseSettings>,
 ): ASTAndDefiniteProgram | undefined {
-  const opened = projectService.openClientFile(
+  projectService.openClientFile(
     absolutify(parseSettings.filePath),
     parseSettings.codeFullText,
     /* scriptKind */ undefined,
     parseSettings.tsconfigRootDir,
   );
-  if (!opened.configFileName) {
-    return undefined;
-  }
 
   const scriptInfo = projectService.getScriptInfo(parseSettings.filePath);
   const program = projectService