From 3d8a1ab5e4816bae01a5e77799b9ec9a09c459ce Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:11:16 -0700 Subject: [PATCH] Add concurrency options --- Herebyfile.mjs | 19 ++++- cmd/tsgo/main.go | 14 +++- internal/compiler/concurrency.go | 81 ++++++++++++++++++++ internal/compiler/fileloader.go | 2 +- internal/compiler/program.go | 12 +-- internal/compiler/program_test.go | 15 ++-- internal/testrunner/compiler_runner.go | 3 +- internal/testutil/harnessutil/harnessutil.go | 8 +- internal/testutil/testutil.go | 18 ++--- 9 files changed, 139 insertions(+), 33 deletions(-) create mode 100644 internal/compiler/concurrency.go diff --git a/Herebyfile.mjs b/Herebyfile.mjs index c6175571be..a396f6ab54 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -48,6 +48,20 @@ function parseEnvBoolean(name, defaultValue = false) { throw new Error(`Invalid value for ${name}: ${value}`); } +/** + * @param {string} name + * @param {string} defaultValue + * @returns {string} + */ +function parseEnvString(name, defaultValue = "") { + name = "TSGO_HEREBY_" + name.toUpperCase(); + const value = process.env[name]; + if (!value) { + return defaultValue; + } + return value; +} + const { values: options } = parseArgs({ args: process.argv.slice(2), options: { @@ -59,7 +73,7 @@ const { values: options } = parseArgs({ race: { type: "boolean", default: parseEnvBoolean("RACE") }, noembed: { type: "boolean", default: parseEnvBoolean("NOEMBED") }, - concurrentTestPrograms: { type: "boolean", default: parseEnvBoolean("CONCURRENT_TEST_PROGRAMS") }, + concurrency: { type: "string", default: parseEnvString("CONCURRENT_TEST_PROGRAMS") }, coverage: { type: "boolean", default: parseEnvBoolean("COVERAGE") }, }, strict: false, @@ -259,11 +273,12 @@ function goTestFlags(taskName) { ...goBuildTags(), ...(options.tests ? [`-run=${options.tests}`] : []), ...(options.coverage ? [`-coverprofile=${path.join(coverageDir, "coverage." + taskName + ".out")}`, "-coverpkg=./..."] : []), + ...(options.concurrency === "checker-per-file" ? ["-parallel=1"] : []), ]; } const goTestEnv = { - ...(options.concurrentTestPrograms ? { TS_TEST_PROGRAM_SINGLE_THREADED: "false" } : {}), + ...(typeof options.concurrency === "string" ? { TSGO_TEST_CONCURRENCY: options.concurrency } : {}), // Go test caching takes a long time on Windows. // https://github.com/golang/go/issues/72992 ...(process.platform === "win32" ? { GOFLAGS: "-count=1" } : {}), diff --git a/cmd/tsgo/main.go b/cmd/tsgo/main.go index 5fb70f9634..bd2e506c45 100644 --- a/cmd/tsgo/main.go +++ b/cmd/tsgo/main.go @@ -67,6 +67,7 @@ type cliOptions struct { devel struct { quiet bool singleThreaded bool + concurrency string printTypes bool pprofDir string } @@ -111,6 +112,7 @@ func parseArgs() *cliOptions { flag.BoolVar(&opts.devel.quiet, "q", false, "Do not print diagnostics.") flag.BoolVar(&opts.devel.quiet, "quiet", false, "Do not print diagnostics.") flag.BoolVar(&opts.devel.singleThreaded, "singleThreaded", false, "Run in single threaded mode.") + flag.StringVar(&opts.devel.concurrency, "concurrency", "default", "Set the concurrency level for the compiler. Options: default, single-threaded, max-procs, checker-per-file, or a number.") flag.BoolVar(&opts.devel.printTypes, "printTypes", false, "Print types defined in 'main.ts'.") flag.StringVar(&opts.devel.pprofDir, "pprofDir", "", "Generate pprof CPU/memory profiles to the given directory.") flag.Parse() @@ -120,6 +122,10 @@ func parseArgs() *cliOptions { os.Exit(1) } + if opts.devel.singleThreaded { + opts.devel.concurrency = "single-threaded" + } + return opts } @@ -163,6 +169,12 @@ func runMain() int { return 0 } + concurrency, err := compiler.ParseConcurrency(opts.devel.concurrency) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing concurrency: %v\n", err) + return 1 + } + startTime := time.Now() currentDirectory, err := os.Getwd() @@ -194,7 +206,7 @@ func runMain() int { program := compiler.NewProgram(compiler.ProgramOptions{ ConfigFileName: configFileName, Options: compilerOptions, - SingleThreaded: opts.devel.singleThreaded, + Concurrency: concurrency, Host: host, }) parseTime := time.Since(parseStart) diff --git a/internal/compiler/concurrency.go b/internal/compiler/concurrency.go new file mode 100644 index 0000000000..c4d1cbe9f2 --- /dev/null +++ b/internal/compiler/concurrency.go @@ -0,0 +1,81 @@ +package compiler + +import ( + "runtime" + "strconv" + "strings" +) + +// Concurrency controls the number of concurrent operations used by the Program. +// If greater than 0, it specifies the number of concurrent checkers to use. +type Concurrency int + +const ( + // Use 4 checkers for checking, but maximum concurrency for all other operations. + ConcurrencyDefault Concurrency = 0 + // Use a single thread for all operations. + ConcurrencySingleThreaded Concurrency = -1 + // Use all available threads for all operations. + ConcurrencyMaxProcs Concurrency = -2 + // Create a checker for each file in the program. + ConcurrencyCheckerPerFile Concurrency = -3 +) + +func (c Concurrency) IsSingleThreaded() bool { + return c == ConcurrencySingleThreaded +} + +func (c Concurrency) String() string { + switch c { + case ConcurrencyDefault: + return "default" + case ConcurrencySingleThreaded: + return "single" + case ConcurrencyMaxProcs: + return "max" + case ConcurrencyCheckerPerFile: + return "checker-per-file" + default: + return "concurrency-" + strconv.Itoa(int(c)) + } +} + +func (c Concurrency) Checkers(numFiles int) int { + switch c { + case ConcurrencyDefault: + return 4 + case ConcurrencySingleThreaded: + return 1 + case ConcurrencyMaxProcs: + return runtime.GOMAXPROCS(0) + case ConcurrencyCheckerPerFile: + return numFiles + default: + return int(c) + } +} + +func ParseConcurrency(s string) (Concurrency, error) { + s = strings.ToLower(s) + switch s { + case "default": + return ConcurrencyDefault, nil + case "single": + return ConcurrencySingleThreaded, nil + case "max": + return ConcurrencyMaxProcs, nil + case "checker-per-file": + return ConcurrencyCheckerPerFile, nil + default: + c, err := strconv.Atoi(s) + return Concurrency(c), err + } +} + +func MustParseConcurrency(s string) Concurrency { + c, err := ParseConcurrency(s) + if err != nil { + panic(err) + } + return c +} diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index f4fc8c1ecf..a8302f1cf6 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -68,7 +68,7 @@ func processAllProgramFiles( UseCaseSensitiveFileNames: host.FS().UseCaseSensitiveFileNames(), CurrentDirectory: host.GetCurrentDirectory(), }, - wg: core.NewWorkGroup(programOptions.SingleThreaded), + wg: core.NewWorkGroup(programOptions.Concurrency.IsSingleThreaded()), rootTasks: make([]*parseTask, 0, len(rootFiles)+len(libs)), supportedExtensions: core.Flatten(tsoptions.GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions, supportedExtensions)), } diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 1b798036ce..451d88b7fa 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -25,7 +25,7 @@ type ProgramOptions struct { RootFiles []string Host CompilerHost Options *core.CompilerOptions - SingleThreaded bool + Concurrency Concurrency ProjectReference []core.ProjectReference ConfigFileParsingDiagnostics []*ast.Diagnostic } @@ -196,7 +196,7 @@ func (p *Program) getSourceAffectingCompilerOptions() *core.SourceFileAffectingC } func (p *Program) BindSourceFiles() { - wg := core.NewWorkGroup(p.programOptions.SingleThreaded) + wg := core.NewWorkGroup(p.programOptions.Concurrency.IsSingleThreaded()) for _, file := range p.files { if !file.IsBound() { wg.Queue(func() { @@ -209,7 +209,7 @@ func (p *Program) BindSourceFiles() { func (p *Program) CheckSourceFiles(ctx context.Context) { p.createCheckers() - wg := core.NewWorkGroup(p.programOptions.SingleThreaded) + wg := core.NewWorkGroup(p.programOptions.Concurrency.IsSingleThreaded()) for index, checker := range p.checkers { wg.Queue(func() { for i := index; i < len(p.files); i += len(p.checkers) { @@ -222,8 +222,8 @@ func (p *Program) CheckSourceFiles(ctx context.Context) { func (p *Program) createCheckers() { p.checkersOnce.Do(func() { - p.checkers = make([]*checker.Checker, core.IfElse(p.programOptions.SingleThreaded, 1, 4)) - wg := core.NewWorkGroup(p.programOptions.SingleThreaded) + p.checkers = make([]*checker.Checker, p.programOptions.Concurrency.Checkers(len(p.files))) + wg := core.NewWorkGroup(p.programOptions.Concurrency.IsSingleThreaded()) for i := range p.checkers { wg.Queue(func() { p.checkers[i] = checker.NewChecker(p) @@ -636,7 +636,7 @@ func (p *Program) Emit(options EmitOptions) *EmitResult { return printer.NewTextWriter(host.Options().NewLine.GetNewLineCharacter()) }, } - wg := core.NewWorkGroup(p.programOptions.SingleThreaded) + wg := core.NewWorkGroup(p.programOptions.Concurrency.IsSingleThreaded()) var emitters []*emitter sourceFiles := getSourceFilesToEmit(host, options.TargetSourceFile, options.forceDtsEmit) diff --git a/internal/compiler/program_test.go b/internal/compiler/program_test.go index 0379028c37..c290a42204 100644 --- a/internal/compiler/program_test.go +++ b/internal/compiler/program_test.go @@ -231,10 +231,9 @@ func TestProgram(t *testing.T) { opts := core.CompilerOptions{Target: testCase.target} program := NewProgram(ProgramOptions{ - RootFiles: []string{"c:/dev/src/index.ts"}, - Host: NewCompilerHost(&opts, "c:/dev/src", fs, bundled.LibPath()), - Options: &opts, - SingleThreaded: false, + RootFiles: []string{"c:/dev/src/index.ts"}, + Host: NewCompilerHost(&opts, "c:/dev/src", fs, bundled.LibPath()), + Options: &opts, }) actualFiles := []string{} @@ -265,10 +264,9 @@ func BenchmarkNewProgram(b *testing.B) { opts := core.CompilerOptions{Target: testCase.target} programOpts := ProgramOptions{ - RootFiles: []string{"c:/dev/src/index.ts"}, - Host: NewCompilerHost(&opts, "c:/dev/src", fs, bundled.LibPath()), - Options: &opts, - SingleThreaded: false, + RootFiles: []string{"c:/dev/src/index.ts"}, + Host: NewCompilerHost(&opts, "c:/dev/src", fs, bundled.LibPath()), + Options: &opts, } for b.Loop() { @@ -288,7 +286,6 @@ func BenchmarkNewProgram(b *testing.B) { opts := ProgramOptions{ ConfigFileName: tspath.CombinePaths(compilerDir, "tsconfig.json"), Host: NewCompilerHost(nil, compilerDir, fs, bundled.LibPath()), - SingleThreaded: false, } for b.Loop() { diff --git a/internal/testrunner/compiler_runner.go b/internal/testrunner/compiler_runner.go index 5f6c48091b..aee8407630 100644 --- a/internal/testrunner/compiler_runner.go +++ b/internal/testrunner/compiler_runner.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/microsoft/typescript-go/internal/checker" + "github.com/microsoft/typescript-go/internal/compiler" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/repo" "github.com/microsoft/typescript-go/internal/testutil" @@ -334,7 +335,7 @@ var concurrentSkippedErrorBaselines = core.NewSetFromItems( func (c *compilerTest) verifyDiagnostics(t *testing.T, suiteName string, isSubmodule bool) { t.Run("error", func(t *testing.T) { - if !testutil.TestProgramIsSingleThreaded() && concurrentSkippedErrorBaselines.Has(c.testName) { + if !compiler.MustParseConcurrency(testutil.TestConcurrency()).IsSingleThreaded() && concurrentSkippedErrorBaselines.Has(c.testName) { t.Skip("Skipping error baseline in concurrent mode") } diff --git a/internal/testutil/harnessutil/harnessutil.go b/internal/testutil/harnessutil/harnessutil.go index ce41ce3f08..9373dab0e6 100644 --- a/internal/testutil/harnessutil/harnessutil.go +++ b/internal/testutil/harnessutil/harnessutil.go @@ -781,10 +781,10 @@ func (c *CompilationResult) GetSourceMapRecord() string { func createProgram(host compiler.CompilerHost, options *core.CompilerOptions, rootFiles []string) *compiler.Program { programOptions := compiler.ProgramOptions{ - RootFiles: rootFiles, - Host: host, - Options: options, - SingleThreaded: testutil.TestProgramIsSingleThreaded(), + RootFiles: rootFiles, + Host: host, + Options: options, + Concurrency: compiler.MustParseConcurrency(testutil.TestConcurrency()), } program := compiler.NewProgram(programOptions) return program diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 279258f27a..ae683716e4 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -3,7 +3,6 @@ package testutil import ( "os" "runtime/debug" - "strconv" "sync" "testing" @@ -34,16 +33,17 @@ func RecoverAndFail(t *testing.T, msg string) { } } -var testProgramIsSingleThreaded = sync.OnceValue(func() bool { +var testConcurrency = sync.OnceValue(func() string { // Leave Program in SingleThreaded mode unless explicitly configured or in race mode. - if v := os.Getenv("TS_TEST_PROGRAM_SINGLE_THREADED"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } + if v := os.Getenv("TSGO_TEST_CONCURRENCY"); v != "" { + return v } - return !race.Enabled + if race.Enabled { + return "default" + } + return "single" }) -func TestProgramIsSingleThreaded() bool { - return testProgramIsSingleThreaded() +func TestConcurrency() string { + return testConcurrency() }