From 543663f2d980bcdae930845eab9d4a527fed1884 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 29 Nov 2016 16:55:13 +0100 Subject: [PATCH 1/2] Parallel compiling: arduino-builder will leverage your multi core pc. If -jobs X param is specified and X is greater than 0, arduino-builder won't spawn more than X processes. Fixes #17 Signed-off-by: Federico Fissore --- arduino-builder/main.go | 10 ++++++ builder_utils/utils.go | 60 ++++++++++++++++++++++++++++++----- ctags_runner.go | 4 +-- i18n/i18n.go | 70 +++++++++++++++++++++++++++++++++++------ types/accessories.go | 43 +++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 20 deletions(-) diff --git a/arduino-builder/main.go b/arduino-builder/main.go index d2ca98c4..a2700a79 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -37,6 +37,7 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strings" "syscall" @@ -79,6 +80,7 @@ const FLAG_LOGGER_HUMANTAGS = "humantags" const FLAG_LOGGER_MACHINE = "machine" const FLAG_VERSION = "version" const FLAG_VID_PID = "vid-pid" +const FLAG_JOBS = "jobs" type foldersFlag []string @@ -136,6 +138,7 @@ var warningsLevelFlag *string var loggerFlag *string var versionFlag *bool var vidPidFlag *string +var jobsFlag *int func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") @@ -159,6 +162,7 @@ func init() { loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'") versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits") vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") + jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") } func main() { @@ -173,6 +177,12 @@ func main() { return } + if *jobsFlag > 0 { + runtime.GOMAXPROCS(*jobsFlag) + } else { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + ctx := &types.Context{} if *buildOptionsFileFlag != "" { diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 139ad770..76598b18 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -31,15 +31,16 @@ package builder_utils import ( "bytes" - "fmt" "io" "os" "os/exec" "path/filepath" "strings" + "sync" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) @@ -146,15 +147,48 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { } func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { + if len(sources) == 0 { + return objectFiles, nil + } + objectFilesChan := make(chan string) + errorsChan := make(chan error) + doneChan := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(len(sources)) + for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) - if err != nil { + go func(source string) { + defer wg.Done() + objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + if err != nil { + errorsChan <- err + } else { + objectFilesChan <- objectFile + } + }(source) + } + + go func() { + wg.Wait() + doneChan <- struct{}{} + }() + + for { + select { + case objectFile := <-objectFilesChan: + objectFiles = append(objectFiles, objectFile) + case err := <-errorsChan: return nil, i18n.WrapError(err) + case <-doneChan: + close(objectFilesChan) + for objectFile := range objectFilesChan { + objectFiles = append(objectFiles, objectFile) + } + return objectFiles, nil } - objectFiles = append(objectFiles, objectFile) } - return objectFiles, nil } func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { @@ -361,10 +395,20 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties } if echoOutput { - command.Stdout = os.Stdout + printToStdOut := func(data []byte) { + logger.UnformattedWrite(os.Stdout, data) + } + stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}} + defer stdout.Flush() + command.Stdout = stdout } - command.Stderr = os.Stderr + printToStdErr := func(data []byte) { + logger.UnformattedWrite(os.Stderr, data) + } + stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}} + defer stderr.Flush() + command.Stderr = stderr if echoOutput { err := command.Run() @@ -393,7 +437,7 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo } if echoCommandLine { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } return command, nil diff --git a/ctags_runner.go b/ctags_runner.go index 82e4165c..86bb3cfb 100644 --- a/ctags_runner.go +++ b/ctags_runner.go @@ -30,7 +30,7 @@ package builder import ( - "fmt" + "os" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/ctags" @@ -63,7 +63,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { verbose := ctx.Verbose if verbose { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } sourceBytes, err := command.Output() diff --git a/i18n/i18n.go b/i18n/i18n.go index ce85d22c..3c188294 100644 --- a/i18n/i18n.go +++ b/i18n/i18n.go @@ -38,12 +38,15 @@ import ( "regexp" "strconv" "strings" + "sync" ) var PLACEHOLDER = regexp.MustCompile("{(\\d)}") type Logger interface { Fprintln(w io.Writer, level string, format string, a ...interface{}) + UnformattedFprintln(w io.Writer, s string) + UnformattedWrite(w io.Writer, data []byte) Println(level string, format string, a ...interface{}) Name() string } @@ -52,6 +55,10 @@ type NoopLogger struct{} func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {} +func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {} + +func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {} + func (s NoopLogger) Println(level string, format string, a ...interface{}) {} func (s NoopLogger) Name() string { @@ -69,6 +76,14 @@ func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanTagsLogger) Name() string { return "humantags" } @@ -76,20 +91,48 @@ func (s HumanTagsLogger) Name() string { type HumanLogger struct{} func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanLogger) Name() string { return "human" } type MachineLogger struct{} -func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) { +func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + printMachineFormattedLogLine(w, level, format, a) +} + +func (s MachineLogger) Println(level string, format string, a ...interface{}) { + printMachineFormattedLogLine(os.Stdout, level, format, a) +} + +func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s MachineLogger) Name() string { + return "machine" +} + +func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) { a = append([]interface{}(nil), a...) for idx, value := range a { typeof := reflect.Indirect(reflect.ValueOf(value)).Kind() @@ -97,20 +140,27 @@ func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format a[idx] = url.QueryEscape(value.(string)) } } - fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a) - fmt.Fprintln(w) + fprintf(w, "===%s ||| %s ||| %s\n", level, format, a) } -func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - s.printWithoutFormatting(w, level, format, a) +var lock sync.Mutex + +func fprintln(w io.Writer, s string) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintln(w, s) } -func (s MachineLogger) Println(level string, format string, a ...interface{}) { - s.printWithoutFormatting(os.Stdout, level, format, a) +func write(w io.Writer, data []byte) { + lock.Lock() + defer lock.Unlock() + w.Write(data) } -func (s MachineLogger) Name() string { - return "machine" +func fprintf(w io.Writer, format string, a ...interface{}) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintf(w, format, a...) } func FromJavaToGoSyntax(s string) string { diff --git a/types/accessories.go b/types/accessories.go index 6601bbaa..42d69734 100644 --- a/types/accessories.go +++ b/types/accessories.go @@ -29,6 +29,11 @@ package types +import ( + "bytes" + "sync" +) + type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -74,3 +79,41 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile { func (queue *UniqueSourceFileQueue) Empty() bool { return queue.Len() == 0 } + +type BufferedUntilNewLineWriter struct { + PrintFunc PrintFunc + Buffer bytes.Buffer + lock sync.Mutex +} + +type PrintFunc func([]byte) + +func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) { + w.lock.Lock() + defer w.lock.Unlock() + + writtenToBuffer, err := w.Buffer.Write(p) + if err != nil { + return writtenToBuffer, err + } + + bytesUntilNewLine, err := w.Buffer.ReadBytes('\n') + if err == nil { + w.PrintFunc(bytesUntilNewLine) + } + + return writtenToBuffer, err +} + +func (w *BufferedUntilNewLineWriter) Flush() { + w.lock.Lock() + defer w.lock.Unlock() + + remainingBytes := w.Buffer.Bytes() + if len(remainingBytes) > 0 { + if remainingBytes[len(remainingBytes)-1] != '\n' { + remainingBytes = append(remainingBytes, '\n') + } + w.PrintFunc(remainingBytes) + } +} From 0051a834e363d386caac04c70a5d0faa15dc2265 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 30 Nov 2016 12:19:55 +0100 Subject: [PATCH 2/2] Fix syncronization issues with "Multiple libraries found" Signed-off-by: Martino Facchin --- arduino-builder/main.go | 14 ++++++++------ print_used_and_not_used_libraries.go | 6 ++++-- types/context.go | 3 +++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/arduino-builder/main.go b/arduino-builder/main.go index a2700a79..08163b4b 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -177,12 +177,6 @@ func main() { return } - if *jobsFlag > 0 { - runtime.GOMAXPROCS(*jobsFlag) - } else { - runtime.GOMAXPROCS(runtime.NumCPU()) - } - ctx := &types.Context{} if *buildOptionsFileFlag != "" { @@ -200,6 +194,14 @@ func main() { ctx.InjectBuildOptions(buildOptions) } + // Parallel Jobs flag + if *jobsFlag > 0 { + ctx.Jobs = *jobsFlag + } else { + ctx.Jobs = runtime.NumCPU() + } + runtime.GOMAXPROCS(ctx.Jobs) + // FLAG_HARDWARE if hardwareFolders, err := toSliceOfUnquoted(hardwareFoldersFlag); err != nil { printCompleteError(err) diff --git a/print_used_and_not_used_libraries.go b/print_used_and_not_used_libraries.go index b60a6bf0..9d0817b1 100644 --- a/print_used_and_not_used_libraries.go +++ b/print_used_and_not_used_libraries.go @@ -53,6 +53,10 @@ func (s *PrintUsedAndNotUsedLibraries) Run(ctx *types.Context) error { return nil } + if ctx.Jobs > 1 { + time.Sleep(100 * time.Millisecond) + } + logger := ctx.GetLogger() libraryResolutionResults := ctx.LibrariesResolutionResults @@ -64,7 +68,5 @@ func (s *PrintUsedAndNotUsedLibraries) Run(ctx *types.Context) error { } } - time.Sleep(100 * time.Millisecond) - return nil } diff --git a/types/context.go b/types/context.go index bc9491dd..cf21a8cb 100644 --- a/types/context.go +++ b/types/context.go @@ -88,6 +88,9 @@ type Context struct { // ReadFileAndStoreInContext command FileToRead string + + // Max jobs + Jobs int } func (ctx *Context) ExtractBuildOptions() properties.Map {