diff --git a/main.go b/main.go index cdba4273..8ce072c3 100644 --- a/main.go +++ b/main.go @@ -43,11 +43,12 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strings" "syscall" ) -const VERSION = "1.3.8" +const VERSION = "1.3.8-dev-parallel" const FLAG_ACTION_COMPILE = "compile" const FLAG_ACTION_PREPROCESS = "preprocess" @@ -74,6 +75,7 @@ const FLAG_LOGGER_HUMAN = "human" const FLAG_LOGGER_MACHINE = "machine" const FLAG_VERSION = "version" const FLAG_VID_PID = "vid-pid" +const FLAG_JOBS = "jobs" type slice []string @@ -116,6 +118,7 @@ var warningsLevelFlag *string var loggerFlag *string var versionFlag *bool var vidPidFlag *string +var jobsFlag *int func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") @@ -137,6 +140,7 @@ func init() { loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_MACHINE+"'") versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits") vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") + jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") } func main() { @@ -152,6 +156,12 @@ func main() { return } + if *jobsFlag > 0 { + runtime.GOMAXPROCS(*jobsFlag) + } else { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + context := make(map[string]interface{}) buildOptions := make(map[string]string) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 18ab284e..6fa5f0ce 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -33,13 +33,14 @@ import ( "arduino.cc/builder/constants" "arduino.cc/builder/i18n" "arduino.cc/builder/props" + "arduino.cc/builder/types" "arduino.cc/builder/utils" "bytes" - "fmt" "os" "os/exec" "path/filepath" "strings" + "sync" ) func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties map[string]string, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { @@ -116,15 +117,47 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str } func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties map[string]string, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { + if len(sources) == 0 { + return objectFiles, nil + } + objectFilesChan := make(chan string) + errorsChan := make(chan error) + doneChan := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(len(sources)) + for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) - if err != nil { + go func(source string) { + defer wg.Done() + objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + if err != nil { + errorsChan <- err + } else { + objectFilesChan <- objectFile + } + }(source) + } + + go func() { + wg.Wait() + doneChan <- struct{}{} + }() + + for { + select { + case objectFile := <-objectFilesChan: + objectFiles = append(objectFiles, objectFile) + case err := <-errorsChan: return nil, utils.WrapError(err) + case <-doneChan: + close(objectFilesChan) + for objectFile := range objectFilesChan { + objectFiles = append(objectFiles, objectFile) + } + return objectFiles, nil } - - objectFiles = append(objectFiles, objectFile) } - return objectFiles, nil } func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties map[string]string, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { @@ -286,10 +319,20 @@ func ExecRecipe(properties map[string]string, recipe string, removeUnsetProperti } if echoOutput { - command.Stdout = os.Stdout + printToStdOut := func(data []byte) { + logger.UnformattedWrite(os.Stdout, data) + } + stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}} + defer stdout.Flush() + command.Stdout = stdout } - command.Stderr = os.Stderr + printToStdErr := func(data []byte) { + logger.UnformattedWrite(os.Stderr, data) + } + stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}} + defer stderr.Flush() + command.Stderr = stderr if echoOutput { err := command.Run() @@ -321,7 +364,7 @@ func PrepareCommandForRecipe(properties map[string]string, recipe string, remove } if echoCommandLine { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } return command, nil diff --git a/src/arduino.cc/builder/coan_runner.go b/src/arduino.cc/builder/coan_runner.go index f4630621..e776c5af 100644 --- a/src/arduino.cc/builder/coan_runner.go +++ b/src/arduino.cc/builder/coan_runner.go @@ -34,7 +34,7 @@ import ( "arduino.cc/builder/i18n" "arduino.cc/builder/props" "arduino.cc/builder/utils" - "fmt" + "os" "path/filepath" "regexp" ) @@ -74,7 +74,7 @@ func (s *CoanRunner) Run(context map[string]interface{}) error { command, err := utils.PrepareCommandFilteredArgs(commandLine, filterAllowedArg, logger) if verbose { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } sourceBytes, _ := command.Output() diff --git a/src/arduino.cc/builder/ctags_runner.go b/src/arduino.cc/builder/ctags_runner.go index 20c9eb7c..b7ca3877 100644 --- a/src/arduino.cc/builder/ctags_runner.go +++ b/src/arduino.cc/builder/ctags_runner.go @@ -34,7 +34,7 @@ import ( "arduino.cc/builder/i18n" "arduino.cc/builder/props" "arduino.cc/builder/utils" - "fmt" + "os" ) type CTagsRunner struct{} @@ -60,7 +60,7 @@ func (s *CTagsRunner) Run(context map[string]interface{}) error { verbose := context[constants.CTX_VERBOSE].(bool) if verbose { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } sourceBytes, err := command.Output() diff --git a/src/arduino.cc/builder/i18n/i18n.go b/src/arduino.cc/builder/i18n/i18n.go index 2769e587..6540da79 100644 --- a/src/arduino.cc/builder/i18n/i18n.go +++ b/src/arduino.cc/builder/i18n/i18n.go @@ -38,12 +38,15 @@ import ( "regexp" "strconv" "strings" + "sync" ) var PLACEHOLDER = regexp.MustCompile("{(\\d)}") type Logger interface { Fprintln(w io.Writer, level string, format string, a ...interface{}) + UnformattedFprintln(w io.Writer, s string) + UnformattedWrite(w io.Writer, data []byte) Println(level string, format string, a ...interface{}) Name() string } @@ -52,6 +55,10 @@ type NoopLogger struct{} func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {} +func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {} + +func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {} + func (s NoopLogger) Println(level string, format string, a ...interface{}) {} func (s NoopLogger) Name() string { @@ -61,20 +68,48 @@ func (s NoopLogger) Name() string { type HumanLogger struct{} func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) +} + +func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) } -func (s HumanLogger) Println(level string, format string, a ...interface{}) { +func (s HumanLogger) Println(format string, level string, a ...interface{}) { s.Fprintln(os.Stdout, level, Format(format, a...)) } +func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanLogger) Name() string { return "human" } type MachineLogger struct{} -func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) { +func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + printMachineFormattedLogLine(w, level, format, a) +} + +func (s MachineLogger) Println(level string, format string, a ...interface{}) { + printMachineFormattedLogLine(os.Stdout, level, format, a) +} + +func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s MachineLogger) Name() string { + return "machine" +} + +func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) { a = append([]interface{}(nil), a...) for idx, value := range a { typeof := reflect.Indirect(reflect.ValueOf(value)).Kind() @@ -82,20 +117,27 @@ func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format a[idx] = url.QueryEscape(value.(string)) } } - fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a) - fmt.Fprintln(w) + fprintf(w, "===%s ||| %s ||| %s\n", level, format, a) } -func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - s.printWithoutFormatting(w, level, format, a) +var lock sync.Mutex + +func fprintln(w io.Writer, s string) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintln(w, s) } -func (s MachineLogger) Println(level string, format string, a ...interface{}) { - s.printWithoutFormatting(os.Stdout, level, format, a) +func write(w io.Writer, data []byte) { + lock.Lock() + defer lock.Unlock() + w.Write(data) } -func (s MachineLogger) Name() string { - return "machine" +func fprintf(w io.Writer, format string, a ...interface{}) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintf(w, format, a...) } func FromJavaToGoSyntax(s string) string { diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index 3ec2c459..31ae5fbc 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -32,15 +32,18 @@ package test import ( "arduino.cc/builder" "arduino.cc/builder/constants" + "arduino.cc/builder/i18n" "github.com/stretchr/testify/require" "os" "os/exec" "path/filepath" + "runtime" "testing" ) func TestBuilderEmptySketch(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -75,6 +78,7 @@ func TestBuilderEmptySketch(t *testing.T) { func TestBuilderBridge(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -89,6 +93,7 @@ func TestBuilderBridge(t *testing.T) { context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"} context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600" context[constants.CTX_VERBOSE] = true + context[constants.CTX_LOGGER] = i18n.MachineLogger{} command := builder.Builder{} err := command.Run(context) @@ -110,6 +115,7 @@ func TestBuilderBridge(t *testing.T) { func TestBuilderSketchWithConfig(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -123,6 +129,8 @@ func TestBuilderSketchWithConfig(t *testing.T) { context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"} context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"} context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600" + context[constants.CTX_VERBOSE] = true + context[constants.CTX_LOGGER] = i18n.NoopLogger{} command := builder.Builder{} err := command.Run(context) @@ -144,6 +152,7 @@ func TestBuilderSketchWithConfig(t *testing.T) { func TestBuilderBridgeTwice(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -182,6 +191,7 @@ func TestBuilderBridgeTwice(t *testing.T) { func TestBuilderBridgeSAM(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -226,6 +236,7 @@ func TestBuilderBridgeSAM(t *testing.T) { func TestBuilderBridgeRedBearLab(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -260,6 +271,7 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { func TestBuilderSketchNoFunctions(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -281,6 +293,7 @@ func TestBuilderSketchNoFunctions(t *testing.T) { func TestBuilderSketchWithBackup(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -302,6 +315,7 @@ func TestBuilderSketchWithBackup(t *testing.T) { func TestBuilderSketchWithOldLib(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -323,6 +337,7 @@ func TestBuilderSketchWithOldLib(t *testing.T) { func TestBuilderSketchWithSubfolders(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) context := make(map[string]interface{}) @@ -370,3 +385,26 @@ func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testin _, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_LIBRARIES, "Bridge")) NoError(t, err) } + +func TestBuilderSketchWithSyntaxError(t *testing.T) { + DownloadCoresAndToolsAndLibraries(t) + runtime.GOMAXPROCS(runtime.NumCPU()) + + context := make(map[string]interface{}) + + buildPath := SetupBuildPath(t, context) + defer os.RemoveAll(buildPath) + + context[constants.CTX_HARDWARE_FOLDERS] = []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"} + context[constants.CTX_TOOLS_FOLDERS] = []string{"downloaded_tools"} + context[constants.CTX_FQBN] = "arduino:avr:uno" + context[constants.CTX_SKETCH_LOCATION] = filepath.Join("sketch_with_syntax_error", "sketch.ino") + context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"} + context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"} + context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600" + context[constants.CTX_VERBOSE] = true + + command := builder.Builder{} + err := command.Run(context) + require.Error(t, err) +} diff --git a/src/arduino.cc/builder/test/sketch_with_syntax_error/sketch.ino b/src/arduino.cc/builder/test/sketch_with_syntax_error/sketch.ino new file mode 100644 index 00000000..ba3f6fce --- /dev/null +++ b/src/arduino.cc/builder/test/sketch_with_syntax_error/sketch.ino @@ -0,0 +1,3 @@ +void setup() { + +void loop() {} diff --git a/src/arduino.cc/builder/types/accessories.go b/src/arduino.cc/builder/types/accessories.go index 1d7b7fc2..54199c05 100644 --- a/src/arduino.cc/builder/types/accessories.go +++ b/src/arduino.cc/builder/types/accessories.go @@ -29,6 +29,11 @@ package types +import ( + "bytes" + "sync" +) + type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -74,3 +79,41 @@ func (queue *UniqueSourceFolderQueue) Pop() interface{} { func (queue *UniqueSourceFolderQueue) Empty() bool { return queue.Len() == 0 } + +type BufferedUntilNewLineWriter struct { + PrintFunc PrintFunc + Buffer bytes.Buffer + lock sync.Mutex +} + +type PrintFunc func([]byte) + +func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) { + w.lock.Lock() + defer w.lock.Unlock() + + writtenToBuffer, err := w.Buffer.Write(p) + if err != nil { + return writtenToBuffer, err + } + + bytesUntilNewLine, err := w.Buffer.ReadBytes('\n') + if err == nil { + w.PrintFunc(bytesUntilNewLine) + } + + return writtenToBuffer, err +} + +func (w *BufferedUntilNewLineWriter) Flush() { + w.lock.Lock() + defer w.lock.Unlock() + + remainingBytes := w.Buffer.Bytes() + if len(remainingBytes) > 0 { + if remainingBytes[len(remainingBytes)-1] != '\n' { + remainingBytes = append(remainingBytes, '\n') + } + w.PrintFunc(remainingBytes) + } +}