Skip to content

Commit 059c5f8

Browse files
cmagliecmaglie
authored
Merge pull request #213 from facchinm/archiveCompiledCore
Archive compiled core and use on subsequent compilations
2 parents 9199e99 + 1125e53 commit 059c5f8

12 files changed

+330
-160
lines changed

‎src/arduino.cc/arduino-builder/main.go

+22
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ const FLAG_FQBN = "fqbn"
6464
const FLAG_IDE_VERSION = "ide-version"
6565
const FLAG_CORE_API_VERSION = "core-api-version"
6666
const FLAG_BUILD_PATH = "build-path"
67+
const FLAG_BUILD_CACHE = "build-cache"
6768
const FLAG_VERBOSE = "verbose"
6869
const FLAG_QUIET = "quiet"
6970
const FLAG_DEBUG_LEVEL = "debug-level"
@@ -126,6 +127,7 @@ var fqbnFlag *string
126127
var coreAPIVersionFlag *string
127128
var ideVersionFlag *string
128129
var buildPathFlag *string
130+
var buildCachePathFlag *string
129131
var verboseFlag *bool
130132
var quietFlag *bool
131133
var debugLevelFlag *int
@@ -148,6 +150,7 @@ func init() {
148150
coreAPIVersionFlag = flag.String(FLAG_CORE_API_VERSION, "10600", "version of core APIs (used to populate ARDUINO #define)")
149151
ideVersionFlag = flag.String(FLAG_IDE_VERSION, "10600", "[deprecated] use '"+FLAG_CORE_API_VERSION+"' instead")
150152
buildPathFlag = flag.String(FLAG_BUILD_PATH, "", "build path")
153+
buildCachePathFlag = flag.String(FLAG_BUILD_CACHE, "", "builds of 'core.a' are saved into this folder to be cached and reused")
151154
verboseFlag = flag.Bool(FLAG_VERBOSE, false, "if 'true' prints lots of stuff")
152155
quietFlag = flag.Bool(FLAG_QUIET, false, "if 'true' doesn't print any warnings or progress or whatever")
153156
debugLevelFlag = flag.Int(FLAG_DEBUG_LEVEL, builder.DEFAULT_DEBUG_LEVEL, "Turns on debugging messages. The higher, the chattier")
@@ -256,6 +259,25 @@ func main() {
256259
}
257260
ctx.BuildPath = buildPath
258261

262+
// FLAG_BUILD_CACHE
263+
buildCachePath, err := gohasissues.Unquote(*buildCachePathFlag)
264+
if err != nil {
265+
printCompleteError(err)
266+
}
267+
if buildCachePath != "" {
268+
_, err := os.Stat(buildCachePath)
269+
if err != nil {
270+
fmt.Fprintln(os.Stderr, err)
271+
os.Exit(1)
272+
}
273+
274+
err = utils.EnsureFolderExists(buildCachePath)
275+
if err != nil {
276+
printCompleteError(err)
277+
}
278+
}
279+
ctx.BuildCachePath = buildCachePath
280+
259281
// FLAG_VID_PID
260282
if *vidPidFlag != "" {
261283
ctx.USBVidPid = *vidPidFlag

‎src/arduino.cc/builder/add_additional_entries_to_context.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,11 @@
3030
package builder
3131

3232
import (
33+
"path/filepath"
34+
3335
"arduino.cc/builder/constants"
3436
"arduino.cc/builder/i18n"
3537
"arduino.cc/builder/types"
36-
"path/filepath"
3738
)
3839

3940
type AddAdditionalEntriesToContext struct{}
@@ -64,6 +65,15 @@ func (s *AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
6465
ctx.CoreBuildPath = coreBuildPath
6566
}
6667

68+
if ctx.BuildCachePath != "" {
69+
coreBuildCachePath, err := filepath.Abs(filepath.Join(ctx.BuildCachePath, constants.FOLDER_CORE))
70+
if err != nil {
71+
return i18n.WrapError(err)
72+
}
73+
74+
ctx.CoreBuildCachePath = coreBuildCachePath
75+
}
76+
6777
if ctx.WarningsLevel == "" {
6878
ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL
6979
}

‎src/arduino.cc/builder/builder_utils/utils.go

+107
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ package builder_utils
3232
import (
3333
"bytes"
3434
"fmt"
35+
"io"
3536
"os"
3637
"os/exec"
3738
"path/filepath"
@@ -116,6 +117,34 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str
116117
return sources, nil
117118
}
118119

120+
func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
121+
files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles())
122+
if err != nil {
123+
return nil, i18n.WrapError(err)
124+
}
125+
var sources []string
126+
for _, file := range files {
127+
sources = append(sources, filepath.Join(sourcePath, file.Name()))
128+
}
129+
130+
if recurse {
131+
folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs)
132+
if err != nil {
133+
return nil, i18n.WrapError(err)
134+
}
135+
136+
for _, folder := range folders {
137+
otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse)
138+
if err != nil {
139+
return nil, i18n.WrapError(err)
140+
}
141+
sources = append(sources, otherSources...)
142+
}
143+
}
144+
145+
return sources, nil
146+
}
147+
119148
func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
120149
for _, source := range sources {
121150
objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
@@ -258,6 +287,28 @@ func nonEmptyString(s string) bool {
258287
return s != constants.EMPTY_STRING
259288
}
260289

290+
func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) bool {
291+
292+
targetFileStat, err := os.Stat(targetFile)
293+
if err == nil {
294+
files, err := findAllFilesInFolder(corePath, true)
295+
if err != nil {
296+
return true
297+
}
298+
for _, file := range files {
299+
fileStat, err := os.Stat(file)
300+
if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) {
301+
return true
302+
}
303+
}
304+
if targetCorePath != constants.EMPTY_STRING && !strings.EqualFold(corePath, targetCorePath) {
305+
return CoreOrReferencedCoreHasChanged(targetCorePath, constants.EMPTY_STRING, targetFile)
306+
}
307+
return false
308+
}
309+
return true
310+
}
311+
261312
func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) {
262313
archiveFilePath := filepath.Join(buildPath, archiveFile)
263314

@@ -366,3 +417,59 @@ func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, remo
366417
func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) {
367418
buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1)
368419
}
420+
421+
// CopyFile copies the contents of the file named src to the file named
422+
// by dst. The file will be created if it does not already exist. If the
423+
// destination file exists, all it's contents will be replaced by the contents
424+
// of the source file. The file mode will be copied from the source and
425+
// the copied data is synced/flushed to stable storage.
426+
func CopyFile(src, dst string) (err error) {
427+
in, err := os.Open(src)
428+
if err != nil {
429+
return
430+
}
431+
defer in.Close()
432+
433+
out, err := os.Create(dst)
434+
if err != nil {
435+
return
436+
}
437+
defer func() {
438+
if e := out.Close(); e != nil {
439+
err = e
440+
}
441+
}()
442+
443+
_, err = io.Copy(out, in)
444+
if err != nil {
445+
return
446+
}
447+
448+
err = out.Sync()
449+
if err != nil {
450+
return
451+
}
452+
453+
si, err := os.Stat(src)
454+
if err != nil {
455+
return
456+
}
457+
err = os.Chmod(dst, si.Mode())
458+
if err != nil {
459+
return
460+
}
461+
462+
return
463+
}
464+
465+
// GetCachedCoreArchiveFileName returns the filename to be used to store
466+
// the global cached core.a.
467+
func GetCachedCoreArchiveFileName(fqbn, coreFolder string) string {
468+
fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1)
469+
fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1)
470+
if absCoreFolder, err := filepath.Abs(coreFolder); err == nil {
471+
coreFolder = absCoreFolder
472+
} // silently continue if absolute path can't be detected
473+
hash := utils.MD5Sum([]byte(coreFolder))
474+
return "core_" + fqbnToUnderscore + "_" + hash + ".a"
475+
}

‎src/arduino.cc/builder/constants/constants.go

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst"
5555
const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local"
5656
const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc"
5757
const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone"
58+
const BUILD_PROPERTIES_FQBN = "build.fqbn"
5859
const BUILD_PROPERTIES_INCLUDES = "includes"
5960
const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
6061
const BUILD_PROPERTIES_OBJECT_FILES = "object_files"

‎src/arduino.cc/builder/phases/core_builder.go

+38-2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
package phases
3131

3232
import (
33+
"path/filepath"
34+
3335
"arduino.cc/builder/builder_utils"
3436
"arduino.cc/builder/constants"
3537
"arduino.cc/builder/i18n"
@@ -42,6 +44,7 @@ type CoreBuilder struct{}
4244

4345
func (s *CoreBuilder) Run(ctx *types.Context) error {
4446
coreBuildPath := ctx.CoreBuildPath
47+
coreBuildCachePath := ctx.CoreBuildCachePath
4548
buildProperties := ctx.BuildProperties
4649
verbose := ctx.Verbose
4750
warningsLevel := ctx.WarningsLevel
@@ -52,7 +55,14 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
5255
return i18n.WrapError(err)
5356
}
5457

55-
archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger)
58+
if coreBuildCachePath != "" {
59+
err := utils.EnsureFolderExists(coreBuildCachePath)
60+
if err != nil {
61+
return i18n.WrapError(err)
62+
}
63+
}
64+
65+
archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger)
5666
if err != nil {
5767
return i18n.WrapError(err)
5868
}
@@ -63,10 +73,12 @@ func (s *CoreBuilder) Run(ctx *types.Context) error {
6373
return nil
6474
}
6575

66-
func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) {
76+
func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) {
6777
coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH]
6878
variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH]
6979

80+
targetCoreFolder := buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH]
81+
7082
includes := []string{}
7183
includes = append(includes, coreFolder)
7284
if variantFolder != constants.EMPTY_STRING {
@@ -84,6 +96,24 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,
8496
}
8597
}
8698

99+
// Recreate the archive if ANY of the core files (including platform.txt) has changed
100+
realCoreFolder := utils.GetParentFolder(coreFolder, 2)
101+
102+
var targetArchivedCore string
103+
if buildCachePath != "" {
104+
archivedCoreName := builder_utils.GetCachedCoreArchiveFileName(buildProperties[constants.BUILD_PROPERTIES_FQBN], realCoreFolder)
105+
targetArchivedCore = filepath.Join(buildCachePath, archivedCoreName)
106+
canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
107+
108+
if canUseArchivedCore {
109+
// use archived core
110+
if verbose {
111+
logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core")
112+
}
113+
return targetArchivedCore, variantObjectFiles, nil
114+
}
115+
}
116+
87117
coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
88118
if err != nil {
89119
return "", nil, i18n.WrapError(err)
@@ -94,5 +124,11 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool,
94124
return "", nil, i18n.WrapError(err)
95125
}
96126

127+
// archive core.a
128+
if targetArchivedCore != "" {
129+
logger.Println(constants.LOG_LEVEL_DEBUG, "Archiving built core (caching) in: "+targetArchivedCore)
130+
builder_utils.CopyFile(archiveFile, targetArchivedCore)
131+
}
132+
97133
return archiveFile, variantObjectFiles, nil
98134
}

‎src/arduino.cc/builder/setup_build_properties.go

+1
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error {
7070
buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder
7171
buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..")
7272
buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion
73+
buildProperties[constants.BUILD_PROPERTIES_FQBN] = ctx.FQBN
7374
buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion
7475
buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName()
7576

0 commit comments

Comments
 (0)