Merge branch 'arduino:master' into master

arduino/builder/builder.go

@@ -26,6 +26,9 @@ type Builder struct {
 	sketch          *sketch.Sketch
 	buildProperties *properties.Map
 
+	// Parallel processes
+	jobs int
+
 	// core related
 	coreBuildCachePath *paths.Path
 }
@@ -37,6 +40,7 @@ func NewBuilder(
 	buildPath *paths.Path,
 	optimizeForDebug bool,
 	coreBuildCachePath *paths.Path,
+	jobs int,
 ) *Builder {
 	buildProperties := properties.NewMap()
 	if boardBuildProperties != nil {
@@ -64,10 +68,16 @@ func NewBuilder(
 		sketch:             sk,
 		buildProperties:    buildProperties,
 		coreBuildCachePath: coreBuildCachePath,
+		jobs:               jobs,
 	}
 }
 
 // GetBuildProperties returns the build properties for running this build
 func (b *Builder) GetBuildProperties() *properties.Map {
 	return b.buildProperties
 }
+
+// Jobs number of parallel processes
+func (b *Builder) Jobs() int {
+	return b.jobs
+}

arduino/builder/compilation_database.go → arduino/builder/compilation/database.go

@@ -13,52 +13,55 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to [email protected].
 
-package builder
+package compilation
 
 import (
 	"encoding/json"
 	"fmt"
 	"os"
 
 	"github.com/arduino/arduino-cli/executils"
+	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/go-paths-helper"
 )
 
-// CompilationDatabase keeps track of all the compile commands run by the builder
-type CompilationDatabase struct {
-	Contents []CompilationCommand
+var tr = i18n.Tr
+
+// Database keeps track of all the compile commands run by the builder
+type Database struct {
+	Contents []Command
 	File     *paths.Path
 }
 
-// CompilationCommand keeps track of a single run of a compile command
-type CompilationCommand struct {
+// Command keeps track of a single run of a compile command
+type Command struct {
 	Directory string   `json:"directory"`
 	Command   string   `json:"command,omitempty"`
 	Arguments []string `json:"arguments,omitempty"`
 	File      string   `json:"file"`
 }
 
-// NewCompilationDatabase creates an empty CompilationDatabase
-func NewCompilationDatabase(filename *paths.Path) *CompilationDatabase {
-	return &CompilationDatabase{
+// NewDatabase creates an empty CompilationDatabase
+func NewDatabase(filename *paths.Path) *Database {
+	return &Database{
 		File:     filename,
-		Contents: []CompilationCommand{},
+		Contents: []Command{},
 	}
 }
 
-// LoadCompilationDatabase reads a compilation database from a file
-func LoadCompilationDatabase(file *paths.Path) (*CompilationDatabase, error) {
+// LoadDatabase reads a compilation database from a file
+func LoadDatabase(file *paths.Path) (*Database, error) {
 	f, err := file.ReadFile()
 	if err != nil {
 		return nil, err
 	}
-	res := NewCompilationDatabase(file)
+	res := NewDatabase(file)
 	return res, json.Unmarshal(f, &res.Contents)
 }
 
 // SaveToFile save the CompilationDatabase to file as a clangd-compatible compile_commands.json,
 // see https://clang.llvm.org/docs/JSONCompilationDatabase.html
-func (db *CompilationDatabase) SaveToFile() {
+func (db *Database) SaveToFile() {
 	if jsonContents, err := json.MarshalIndent(db.Contents, "", " "); err != nil {
 		fmt.Println(tr("Error serializing compilation database: %s", err))
 		return
@@ -68,7 +71,7 @@ func (db *CompilationDatabase) SaveToFile() {
 }
 
 // Add adds a new CompilationDatabase entry
-func (db *CompilationDatabase) Add(target *paths.Path, command *executils.Process) {
+func (db *Database) Add(target *paths.Path, command *executils.Process) {
 	commandDir := command.GetDir()
 	if commandDir == "" {
 		// This mimics what Cmd.Run also does: Use Dir if specified,
@@ -80,7 +83,7 @@ func (db *CompilationDatabase) Add(target *paths.Path, command *executils.Proces
 		commandDir = dir
 	}
 
-	entry := CompilationCommand{
+	entry := Command{
 		Directory: commandDir,
 		Arguments: command.GetArgs(),
 		File:      target.String(),

arduino/builder/compilation_database_test.go → arduino/builder/compilation/database_test.go

@@ -13,7 +13,7 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to [email protected].
 
-package builder
+package compilation
 
 import (
 	"testing"
@@ -30,11 +30,11 @@ func TestCompilationDatabase(t *testing.T) {
 
 	cmd, err := executils.NewProcess(nil, "gcc", "arg1", "arg2")
 	require.NoError(t, err)
-	db := NewCompilationDatabase(tmpfile)
+	db := NewDatabase(tmpfile)
 	db.Add(paths.New("test"), cmd)
 	db.SaveToFile()
 
-	db2, err := LoadCompilationDatabase(tmpfile)
+	db2, err := LoadDatabase(tmpfile)
 	require.NoError(t, err)
 	require.Equal(t, db, db2)
 	require.Len(t, db2.Contents, 1)

arduino/builder/core.go

@@ -1,8 +1,222 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
 package builder
 
-import "github.com/arduino/go-paths-helper"
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/builder/compilation"
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/arduino/builder/logger"
+	"github.com/arduino/arduino-cli/arduino/builder/progress"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
+	"github.com/arduino/arduino-cli/arduino/cores"
+	"github.com/arduino/arduino-cli/buildcache"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	"github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-properties-orderedmap"
+	"github.com/pkg/errors"
+)
 
 // CoreBuildCachePath fixdoc
 func (b *Builder) CoreBuildCachePath() *paths.Path {
 	return b.coreBuildCachePath
 }
+
+// CoreBuilder fixdoc
+func CoreBuilder(
+	buildPath, coreBuildPath, coreBuildCachePath *paths.Path,
+	buildProperties *properties.Map,
+	actualPlatform *cores.PlatformRelease,
+	onlyUpdateCompilationDatabase, clean bool,
+	compilationDatabase *compilation.Database,
+	jobs int,
+	builderLogger *logger.BuilderLogger,
+	progress *progress.Struct, progressCB rpc.TaskProgressCB,
+) (paths.PathList, *paths.Path, error) {
+	if err := coreBuildPath.MkdirAll(); err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	if coreBuildCachePath != nil {
+		if _, err := coreBuildCachePath.RelTo(buildPath); err != nil {
+			builderLogger.Info(tr("Couldn't deeply cache core build: %[1]s", err))
+			builderLogger.Info(tr("Running normal build of the core..."))
+			coreBuildCachePath = nil
+		} else if err := coreBuildCachePath.MkdirAll(); err != nil {
+			return nil, nil, errors.WithStack(err)
+		}
+	}
+
+	archiveFile, objectFiles, err := compileCore(
+		onlyUpdateCompilationDatabase, clean,
+		actualPlatform,
+		coreBuildPath, coreBuildCachePath,
+		buildProperties,
+		compilationDatabase,
+		jobs,
+		builderLogger,
+		progress, progressCB,
+	)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	return objectFiles, archiveFile, nil
+}
+
+func compileCore(
+	onlyUpdateCompilationDatabase, clean bool,
+	actualPlatform *cores.PlatformRelease,
+	buildPath, buildCachePath *paths.Path,
+	buildProperties *properties.Map,
+	compilationDatabase *compilation.Database,
+	jobs int,
+	builderLogger *logger.BuilderLogger,
+	progress *progress.Struct, progressCB rpc.TaskProgressCB,
+) (*paths.Path, paths.PathList, error) {
+	coreFolder := buildProperties.GetPath("build.core.path")
+	variantFolder := buildProperties.GetPath("build.variant.path")
+	targetCoreFolder := buildProperties.GetPath("runtime.platform.path")
+
+	includes := []string{coreFolder.String()}
+	if variantFolder != nil && variantFolder.IsDir() {
+		includes = append(includes, variantFolder.String())
+	}
+	includes = f.Map(includes, cpp.WrapWithHyphenI)
+
+	var err error
+	variantObjectFiles := paths.NewPathList()
+	if variantFolder != nil && variantFolder.IsDir() {
+		variantObjectFiles, err = utils.CompileFilesRecursive(
+			variantFolder, buildPath, buildProperties, includes,
+			onlyUpdateCompilationDatabase,
+			compilationDatabase,
+			jobs,
+			builderLogger,
+			progress, progressCB,
+		)
+		if err != nil {
+			return nil, nil, errors.WithStack(err)
+		}
+	}
+
+	var targetArchivedCore *paths.Path
+	if buildCachePath != nil {
+		realCoreFolder := coreFolder.Parent().Parent()
+		archivedCoreName := GetCachedCoreArchiveDirName(
+			buildProperties.Get("build.fqbn"),
+			buildProperties.Get("compiler.optimization_flags"),
+			realCoreFolder,
+		)
+		targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a")
+
+		if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) {
+			return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err))
+		}
+
+		var canUseArchivedCore bool
+		if onlyUpdateCompilationDatabase || clean {
+			canUseArchivedCore = false
+		} else if isOlder, err := utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder {
+			// Recreate the archive if ANY of the core files (including platform.txt) has changed
+			canUseArchivedCore = false
+		} else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) {
+			canUseArchivedCore = true
+		} else if isOlder, err := utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder {
+			// Recreate the archive if ANY of the build core files (including platform.txt) has changed
+			canUseArchivedCore = false
+		} else {
+			canUseArchivedCore = true
+		}
+
+		if canUseArchivedCore {
+			// use archived core
+			if builderLogger.Verbose() {
+				builderLogger.Info(tr("Using precompiled core: %[1]s", targetArchivedCore))
+			}
+			return targetArchivedCore, variantObjectFiles, nil
+		}
+	}
+
+	coreObjectFiles, err := utils.CompileFilesRecursive(
+		coreFolder, buildPath, buildProperties, includes,
+		onlyUpdateCompilationDatabase,
+		compilationDatabase,
+		jobs,
+		builderLogger,
+		progress, progressCB,
+	)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles(
+		buildPath, paths.New("core.a"), coreObjectFiles, buildProperties,
+		onlyUpdateCompilationDatabase, builderLogger.Verbose(), builderLogger.Stdout(), builderLogger.Stderr(),
+	)
+	if builderLogger.Verbose() {
+		builderLogger.Info(string(verboseInfo))
+	}
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	// archive core.a
+	if targetArchivedCore != nil && !onlyUpdateCompilationDatabase {
+		err := archiveFile.CopyTo(targetArchivedCore)
+		if builderLogger.Verbose() {
+			if err == nil {
+				builderLogger.Info(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore))
+			} else if os.IsNotExist(err) {
+				builderLogger.Info(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s",
+					actualPlatform,
+					"https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file"))
+			} else {
+				builderLogger.Info(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err))
+			}
+		}
+	}
+
+	return archiveFile, variantObjectFiles, nil
+}
+
+// GetCachedCoreArchiveDirName returns the directory name to be used to store
+// the global cached core.a.
+func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string {
+	fqbnToUnderscore := strings.ReplaceAll(fqbn, ":", "_")
+	fqbnToUnderscore = strings.ReplaceAll(fqbnToUnderscore, "=", "_")
+	if absCoreFolder, err := coreFolder.Abs(); err == nil {
+		coreFolder = absCoreFolder
+	} // silently continue if absolute path can't be detected
+
+	md5Sum := func(data []byte) string {
+		md5sumBytes := md5.Sum(data)
+		return hex.EncodeToString(md5sumBytes[:])
+	}
+	hash := md5Sum([]byte(coreFolder.String() + optimizationFlags))
+	realName := fqbnToUnderscore + "_" + hash
+	if len(realName) > 100 {
+		// avoid really long names, simply hash the name again
+		realName = md5Sum([]byte(realName))
+	}
+	return realName
+}