arduino · cmaglie · Aug 12, 2016 · Aug 5, 2016 · Aug 11, 2016 · Aug 11, 2016
diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java
@@ -344,13 +344,15 @@ public Base(String[] args) throws Exception {
     PreferencesData.save();
 
     if (parser.isInstallBoard()) {
-      ContributionsIndexer indexer = new ContributionsIndexer(BaseNoGui.getSettingsFolder(), BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
+      ContributionsIndexer indexer = new ContributionsIndexer(
+          BaseNoGui.getSettingsFolder(), BaseNoGui.getHardwareFolder(),
+          BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
       ProgressListener progressListener = new ConsoleProgressListener();
 
       List<String> downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener);
       contributionInstaller.deleteUnknownFiles(downloadedPackageIndexFiles);
       indexer.parseIndex();
-      indexer.syncWithFilesystem(BaseNoGui.getHardwareFolder());
+      indexer.syncWithFilesystem();
 
       String[] boardToInstallParts = parser.getBoardToInstall().split(":");
 

diff --git a/arduino-core/src/cc/arduino/Constants.java b/arduino-core/src/cc/arduino/Constants.java
@@ -39,6 +39,7 @@ public class Constants {
   public static final String PREF_CONTRIBUTIONS_TRUST_ALL = "contributions.trust.all";
 
   public static final String DEFAULT_INDEX_FILE_NAME = "package_index.json";
+  public static final String BUNDLED_INDEX_FILE_NAME = "package_index_bundled.json";
   public static final List<String> PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel");
 
   public static final String LIBRARY_DEVELOPMENT_FLAG_FILE = ".development";

diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java
@@ -79,16 +79,15 @@ public synchronized List<String> install(ContributedPlatform contributedPlatform
     }
 
     // Do not download already installed tools
-    List<ContributedTool> tools = new LinkedList<>(contributedPlatform.getResolvedTools());
-    Iterator<ContributedTool> toolsIterator = tools.iterator();
-    while (toolsIterator.hasNext()) {
-      ContributedTool tool = toolsIterator.next();
+    List<ContributedTool> tools = new ArrayList<>();
+    for (ContributedTool tool : contributedPlatform.getResolvedTools()) {
       DownloadableContribution downloadable = tool.getDownloadableContribution(platform);
       if (downloadable == null) {
         throw new Exception(format(tr("Tool {0} is not available for your operating system."), tool.getName()));
       }
-      if (downloadable.isInstalled()) {
-        toolsIterator.remove();
+      // Download the tool if it's not installed or it's a built-in tool
+      if (!downloadable.isInstalled() || downloadable.isReadOnly()) {
+        tools.add(tool);
       }
     }
 
@@ -125,10 +124,10 @@ public synchronized List<String> install(ContributedPlatform contributedPlatform
 
     List<Map.Entry<ContributedToolReference, ContributedTool>> resolvedToolReferences = contributedPlatform.getResolvedToolReferences().entrySet()
       .stream()
-      .filter((entry) -> !entry.getValue().getDownloadableContribution(platform).isInstalled())
+      .filter((entry) -> !entry.getValue().getDownloadableContribution(platform).isInstalled()
+          || entry.getValue().getDownloadableContribution(platform).isReadOnly())
       .collect(Collectors.toList());
 
-
     int i = 1;
     for (Map.Entry<ContributedToolReference, ContributedTool> entry : resolvedToolReferences) {
       progress.setStatus(format(tr("Installing tools ({0}/{1})..."), i, resolvedToolReferences.size()));
@@ -249,11 +248,16 @@ public synchronized List<String> remove(ContributedPlatform contributedPlatform)
 
     // Check if the tools are no longer needed
     for (ContributedTool tool : contributedPlatform.getResolvedTools()) {
-      if (BaseNoGui.indexer.isContributedToolUsed(contributedPlatform, tool)) {
+      // Do not remove used tools
+      if (BaseNoGui.indexer.isContributedToolUsed(contributedPlatform, tool))
         continue;
-      }
 
+      // Do not remove built-in tools
       DownloadableContribution toolContrib = tool.getDownloadableContribution(platform);
+      if (toolContrib.isReadOnly())
+        continue;
+
+      // Ok, delete the tool
       File destFolder = toolContrib.getInstalledFolder();
       FileUtils.recursiveDelete(destFolder);
       toolContrib.setInstalled(false);

diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java
@@ -141,8 +141,4 @@ public String toString() {
       res += pack + "\n";
     return res;
   }
-
-  public void setTrusted() {
-    getPackages().stream().forEach(pack -> pack.setTrusted(true));
-  }
 }
diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java
@@ -66,26 +66,40 @@ public class ContributionsIndexer {
   private final File packagesFolder;
   private final File stagingFolder;
   private final File preferencesFolder;
+  private final File builtInHardwareFolder;
   private final Platform platform;
   private final SignatureVerifier signatureVerifier;
   private ContributionsIndex index;
 
-  public ContributionsIndexer(File preferencesFolder, Platform platform, SignatureVerifier signatureVerifier) {
+  public ContributionsIndexer(File preferencesFolder, File builtInHardwareFolder, Platform platform, SignatureVerifier signatureVerifier) {
     this.preferencesFolder = preferencesFolder;
+    this.builtInHardwareFolder = builtInHardwareFolder;
     this.platform = platform;
     this.signatureVerifier = signatureVerifier;
     packagesFolder = new File(preferencesFolder, "packages");
     stagingFolder = new File(preferencesFolder, "staging" + File.separator + "packages");
   }
 
   public void parseIndex() throws Exception {
+    // Read bundled index...
+    File bundledIndexFile = new File(builtInHardwareFolder, Constants.BUNDLED_INDEX_FILE_NAME);
+    index = parseIndex(bundledIndexFile);
+
+    // ...and overlay the default index if present
     File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME);
-    if (!PreferencesData.getBoolean("allow_insecure_packages") && !signatureVerifier.isSigned(defaultIndexFile)) {
-      throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME);
+    if (defaultIndexFile.exists()) {
+      // Check main index signature
+      if (!PreferencesData.getBoolean("allow_insecure_packages") && !signatureVerifier.isSigned(defaultIndexFile)) {
+        throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME);
+      }
+
+      mergeContributions(parseIndex(defaultIndexFile), defaultIndexFile);
     }
-    index = parseIndex(defaultIndexFile);
-    index.setTrusted();
 
+    // Set main and bundled indexes as trusted
+    index.getPackages().forEach(pack -> pack.setTrusted(true));
+
+    // Overlay 3rd party indexes
     File[] indexFiles = preferencesFolder.listFiles(new TestPackageIndexFilenameFilter(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)));
 
     for (File indexFile : indexFiles) {
@@ -98,6 +112,7 @@ public void parseIndex() throws Exception {
       }
     }
 
+    // Fill tools and toolsDependency cross references
     List<ContributedPackage> packages = index.getPackages();
     Collection<ContributedPackage> packagesWithTools = packages.stream()
       .filter(input -> input.getTools() != null && !input.getTools().isEmpty())
@@ -188,32 +203,42 @@ private ContributionsIndex parseIndex(File indexFile) throws IOException {
     }
   }
 
-  public void syncWithFilesystem(File hardwareFolder) throws IOException {
-    syncBuiltInHardwareFolder(hardwareFolder);
+  public void syncWithFilesystem() throws IOException {
+    syncBuiltInHardware();
 
-    syncLocalPackagesFolder();
+    syncLocalPackages();
   }
 
-  private void syncBuiltInHardwareFolder(File hardwareFolder) throws IOException {
+  private void syncBuiltInHardware() throws IOException {
     if (index == null) {
       return;
     }
-    for (File folder : hardwareFolder.listFiles(ONLY_DIRS)) {
+    for (File folder : builtInHardwareFolder.listFiles(ONLY_DIRS)) {
       ContributedPackage pack = index.findPackage(folder.getName());
-      if (pack != null) {
-        syncBuiltInPackageWithFilesystem(pack, folder);
-
-        File toolsFolder = new File(hardwareFolder, "tools");
-        if (toolsFolder.isDirectory()) {
-          for (File toolFolder : toolsFolder.listFiles(ONLY_DIRS)) {
-            File builtInToolsMetadata = new File(toolFolder, "builtin_tools_versions.txt");
-            if (builtInToolsMetadata.isFile()) {
-              PreferencesMap toolsMetadata = new PreferencesMap(builtInToolsMetadata).subTree(pack.getName());
-              for (Map.Entry<String, String> toolMetadata : toolsMetadata.entrySet()) {
-                syncToolWithFilesystem(pack, toolFolder, toolMetadata.getKey(), toolMetadata.getValue());
-              }
-            }
-          }
+      if (pack == null)
+        continue;
+      syncBuiltInPackageWithFilesystem(pack, folder);
+
+      File toolsFolder = new File(builtInHardwareFolder, "tools");
+      if (!toolsFolder.isDirectory())
+        continue;
+
+      for (File toolFolder : toolsFolder.listFiles(ONLY_DIRS)) {
+
+        // builtin_tools_versions.txt contains tools versions in the format:
+        // "PACKAGER.TOOL_NAME=TOOL_VERSION"
+        // for example:
+        // "arduino.avrdude=6.0.1-arduino5"
+
+        File versionsFile = new File(toolFolder, "builtin_tools_versions.txt");
+        if (!versionsFile.isFile())
+          continue;
+        PreferencesMap toolsVersion = new PreferencesMap(versionsFile).subTree(pack.getName());
+        for (String name : toolsVersion.keySet()) {
+          String version = toolsVersion.get(name);
+          DownloadableContribution tool = syncToolWithFilesystem(pack, toolFolder, name, version);
+          if (tool != null)
+            tool.setReadOnly(true);
         }
       }
     }
@@ -231,7 +256,7 @@ private void syncBuiltInPackageWithFilesystem(ContributedPackage pack, File hard
     }
   }
 
-  private void syncLocalPackagesFolder() {
+  private void syncLocalPackages() {
     if (!packagesFolder.isDirectory()) {
       return;
     }
@@ -272,21 +297,23 @@ private void syncPackageWithFilesystem(ContributedPackage pack, File root) {
     }
   }
 
-  private void syncToolWithFilesystem(ContributedPackage pack, File installationFolder, String toolName, String version) {
+  private DownloadableContribution syncToolWithFilesystem(ContributedPackage pack, File installationFolder, String toolName, String version) {
     ContributedTool tool = pack.findTool(toolName, version);
     if (tool == null) {
       tool = pack.findResolvedTool(toolName, version);
     }
     if (tool == null) {
-      return;
+      return null;
     }
     DownloadableContribution contrib = tool.getDownloadableContribution(platform);
     if (contrib == null) {
       System.err.println(tool + " seems to have no downloadable contributions for your operating system, but it is installed in\n" + installationFolder);
-      return;
+      return null;
     }
     contrib.setInstalled(true);
     contrib.setInstalledFolder(installationFolder);
+    contrib.setReadOnly(false);
+    return contrib;
   }
 
   private ContributedPlatform syncHardwareWithFilesystem(ContributedPackage pack, File installationFolder, String architecture, String version) {
@@ -350,7 +377,7 @@ public boolean isContributedToolUsed(ContributedPlatform platformToIgnore, Contr
         if (platformToIgnore.equals(platform)) {
           continue;
         }
-        if (!platform.isInstalled()) {
+        if (!platform.isInstalled() || platform.isReadOnly()) {
           continue;
         }
         for (ContributedTool requiredTool : platform.getResolvedTools()) {

diff --git a/arduino-core/src/processing/app/BaseNoGui.java b/arduino-core/src/processing/app/BaseNoGui.java
@@ -608,36 +608,19 @@ static public void initLogger() {
   }
 
   static public void initPackages() throws Exception {
-    indexer = new ContributionsIndexer(BaseNoGui.getSettingsFolder(), BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
-    File indexFile = indexer.getIndexFile("package_index.json");
-    File defaultPackageJsonFile = new File(getContentFile("dist"), "package_index.json");
-    if (!indexFile.isFile() || (defaultPackageJsonFile.isFile() && defaultPackageJsonFile.lastModified() > indexFile.lastModified())) {
-      FileUtils.copyFile(defaultPackageJsonFile, indexFile);
-    } else if (!indexFile.isFile()) {
-      // Otherwise create an empty packages index
-      FileOutputStream out = null;
-      try {
-        out = new FileOutputStream(indexFile);
-        out.write("{ \"packages\" : [ ] }".getBytes());
-      } finally {
-        IOUtils.closeQuietly(out);
-      }
-    }
-
-    File indexSignatureFile = indexer.getIndexFile("package_index.json.sig");
-    File defaultPackageJsonSignatureFile = new File(getContentFile("dist"), "package_index.json.sig");
-    if (!indexSignatureFile.isFile() || (defaultPackageJsonSignatureFile.isFile() && defaultPackageJsonSignatureFile.lastModified() > indexSignatureFile.lastModified())) {
-      FileUtils.copyFile(defaultPackageJsonSignatureFile, indexSignatureFile);
-    }
+    indexer = new ContributionsIndexer(getSettingsFolder(), getHardwareFolder(), getPlatform(),
+        new GPGDetachedSignatureVerifier());
 
     try {
       indexer.parseIndex();
     } catch (JsonProcessingException | SignatureVerificationFailedException e) {
+      File indexFile = indexer.getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME);
+      File indexSignatureFile = indexer.getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME + ".sig");
       FileUtils.deleteIfExists(indexFile);
       FileUtils.deleteIfExists(indexSignatureFile);
       throw e;
     }
-    indexer.syncWithFilesystem(getHardwareFolder());
+    indexer.syncWithFilesystem();
 
     packages = new LinkedHashMap<String, TargetPackage>();
     loadHardware(getHardwareFolder());