From e6fed302c790584dc5be2c8904db0156df5ac46d Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 15 Jul 2021 00:26:03 -0700 Subject: [PATCH 1/3] Export `libraries.loadRepoListFromFile()` Access to this function from outside the package is needed for validating the registry data file. --- internal/libraries/repolist.go | 7 +-- internal/libraries/repolist_test.go | 79 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/internal/libraries/repolist.go b/internal/libraries/repolist.go index c3b873b9..911c677d 100644 --- a/internal/libraries/repolist.go +++ b/internal/libraries/repolist.go @@ -33,7 +33,8 @@ import ( "strings" ) -func loadRepoListFromFile(filename string) ([]*Repo, error) { +// LoadRepoListFromFile returns an unfiltered list of library registry entries loaded from the given data file. +func LoadRepoListFromFile(filename string) ([]*Repo, error) { file, err := os.Open(filename) if err != nil { return nil, err @@ -153,9 +154,9 @@ func toListOfUniqueRepos(repos []*Repo) []*Repo { return finalRepos } -// ListRepos loads a list from the given filename. +// ListRepos returns a filtered list of library registry entries loaded from the given data file. func ListRepos(reposFilename string) ([]*Repo, error) { - repos, err := loadRepoListFromFile(reposFilename) + repos, err := LoadRepoListFromFile(reposFilename) if err != nil { return nil, err } diff --git a/internal/libraries/repolist_test.go b/internal/libraries/repolist_test.go index 8b00fd67..9e3c2ca8 100644 --- a/internal/libraries/repolist_test.go +++ b/internal/libraries/repolist_test.go @@ -26,9 +26,88 @@ package libraries import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestLoadRepoListFromFile(t *testing.T) { + _, err := LoadRepoListFromFile("./testdata/nonexistent.txt") + assert.Error(t, err, "Attempt to load non-existent registry data file") + + repos, err := LoadRepoListFromFile("./testdata/git_test_repos.txt") + require.NoError(t, err) + + reposAssertion := []*Repo{ + { + URL: "https://github.com/arduino-libraries", + Types: []string{"Arduino"}, + LibraryName: "libraries", + }, + { + URL: "git@github.com:PaulStoffregen/Audio.git", + Types: []string{"Contributed"}, + LibraryName: "Audio", + }, + { + URL: "https://github.com/PaulStoffregen/OctoWS2811.git", + Types: []string{"Arduino", "Contributed"}, + LibraryName: "OctoWS2811", + }, + { + URL: "https://github.com/PaulStoffregen/AltSoftSerial.git", + Types: []string{"Contributed"}, + LibraryName: "AltSoftSerial", + }, + { + URL: "https://github.com/Cheong2K/ble-sdk-arduino.git", + Types: []string{"Contributed"}, + LibraryName: "ble-sdk-arduino", + }, + { + URL: "https://github.com/arduino-libraries/Bridge.git", + Types: []string{"Contributed"}, + LibraryName: "Bridge", + }, + { + URL: "https://github.com/adafruit/Adafruit_ADS1X15.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_ADS1X15", + }, + { + URL: "https://github.com/adafruit/Adafruit_ADXL345.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_ADXL345", + }, + { + URL: "https://github.com/adafruit/Adafruit_AHRS.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_AHRS", + }, + { + URL: "https://github.com/adafruit/Adafruit_AM2315.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_AM2315", + }, + { + URL: "https://github.com/arduino-libraries/Scheduler.git", + Types: []string{"Arduino"}, + LibraryName: "Scheduler", + }, + { + URL: "https://github.com/arduino-libraries/SD.git", + Types: []string{"Arduino"}, + LibraryName: "SD", + }, + { + URL: "https://github.com/arduino-libraries/Servo.git", + Types: []string{"Arduino"}, + LibraryName: "Servo", + }, + } + + assert.Equal(t, reposAssertion, repos) +} + func TestListRepos(t *testing.T) { repos, err := ListRepos("./testdata/git_test_repos.txt") From 81a15cf616f76ec5493342c8203d1ad961a217d5 Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 15 Jul 2021 01:09:15 -0700 Subject: [PATCH 2/3] Expose registry handling API externally Since they were intended only for use in the "sync_libraries" application, the module's packages were made `internal` to clearly communicate that they are not part of a public API so that we will not need to be concerned with unintentional impacts on other projects if breaking changes to the internal API are required in the course of development on the code base. Select parts of the API may be exposed externally as they are determined to be useful for other projects. Such a case has arisen with the need to validate the library registry data file format. --- internal/libraries/repolist_test.go | 104 -------------- libraries/repolist.go | 41 ++++++ libraries/repolist_test.go | 135 ++++++++++++++++++ .../testdata/git_test_repos.txt | 0 4 files changed, 176 insertions(+), 104 deletions(-) create mode 100644 libraries/repolist.go create mode 100644 libraries/repolist_test.go rename {internal/libraries => libraries}/testdata/git_test_repos.txt (100%) diff --git a/internal/libraries/repolist_test.go b/internal/libraries/repolist_test.go index 9e3c2ca8..012980b1 100644 --- a/internal/libraries/repolist_test.go +++ b/internal/libraries/repolist_test.go @@ -26,113 +26,9 @@ package libraries import ( "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestLoadRepoListFromFile(t *testing.T) { - _, err := LoadRepoListFromFile("./testdata/nonexistent.txt") - assert.Error(t, err, "Attempt to load non-existent registry data file") - - repos, err := LoadRepoListFromFile("./testdata/git_test_repos.txt") - require.NoError(t, err) - - reposAssertion := []*Repo{ - { - URL: "https://github.com/arduino-libraries", - Types: []string{"Arduino"}, - LibraryName: "libraries", - }, - { - URL: "git@github.com:PaulStoffregen/Audio.git", - Types: []string{"Contributed"}, - LibraryName: "Audio", - }, - { - URL: "https://github.com/PaulStoffregen/OctoWS2811.git", - Types: []string{"Arduino", "Contributed"}, - LibraryName: "OctoWS2811", - }, - { - URL: "https://github.com/PaulStoffregen/AltSoftSerial.git", - Types: []string{"Contributed"}, - LibraryName: "AltSoftSerial", - }, - { - URL: "https://github.com/Cheong2K/ble-sdk-arduino.git", - Types: []string{"Contributed"}, - LibraryName: "ble-sdk-arduino", - }, - { - URL: "https://github.com/arduino-libraries/Bridge.git", - Types: []string{"Contributed"}, - LibraryName: "Bridge", - }, - { - URL: "https://github.com/adafruit/Adafruit_ADS1X15.git", - Types: []string{"Recommended"}, - LibraryName: "Adafruit_ADS1X15", - }, - { - URL: "https://github.com/adafruit/Adafruit_ADXL345.git", - Types: []string{"Recommended"}, - LibraryName: "Adafruit_ADXL345", - }, - { - URL: "https://github.com/adafruit/Adafruit_AHRS.git", - Types: []string{"Recommended"}, - LibraryName: "Adafruit_AHRS", - }, - { - URL: "https://github.com/adafruit/Adafruit_AM2315.git", - Types: []string{"Recommended"}, - LibraryName: "Adafruit_AM2315", - }, - { - URL: "https://github.com/arduino-libraries/Scheduler.git", - Types: []string{"Arduino"}, - LibraryName: "Scheduler", - }, - { - URL: "https://github.com/arduino-libraries/SD.git", - Types: []string{"Arduino"}, - LibraryName: "SD", - }, - { - URL: "https://github.com/arduino-libraries/Servo.git", - Types: []string{"Arduino"}, - LibraryName: "Servo", - }, - } - - assert.Equal(t, reposAssertion, repos) -} - -func TestListRepos(t *testing.T) { - repos, err := ListRepos("./testdata/git_test_repos.txt") - - require.Equal(t, 11, len(repos)) - - require.Equal(t, "https://github.com/PaulStoffregen/OctoWS2811.git", repos[0].URL) - require.Equal(t, "https://github.com/PaulStoffregen/AltSoftSerial.git", repos[1].URL) - - require.Equal(t, "https://github.com/Cheong2K/ble-sdk-arduino.git", repos[2].URL) - require.Equal(t, "https://github.com/arduino-libraries/Bridge.git", repos[3].URL) - require.Equal(t, "https://github.com/adafruit/Adafruit_ADS1X15.git", repos[4].URL) - require.Equal(t, "https://github.com/adafruit/Adafruit_ADXL345.git", repos[5].URL) - require.Equal(t, "https://github.com/adafruit/Adafruit_AHRS.git", repos[6].URL) - require.Equal(t, "https://github.com/adafruit/Adafruit_AM2315.git", repos[7].URL) - require.Equal(t, "https://github.com/arduino-libraries/Scheduler.git", repos[8].URL) - require.Equal(t, "https://github.com/arduino-libraries/SD.git", repos[9].URL) - require.Equal(t, "https://github.com/arduino-libraries/Servo.git", repos[10].URL) - require.Error(t, err) - - error, ok := err.(GitURLsError) - require.True(t, ok) - require.Equal(t, "https://github.com/arduino-libraries", error.Repos[0].URL) - require.Equal(t, "git@github.com:PaulStoffregen/Audio.git", error.Repos[1].URL) -} - func TestRepoFolderPathDetermination(t *testing.T) { repo := &Repo{URL: "https://github.com/arduino-libraries/Servo.git"} f, err := repo.AsFolder() diff --git a/libraries/repolist.go b/libraries/repolist.go new file mode 100644 index 00000000..ab806e4f --- /dev/null +++ b/libraries/repolist.go @@ -0,0 +1,41 @@ +// This file is part of libraries-repository-engine. +// +// Copyright 2021 ARDUINO SA (http://www.arduino.cc/) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package libraries + +import ( + "arduino.cc/repository/internal/libraries" +) + +// LoadRepoListFromFile returns an unfiltered list of library registry entries loaded from the given data file. +func LoadRepoListFromFile(filename string) ([]*Repo, error) { + return libraries.LoadRepoListFromFile(filename) +} + +// Repo is the type for the library repository data. +type Repo = libraries.Repo + +// ListRepos returns a filtered list of library registry entries loaded from the given data file. +func ListRepos(reposFilename string) ([]*Repo, error) { + return libraries.ListRepos(reposFilename) +} diff --git a/libraries/repolist_test.go b/libraries/repolist_test.go new file mode 100644 index 00000000..da1313e0 --- /dev/null +++ b/libraries/repolist_test.go @@ -0,0 +1,135 @@ +// This file is part of libraries-repository-engine. +// +// Copyright 2021 ARDUINO SA (http://www.arduino.cc/) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package libraries + +import ( + "testing" + + "arduino.cc/repository/internal/libraries" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadRepoListFromFile(t *testing.T) { + _, err := LoadRepoListFromFile("./testdata/nonexistent.txt") + assert.Error(t, err, "Attempt to load non-existent registry data file") + + repos, err := LoadRepoListFromFile("./testdata/git_test_repos.txt") + require.NoError(t, err) + + reposAssertion := []*Repo{ + { + URL: "https://github.com/arduino-libraries", + Types: []string{"Arduino"}, + LibraryName: "libraries", + }, + { + URL: "git@github.com:PaulStoffregen/Audio.git", + Types: []string{"Contributed"}, + LibraryName: "Audio", + }, + { + URL: "https://github.com/PaulStoffregen/OctoWS2811.git", + Types: []string{"Arduino", "Contributed"}, + LibraryName: "OctoWS2811", + }, + { + URL: "https://github.com/PaulStoffregen/AltSoftSerial.git", + Types: []string{"Contributed"}, + LibraryName: "AltSoftSerial", + }, + { + URL: "https://github.com/Cheong2K/ble-sdk-arduino.git", + Types: []string{"Contributed"}, + LibraryName: "ble-sdk-arduino", + }, + { + URL: "https://github.com/arduino-libraries/Bridge.git", + Types: []string{"Contributed"}, + LibraryName: "Bridge", + }, + { + URL: "https://github.com/adafruit/Adafruit_ADS1X15.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_ADS1X15", + }, + { + URL: "https://github.com/adafruit/Adafruit_ADXL345.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_ADXL345", + }, + { + URL: "https://github.com/adafruit/Adafruit_AHRS.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_AHRS", + }, + { + URL: "https://github.com/adafruit/Adafruit_AM2315.git", + Types: []string{"Recommended"}, + LibraryName: "Adafruit_AM2315", + }, + { + URL: "https://github.com/arduino-libraries/Scheduler.git", + Types: []string{"Arduino"}, + LibraryName: "Scheduler", + }, + { + URL: "https://github.com/arduino-libraries/SD.git", + Types: []string{"Arduino"}, + LibraryName: "SD", + }, + { + URL: "https://github.com/arduino-libraries/Servo.git", + Types: []string{"Arduino"}, + LibraryName: "Servo", + }, + } + + assert.Equal(t, reposAssertion, repos) +} + +func TestListRepos(t *testing.T) { + repos, err := ListRepos("./testdata/git_test_repos.txt") + + require.Equal(t, 11, len(repos)) + + require.Equal(t, "https://github.com/PaulStoffregen/OctoWS2811.git", repos[0].URL) + require.Equal(t, "https://github.com/PaulStoffregen/AltSoftSerial.git", repos[1].URL) + + require.Equal(t, "https://github.com/Cheong2K/ble-sdk-arduino.git", repos[2].URL) + require.Equal(t, "https://github.com/arduino-libraries/Bridge.git", repos[3].URL) + require.Equal(t, "https://github.com/adafruit/Adafruit_ADS1X15.git", repos[4].URL) + require.Equal(t, "https://github.com/adafruit/Adafruit_ADXL345.git", repos[5].URL) + require.Equal(t, "https://github.com/adafruit/Adafruit_AHRS.git", repos[6].URL) + require.Equal(t, "https://github.com/adafruit/Adafruit_AM2315.git", repos[7].URL) + require.Equal(t, "https://github.com/arduino-libraries/Scheduler.git", repos[8].URL) + require.Equal(t, "https://github.com/arduino-libraries/SD.git", repos[9].URL) + require.Equal(t, "https://github.com/arduino-libraries/Servo.git", repos[10].URL) + require.Error(t, err) + + error, ok := err.(libraries.GitURLsError) + require.True(t, ok) + require.Equal(t, "https://github.com/arduino-libraries", error.Repos[0].URL) + require.Equal(t, "git@github.com:PaulStoffregen/Audio.git", error.Repos[1].URL) +} diff --git a/internal/libraries/testdata/git_test_repos.txt b/libraries/testdata/git_test_repos.txt similarity index 100% rename from internal/libraries/testdata/git_test_repos.txt rename to libraries/testdata/git_test_repos.txt From 121df84fc9c7e385ca190d29f8e73b03ced06d9f Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 15 Jul 2021 04:02:27 -0700 Subject: [PATCH 3/3] Use true module path The previous module path `arduino.cc/repository` only served as an arbitrary identifier, but did not have the expected support for `go get`/`go install`: ``` $ go get arduino.cc/repository go get arduino.cc/repository: unrecognized import path "arduino.cc/repository": reading https://arduino.cc/repository?go-get=1: 404 Not Found ``` --- go.mod | 2 +- internal/libraries/db/library.go | 2 +- internal/libraries/git_integration_test.go | 4 ++-- internal/libraries/repoarchive.go | 4 ++-- internal/libraries/repoclone.go | 4 ++-- internal/libraries/zip/ziphelper.go | 2 +- libraries/repolist.go | 2 +- libraries/repolist_test.go | 2 +- sync_libraries.go | 8 ++++---- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 47c21b3f..40c0cbeb 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module arduino.cc/repository +module github.com/arduino/libraries-repository-engine go 1.14 diff --git a/internal/libraries/db/library.go b/internal/libraries/db/library.go index d4218939..e9e572eb 100644 --- a/internal/libraries/db/library.go +++ b/internal/libraries/db/library.go @@ -28,7 +28,7 @@ import ( "regexp" "strings" - "arduino.cc/repository/internal/libraries/metadata" + "github.com/arduino/libraries-repository-engine/internal/libraries/metadata" ) // FromLibraryToRelease extract a Release from LibraryMetadata. LibraryMetadata must be diff --git a/internal/libraries/git_integration_test.go b/internal/libraries/git_integration_test.go index 30c3d0f5..a17748f7 100644 --- a/internal/libraries/git_integration_test.go +++ b/internal/libraries/git_integration_test.go @@ -29,8 +29,8 @@ import ( "path/filepath" "testing" - "arduino.cc/repository/internal/libraries/db" - "arduino.cc/repository/internal/libraries/gitutils" + "github.com/arduino/libraries-repository-engine/internal/libraries/db" + "github.com/arduino/libraries-repository-engine/internal/libraries/gitutils" "github.com/stretchr/testify/require" ) diff --git a/internal/libraries/repoarchive.go b/internal/libraries/repoarchive.go index 25c3122a..ce9eeae5 100644 --- a/internal/libraries/repoarchive.go +++ b/internal/libraries/repoarchive.go @@ -28,8 +28,8 @@ import ( "path/filepath" "regexp" - "arduino.cc/repository/internal/libraries/metadata" - "arduino.cc/repository/internal/libraries/zip" + "github.com/arduino/libraries-repository-engine/internal/libraries/metadata" + "github.com/arduino/libraries-repository-engine/internal/libraries/zip" ) // ZipRepo creates a ZIP archive of the repo folder and returns its path. diff --git a/internal/libraries/repoclone.go b/internal/libraries/repoclone.go index cae368b4..11dcce43 100644 --- a/internal/libraries/repoclone.go +++ b/internal/libraries/repoclone.go @@ -28,11 +28,11 @@ import ( "os" "path/filepath" - "arduino.cc/repository/internal/libraries/db" + "github.com/arduino/libraries-repository-engine/internal/libraries/db" "fmt" - "arduino.cc/repository/internal/libraries/metadata" + "github.com/arduino/libraries-repository-engine/internal/libraries/metadata" "github.com/go-git/go-git/v5" ) diff --git a/internal/libraries/zip/ziphelper.go b/internal/libraries/zip/ziphelper.go index e2ded49d..f9f728b2 100644 --- a/internal/libraries/zip/ziphelper.go +++ b/internal/libraries/zip/ziphelper.go @@ -30,7 +30,7 @@ import ( "os/exec" "path/filepath" - "arduino.cc/repository/internal/libraries/file" + "github.com/arduino/libraries-repository-engine/internal/libraries/file" ) // Directory creates a new zip archive that contains a copy of "rootFolder" into "zipFile". diff --git a/libraries/repolist.go b/libraries/repolist.go index ab806e4f..94f7b78b 100644 --- a/libraries/repolist.go +++ b/libraries/repolist.go @@ -24,7 +24,7 @@ package libraries import ( - "arduino.cc/repository/internal/libraries" + "github.com/arduino/libraries-repository-engine/internal/libraries" ) // LoadRepoListFromFile returns an unfiltered list of library registry entries loaded from the given data file. diff --git a/libraries/repolist_test.go b/libraries/repolist_test.go index da1313e0..dc2e3442 100644 --- a/libraries/repolist_test.go +++ b/libraries/repolist_test.go @@ -26,7 +26,7 @@ package libraries import ( "testing" - "arduino.cc/repository/internal/libraries" + "github.com/arduino/libraries-repository-engine/internal/libraries" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/sync_libraries.go b/sync_libraries.go index 0c38cda1..60f8e0d9 100644 --- a/sync_libraries.go +++ b/sync_libraries.go @@ -32,11 +32,11 @@ import ( "os" "path/filepath" - "arduino.cc/repository/internal/libraries" - "arduino.cc/repository/internal/libraries/db" - "arduino.cc/repository/internal/libraries/gitutils" - "arduino.cc/repository/internal/libraries/hash" cc "github.com/arduino/golang-concurrent-workers" + "github.com/arduino/libraries-repository-engine/internal/libraries" + "github.com/arduino/libraries-repository-engine/internal/libraries/db" + "github.com/arduino/libraries-repository-engine/internal/libraries/gitutils" + "github.com/arduino/libraries-repository-engine/internal/libraries/hash" "github.com/go-git/go-git/v5/plumbing" )