diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index bbccaa36b..fef212763 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -16,7 +16,7 @@ jobs: - name: Install Go uses: actions/setup-go@v1 with: - go-version: 1.14.x + go-version: 1.19.x - name: Checkout code uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1644dcfc9..cefbbdf6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: [1.15.x, 1.16.x] + go-version: [1.18.x, 1.19.x] platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} @@ -33,15 +33,16 @@ jobs: run: make test-coverage - name: Convert coverage to lcov - if: matrix.platform == 'ubuntu-latest' && matrix.go-version == '1.14.x' + if: matrix.platform == 'ubuntu-latest' && matrix.go-version == '1.19.x' uses: jandelgado/gcov2lcov-action@v1.0.0 with: infile: coverage.out outfile: coverage.lcov - name: Coveralls - if: matrix.platform == 'ubuntu-latest' && matrix.go-version == '1.14.x' + if: matrix.platform == 'ubuntu-latest' && matrix.go-version == '1.19.x' uses: coverallsapp/github-action@master + continue-on-error: true with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: coverage.lcov diff --git a/.gitignore b/.gitignore index 038dd9f1e..e8d25f548 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ coverage.out *~ coverage.txt profile.out +.tmp/ \ No newline at end of file diff --git a/_examples/README.md b/_examples/README.md index 92b9d4df4..3a4c539d0 100644 --- a/_examples/README.md +++ b/_examples/README.md @@ -19,6 +19,7 @@ Here you can find a list of annotated _go-git_ examples: - [branch](branch/main.go) - How to create and remove branches or any other kind of reference. - [tag](tag/main.go) - List/print repository tags. - [tag create and push](tag-create-push/main.go) - Create and push a new tag. +- [tag find if head is tagged](find-if-any-tag-point-head/main.go) - Find if `HEAD` is tagged. - [remotes](remotes/main.go) - Working with remotes: adding, removing, etc. - [progress](progress/main.go) - Printing the progress information from the sideband. - [revision](revision/main.go) - Solve a revision into a commit. diff --git a/_examples/azure_devops/main.go b/_examples/azure_devops/main.go new file mode 100644 index 000000000..9c02ca080 --- /dev/null +++ b/_examples/azure_devops/main.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "os" + + git "github.com/go-git/go-git/v5" + . "github.com/go-git/go-git/v5/_examples" + "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +func main() { + CheckArgs("", "", "", "") + url, directory, username, password := os.Args[1], os.Args[2], os.Args[3], os.Args[4] + + // Clone the given repository to the given directory + Info("git clone %s %s", url, directory) + + // Azure DevOps requires capabilities multi_ack / multi_ack_detailed, + // which are not fully implemented and by default are included in + // transport.UnsupportedCapabilities. + // + // The initial clone operations require a full download of the repository, + // and therefore those unsupported capabilities are not as crucial, so + // by removing them from that list allows for the first clone to work + // successfully. + // + // Additional fetches will yield issues, therefore work always from a clean + // clone until those capabilities are fully supported. + // + // New commits and pushes against a remote worked without any issues. + transport.UnsupportedCapabilities = []capability.Capability{ + capability.ThinPack, + } + + r, err := git.PlainClone(directory, false, &git.CloneOptions{ + Auth: &http.BasicAuth{ + Username: username, + Password: password, + }, + URL: url, + Progress: os.Stdout, + }) + CheckIfError(err) + + // ... retrieving the branch being pointed by HEAD + ref, err := r.Head() + CheckIfError(err) + // ... retrieving the commit object + commit, err := r.CommitObject(ref.Hash()) + CheckIfError(err) + + fmt.Println(commit) +} diff --git a/_examples/find-if-any-tag-point-head/main.go b/_examples/find-if-any-tag-point-head/main.go new file mode 100644 index 000000000..834aea2bb --- /dev/null +++ b/_examples/find-if-any-tag-point-head/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "os" + + "github.com/go-git/go-git/v5" + . "github.com/go-git/go-git/v5/_examples" + "github.com/go-git/go-git/v5/plumbing" +) + +// Basic example of how to find if HEAD is tagged. +func main() { + CheckArgs("") + path := os.Args[1] + + // We instantiate a new repository targeting the given path (the .git folder) + r, err := git.PlainOpen(path) + CheckIfError(err) + + // Get HEAD reference to use for comparison later on. + ref, err := r.Head() + CheckIfError(err) + + tags, err := r.Tags() + CheckIfError(err) + + // List all tags, both lightweight tags and annotated tags and see if some tag points to HEAD reference. + err = tags.ForEach(func(t *plumbing.Reference) error { + // This technique should work for both lightweight and annotated tags. + revHash, err := r.ResolveRevision(plumbing.Revision(t.Name())) + CheckIfError(err) + if *revHash == ref.Hash() { + fmt.Printf("Found tag %s with hash %s pointing to HEAD %s\n", t.Name().Short(), revHash, ref.Hash()) + } + return nil + }) +} diff --git a/_examples/remotes/main.go b/_examples/remotes/main.go index b1a91a9ef..d09957eae 100644 --- a/_examples/remotes/main.go +++ b/_examples/remotes/main.go @@ -33,7 +33,7 @@ func main() { CheckIfError(err) // List remotes from a repository - Info("git remotes -v") + Info("git remote -v") list, err := r.Remotes() CheckIfError(err) diff --git a/common_test.go b/common_test.go index 5f5bc4cd2..c0c4009e2 100644 --- a/common_test.go +++ b/common_test.go @@ -7,7 +7,6 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/format/packfile" - "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/filesystem" "github.com/go-git/go-git/v5/storage/memory" @@ -25,8 +24,7 @@ type BaseSuite struct { fixtures.Suite Repository *Repository - backupProtocol transport.Transport - cache map[string]*Repository + cache map[string]*Repository } func (s *BaseSuite) SetUpSuite(c *C) { @@ -198,3 +196,11 @@ func AssertReferences(c *C, r *Repository, expected map[string]string) { c.Assert(obtained, DeepEquals, expected) } } + +func AssertReferencesMissing(c *C, r *Repository, expected []string) { + for _, name := range expected { + _, err := r.Reference(plumbing.ReferenceName(name), false) + c.Assert(err, NotNil) + c.Assert(err, Equals, plumbing.ErrReferenceNotFound) + } +} diff --git a/config/branch.go b/config/branch.go index fe86cf542..652270a28 100644 --- a/config/branch.go +++ b/config/branch.go @@ -2,6 +2,7 @@ package config import ( "errors" + "strings" "github.com/go-git/go-git/v5/plumbing" format "github.com/go-git/go-git/v5/plumbing/format/config" @@ -26,6 +27,12 @@ type Branch struct { // "true" and "interactive". "false" is undocumented and // typically represented by the non-existence of this field Rebase string + // Description explains what the branch is for. + // Multi-line explanations may be used. + // + // Original git command to edit: + // git branch --edit-description + Description string raw *format.Subsection } @@ -75,9 +82,27 @@ func (b *Branch) marshal() *format.Subsection { b.raw.SetOption(rebaseKey, b.Rebase) } + if b.Description == "" { + b.raw.RemoveOption(descriptionKey) + } else { + desc := quoteDescription(b.Description) + b.raw.SetOption(descriptionKey, desc) + } + return b.raw } +// hack to trigger conditional quoting in the +// plumbing/format/config/Encoder.encodeOptions +// +// Current Encoder implementation uses Go %q format if value contains a backslash character, +// which is not consistent with reference git implementation. +// git just replaces newline characters with \n, while Encoder prints them directly. +// Until value quoting fix, we should escape description value by replacing newline characters with \n. +func quoteDescription(desc string) string { + return strings.ReplaceAll(desc, "\n", `\n`) +} + func (b *Branch) unmarshal(s *format.Subsection) error { b.raw = s @@ -85,6 +110,14 @@ func (b *Branch) unmarshal(s *format.Subsection) error { b.Remote = b.raw.Options.Get(remoteSection) b.Merge = plumbing.ReferenceName(b.raw.Options.Get(mergeKey)) b.Rebase = b.raw.Options.Get(rebaseKey) + b.Description = unquoteDescription(b.raw.Options.Get(descriptionKey)) return b.Validate() } + +// hack to enable conditional quoting in the +// plumbing/format/config/Encoder.encodeOptions +// goto quoteDescription for details. +func unquoteDescription(desc string) string { + return strings.ReplaceAll(desc, `\n`, "\n") +} diff --git a/config/config.go b/config/config.go index 1aee25a4c..8051bc145 100644 --- a/config/config.go +++ b/config/config.go @@ -15,7 +15,6 @@ import ( "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/internal/url" format "github.com/go-git/go-git/v5/plumbing/format/config" - "github.com/mitchellh/go-homedir" ) const ( @@ -150,7 +149,7 @@ func ReadConfig(r io.Reader) (*Config, error) { // config file to the given scope, a empty one is returned. func LoadConfig(scope Scope) (*Config, error) { if scope == LocalScope { - return nil, fmt.Errorf("LocalScope should be read from the a ConfigStorer.") + return nil, fmt.Errorf("LocalScope should be read from the a ConfigStorer") } files, err := Paths(scope) @@ -185,7 +184,7 @@ func Paths(scope Scope) ([]string, error) { files = append(files, filepath.Join(xdg, "git/config")) } - home, err := homedir.Dir() + home, err := os.UserHomeDir() if err != nil { return nil, err } @@ -247,6 +246,7 @@ const ( rebaseKey = "rebase" nameKey = "name" emailKey = "email" + descriptionKey = "description" defaultBranchKey = "defaultBranch" // DefaultPackWindow holds the number of previous objects used to diff --git a/config/config_test.go b/config/config_test.go index 6f0242d96..7e9483f6f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -50,6 +50,7 @@ func (s *ConfigSuite) TestUnmarshal(c *C) { [branch "master"] remote = origin merge = refs/heads/master + description = "Add support for branch description.\\n\\nEdit branch description: git branch --edit-description\\n" [init] defaultBranch = main [url "ssh://git@github.com/"] @@ -86,6 +87,7 @@ func (s *ConfigSuite) TestUnmarshal(c *C) { c.Assert(cfg.Submodules["qux"].Branch, Equals, "bar") c.Assert(cfg.Branches["master"].Remote, Equals, "origin") c.Assert(cfg.Branches["master"].Merge, Equals, plumbing.ReferenceName("refs/heads/master")) + c.Assert(cfg.Branches["master"].Description, Equals, "Add support for branch description.\n\nEdit branch description: git branch --edit-description\n") c.Assert(cfg.Init.DefaultBranch, Equals, "main") } @@ -111,6 +113,7 @@ func (s *ConfigSuite) TestMarshal(c *C) { [branch "master"] remote = origin merge = refs/heads/master + description = "Add support for branch description.\\n\\nEdit branch description: git branch --edit-description\\n" [url "ssh://git@github.com/"] insteadOf = https://github.com/ [init] @@ -149,9 +152,10 @@ func (s *ConfigSuite) TestMarshal(c *C) { } cfg.Branches["master"] = &Branch{ - Name: "master", - Remote: "origin", - Merge: "refs/heads/master", + Name: "master", + Remote: "origin", + Merge: "refs/heads/master", + Description: "Add support for branch description.\n\nEdit branch description: git branch --edit-description\n", } cfg.URLs["ssh://git@github.com/"] = &URL{ @@ -361,7 +365,9 @@ func (s *ConfigSuite) TestRemoveUrlOptions(c *C) { cfg.Remotes["alt"].URLs = []string{} buf, err = cfg.Marshal() + c.Assert(err, IsNil) if strings.Contains(string(buf), "url") { c.Fatal("conifg should not contain any url sections") } + c.Assert(err, IsNil) } diff --git a/config/refspec.go b/config/refspec.go index 4bfaa37bb..e2cf8c97b 100644 --- a/config/refspec.go +++ b/config/refspec.go @@ -64,7 +64,7 @@ func (s RefSpec) IsExactSHA1() bool { return plumbing.IsHash(s.Src()) } -// Src return the src side. +// Src returns the src side. func (s RefSpec) Src() string { spec := string(s) diff --git a/go.mod b/go.mod index 402613f0f..b214c8a8d 100644 --- a/go.mod +++ b/go.mod @@ -1,29 +1,28 @@ module github.com/go-git/go-git/v5 require ( - github.com/Microsoft/go-winio v0.4.16 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 + github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 github.com/acomagu/bufpipe v1.0.3 - github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/emirpasic/gods v1.12.0 - github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect - github.com/gliderlabs/ssh v0.2.2 + github.com/emirpasic/gods v1.18.1 + github.com/gliderlabs/ssh v0.3.5 github.com/go-git/gcfg v1.5.0 github.com/go-git/go-billy/v5 v5.3.1 - github.com/go-git/go-git-fixtures/v4 v4.2.1 - github.com/google/go-cmp v0.3.0 - github.com/imdario/mergo v0.3.12 + github.com/go-git/go-git-fixtures/v4 v4.3.1 + github.com/google/go-cmp v0.5.9 + github.com/imdario/mergo v0.3.13 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 github.com/jessevdk/go-flags v1.5.0 - github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 - github.com/mitchellh/go-homedir v1.1.0 + github.com/kevinburke/ssh_config v1.2.0 + github.com/pjbgf/sha1cd v0.2.0 + github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.1.0 - github.com/xanzy/ssh-agent v0.3.0 - golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b - golang.org/x/net v0.0.0-20210326060303-6b1517762897 - golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 - golang.org/x/text v0.3.3 + github.com/skeema/knownhosts v1.1.0 + github.com/xanzy/ssh-agent v0.3.2 + golang.org/x/crypto v0.3.0 + golang.org/x/net v0.2.0 + golang.org/x/sys v0.2.0 + golang.org/x/text v0.4.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 922724775..2c00f0092 100644 --- a/go.sum +++ b/go.sum @@ -1,44 +1,40 @@ -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= +github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= -github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.3.0 h1:KZL1OFdS+afiIjN4hr/zpj5cEtC0OJhbmTA18PsBb8c= -github.com/go-git/go-billy/v5 v5.3.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= -github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= +github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -48,46 +44,74 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pjbgf/sha1cd v0.2.0 h1:gIsJVwjbRviE4gydidGztxH1IlJQoYBcCrwG4Dz8wvM= +github.com/pjbgf/sha1cd v0.2.0/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= +github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= -github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= +github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs= -golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 h1:RX8C8PRZc2hTIod4ds8ij+/4RQX3AqhYj3uOHmyaz4E= -golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -97,7 +121,6 @@ gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/revision/parser.go b/internal/revision/parser.go index 8facf17ff..8a2a7190e 100644 --- a/internal/revision/parser.go +++ b/internal/revision/parser.go @@ -322,6 +322,8 @@ func (p *Parser) parseAt() (Revisioner, error) { } return AtDate{t}, nil + case tok == eof: + return nil, &ErrInvalidRevision{s: `missing "}" in @{} structure`} default: date += lit } @@ -424,6 +426,8 @@ func (p *Parser) parseCaretBraces() (Revisioner, error) { p.unscan() case tok != slash && start: return nil, &ErrInvalidRevision{fmt.Sprintf(`"%s" is not a valid revision suffix brace component`, lit)} + case tok == eof: + return nil, &ErrInvalidRevision{s: `missing "}" in ^{} structure`} case tok != cbrace: p.unscan() re += lit diff --git a/internal/revision/parser_test.go b/internal/revision/parser_test.go index 98403cc23..3a77b2f11 100644 --- a/internal/revision/parser_test.go +++ b/internal/revision/parser_test.go @@ -183,7 +183,7 @@ func (s *ParserSuite) TestParseWithValidExpression(c *C) { } } -func (s *ParserSuite) TestParseWithUnValidExpression(c *C) { +func (s *ParserSuite) TestParseWithInvalidExpression(c *C) { datas := map[string]error{ "..": &ErrInvalidRevision{`must not start with "."`}, "master^1master": &ErrInvalidRevision{`reference must be defined once at the beginning`}, @@ -198,6 +198,9 @@ func (s *ParserSuite) TestParseWithUnValidExpression(c *C) { "~1": &ErrInvalidRevision{`"~" or "^" statement must have a reference defined at the beginning`}, "master:/test": &ErrInvalidRevision{`":" statement is not valid, could be : :/`}, "master:0:README": &ErrInvalidRevision{`":" statement is not valid, could be : ::`}, + "^{/": &ErrInvalidRevision{`missing "}" in ^{} structure`}, + "~@{": &ErrInvalidRevision{`missing "}" in @{} structure`}, + "@@{{0": &ErrInvalidRevision{`missing "}" in @{} structure`}, } for s, e := range datas { @@ -230,7 +233,7 @@ func (s *ParserSuite) TestParseAtWithValidExpression(c *C) { } } -func (s *ParserSuite) TestParseAtWithUnValidExpression(c *C) { +func (s *ParserSuite) TestParseAtWithInvalidExpression(c *C) { datas := map[string]error{ "{test}": &ErrInvalidRevision{`wrong date "test" must fit ISO-8601 format : 2006-01-02T15:04:05Z`}, "{-1": &ErrInvalidRevision{`missing "}" in @{-n} structure`}, diff --git a/object_walker.go b/object_walker.go index 3fcdd2999..3a537bd80 100644 --- a/object_walker.go +++ b/object_walker.go @@ -60,7 +60,7 @@ func (p *objectWalker) walkObjectTree(hash plumbing.Hash) error { // Fetch the object. obj, err := object.GetObject(p.Storer, hash) if err != nil { - return fmt.Errorf("Getting object %s failed: %v", hash, err) + return fmt.Errorf("getting object %s failed: %v", hash, err) } // Walk all children depending on object type. switch obj := obj.(type) { @@ -98,7 +98,7 @@ func (p *objectWalker) walkObjectTree(hash plumbing.Hash) error { return p.walkObjectTree(obj.Target) default: // Error out on unhandled object types. - return fmt.Errorf("Unknown object %X %s %T\n", obj.ID(), obj.Type(), obj) + return fmt.Errorf("unknown object %X %s %T", obj.ID(), obj.Type(), obj) } return nil } diff --git a/options.go b/options.go index 70687965b..7e5c1b434 100644 --- a/options.go +++ b/options.go @@ -91,6 +91,8 @@ func (o *CloneOptions) Validate() error { type PullOptions struct { // Name of the remote to be pulled. If empty, uses the default. RemoteName string + // RemoteURL overrides the remote repo address with a custom URL + RemoteURL string // Remote branch to clone. If empty, uses HEAD. ReferenceName plumbing.ReferenceName // Fetch only ReferenceName if true. @@ -147,7 +149,9 @@ const ( type FetchOptions struct { // Name of the remote to fetch from. Defaults to origin. RemoteName string - RefSpecs []config.RefSpec + // RemoteURL overrides the remote repo address with a custom URL + RemoteURL string + RefSpecs []config.RefSpec // Depth limit fetching to the specified number of commits from the tip of // each remote branch history. Depth int @@ -192,8 +196,16 @@ func (o *FetchOptions) Validate() error { type PushOptions struct { // RemoteName is the name of the remote to be pushed to. RemoteName string - // RefSpecs specify what destination ref to update with what source - // object. A refspec with empty src can be used to delete a reference. + // RemoteURL overrides the remote repo address with a custom URL + RemoteURL string + // RefSpecs specify what destination ref to update with what source object. + // + // The format of a parameter is an optional plus +, followed by + // the source object , followed by a colon :, followed by the destination ref . + // The is often the name of the branch you would want to push, but it can be a SHA-1. + // The tells which ref on the remote side is updated with this push. + // + // A refspec with empty src can be used to delete a reference. RefSpecs []config.RefSpec // Auth credentials, if required, to use with the remote repository. Auth transport.AuthMethod @@ -206,13 +218,35 @@ type PushOptions struct { // Force allows the push to update a remote branch even when the local // branch does not descend from it. Force bool - // InsecureSkipTLS skips ssl verify if protocal is https + // InsecureSkipTLS skips ssl verify if protocol is https InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte // RequireRemoteRefs only allows a remote ref to be updated if its current // value is the one specified here. RequireRemoteRefs []config.RefSpec + // FollowTags will send any annotated tags with a commit target reachable from + // the refs already being pushed + FollowTags bool + // ForceWithLease allows a force push as long as the remote ref adheres to a "lease" + ForceWithLease *ForceWithLease + // PushOptions sets options to be transferred to the server during push. + Options map[string]string + // Atomic sets option to be an atomic push + Atomic bool +} + +// ForceWithLease sets fields on the lease +// If neither RefName nor Hash are set, ForceWithLease protects +// all refs in the refspec by ensuring the ref of the remote in the local repsitory +// matches the one in the ref advertisement. +type ForceWithLease struct { + // RefName, when set will protect the ref by ensuring it matches the + // hash in the ref advertisement. + RefName plumbing.ReferenceName + // Hash is the expected object id of RefName. The push will be rejected unless this + // matches the corresponding object id of RefName in the refs advertisement. + Hash plumbing.Hash } // Validate validates the fields and sets the default values. @@ -274,6 +308,8 @@ type CheckoutOptions struct { // target branch. Force and Keep are mutually exclusive, should not be both // set to true. Keep bool + // SparseCheckoutDirectories + SparseCheckoutDirectories []string } // Validate validates the fields and sets the default values. @@ -366,7 +402,7 @@ type LogOptions struct { // Show only those commits in which the specified file was inserted/updated. // It is equivalent to running `git log -- `. - // this field is kept for compatility, it can be replaced with PathFilter + // this field is kept for compatibility, it can be replaced with PathFilter FileName *string // Filter commits based on the path of files that are updated @@ -571,7 +607,7 @@ func (o *CreateTagOptions) loadConfigTagger(r *Repository) error { type ListOptions struct { // Auth credentials, if required, to use with the remote repository. Auth transport.AuthMethod - // InsecureSkipTLS skips ssl verify if protocal is https + // InsecureSkipTLS skips ssl verify if protocol is https InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte diff --git a/plumbing/format/commitgraph/encoder.go b/plumbing/format/commitgraph/encoder.go index d34076fc3..bcf7d030a 100644 --- a/plumbing/format/commitgraph/encoder.go +++ b/plumbing/format/commitgraph/encoder.go @@ -1,11 +1,11 @@ package commitgraph import ( - "crypto/sha1" - "hash" + "crypto" "io" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -17,7 +17,7 @@ type Encoder struct { // NewEncoder returns a new stream encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := sha1.New() + h := hash.New(crypto.SHA1) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } diff --git a/plumbing/format/commitgraph/file.go b/plumbing/format/commitgraph/file.go index 0ce719823..1d25238f5 100644 --- a/plumbing/format/commitgraph/file.go +++ b/plumbing/format/commitgraph/file.go @@ -14,14 +14,14 @@ import ( var ( // ErrUnsupportedVersion is returned by OpenFileIndex when the commit graph // file version is not supported. - ErrUnsupportedVersion = errors.New("Unsupported version") + ErrUnsupportedVersion = errors.New("unsupported version") // ErrUnsupportedHash is returned by OpenFileIndex when the commit graph // hash function is not supported. Currently only SHA-1 is defined and // supported - ErrUnsupportedHash = errors.New("Unsupported hash algorithm") + ErrUnsupportedHash = errors.New("unsupported hash algorithm") // ErrMalformedCommitGraphFile is returned by OpenFileIndex when the commit // graph file is corrupted. - ErrMalformedCommitGraphFile = errors.New("Malformed commit graph file") + ErrMalformedCommitGraphFile = errors.New("malformed commit graph file") commitFileSignature = []byte{'C', 'G', 'P', 'H'} oidFanoutSignature = []byte{'O', 'I', 'D', 'F'} diff --git a/plumbing/format/config/encoder.go b/plumbing/format/config/encoder.go index 4eac8968a..de069aed5 100644 --- a/plumbing/format/config/encoder.go +++ b/plumbing/format/config/encoder.go @@ -11,6 +11,10 @@ type Encoder struct { w io.Writer } +var ( + subsectionReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) + valueReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`, "\n", `\n`, "\t", `\t`, "\b", `\b`) +) // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { return &Encoder{w} @@ -48,8 +52,7 @@ func (e *Encoder) encodeSection(s *Section) error { } func (e *Encoder) encodeSubsection(sectionName string, s *Subsection) error { - //TODO: escape - if err := e.printf("[%s \"%s\"]\n", sectionName, s.Name); err != nil { + if err := e.printf("[%s \"%s\"]\n", sectionName, subsectionReplacer.Replace(s.Name)); err != nil { return err } @@ -58,12 +61,14 @@ func (e *Encoder) encodeSubsection(sectionName string, s *Subsection) error { func (e *Encoder) encodeOptions(opts Options) error { for _, o := range opts { - pattern := "\t%s = %s\n" - if strings.Contains(o.Value, "\\") { - pattern = "\t%s = %q\n" + var value string + if strings.ContainsAny(o.Value, "#;\"\t\n\\") || strings.HasPrefix(o.Value, " ") || strings.HasSuffix(o.Value, " ") { + value = `"`+valueReplacer.Replace(o.Value)+`"` + } else { + value = o.Value } - if err := e.printf(pattern, o.Key, o.Value); err != nil { + if err := e.printf("\t%s = %s\n", o.Key, value); err != nil { return err } } diff --git a/plumbing/format/config/fixtures_test.go b/plumbing/format/config/fixtures_test.go index f3533dfee..2fa7840b0 100644 --- a/plumbing/format/config/fixtures_test.go +++ b/plumbing/format/config/fixtures_test.go @@ -42,6 +42,41 @@ var fixtures = []*Fixture{ Text: "[core]\n\trepositoryformatversion = 0\n", Config: New().AddOption("core", "", "repositoryformatversion", "0"), }, + { + Raw: `[section] + option1 = "has # hash" + option2 = "has \" quote" + option3 = "has \\ backslash" + option4 = "has ; semicolon" + option5 = "has \n line-feed" + option6 = "has \t tab" + option7 = " has leading spaces" + option8 = "has trailing spaces " + option9 = has no special characters + option10 = has unusual ` + "\x01\x7f\xc8\x80 characters\n", + Text: `[section] + option1 = "has # hash" + option2 = "has \" quote" + option3 = "has \\ backslash" + option4 = "has ; semicolon" + option5 = "has \n line-feed" + option6 = "has \t tab" + option7 = " has leading spaces" + option8 = "has trailing spaces " + option9 = has no special characters + option10 = has unusual ` + "\x01\x7f\xc8\x80 characters\n", + Config: New(). + AddOption("section", "", "option1", `has # hash`). + AddOption("section", "", "option2", `has " quote`). + AddOption("section", "", "option3", `has \ backslash`). + AddOption("section", "", "option4", `has ; semicolon`). + AddOption("section", "", "option5", "has \n line-feed"). + AddOption("section", "", "option6", "has \t tab"). + AddOption("section", "", "option7", ` has leading spaces`). + AddOption("section", "", "option8", `has trailing spaces `). + AddOption("section", "", "option9", `has no special characters`). + AddOption("section", "", "option10", "has unusual \x01\x7f\u0200 characters"), + }, { Raw: ` [sect1] diff --git a/plumbing/format/config/section.go b/plumbing/format/config/section.go index 07f72f35a..4625ac583 100644 --- a/plumbing/format/config/section.go +++ b/plumbing/format/config/section.go @@ -103,7 +103,7 @@ func (s *Section) RemoveSubsection(name string) *Section { return s } -// Option return the value for the specified key. Empty string is returned if +// Option returns the value for the specified key. Empty string is returned if // key does not exists. func (s *Section) Option(key string) string { return s.Options.Get(key) diff --git a/plumbing/format/diff/patch.go b/plumbing/format/diff/patch.go index 39a66a1a8..c7678b01a 100644 --- a/plumbing/format/diff/patch.go +++ b/plumbing/format/diff/patch.go @@ -9,7 +9,7 @@ import ( type Operation int const ( - // Equal item represents a equals diff. + // Equal item represents an equals diff. Equal Operation = iota // Add item represents an insert diff. Add @@ -26,15 +26,15 @@ type Patch interface { Message() string } -// FilePatch represents the necessary steps to transform one file to another. +// FilePatch represents the necessary steps to transform one file into another. type FilePatch interface { // IsBinary returns true if this patch is representing a binary file. IsBinary() bool - // Files returns the from and to Files, with all the necessary metadata to + // Files returns the from and to Files, with all the necessary metadata // about them. If the patch creates a new file, "from" will be nil. // If the patch deletes a file, "to" will be nil. Files() (from, to File) - // Chunks returns a slice of ordered changes to transform "from" File to + // Chunks returns a slice of ordered changes to transform "from" File into // "to" File. If the file is a binary one, Chunks will be empty. Chunks() []Chunk } @@ -49,7 +49,7 @@ type File interface { Path() string } -// Chunk represents a portion of a file transformation to another. +// Chunk represents a portion of a file transformation into another. type Chunk interface { // Content contains the portion of the file. Content() string diff --git a/plumbing/format/gitattributes/attributes.go b/plumbing/format/gitattributes/attributes.go index d13c2a903..329e66731 100644 --- a/plumbing/format/gitattributes/attributes.go +++ b/plumbing/format/gitattributes/attributes.go @@ -15,7 +15,7 @@ const ( var ( ErrMacroNotAllowed = errors.New("macro not allowed") - ErrInvalidAttributeName = errors.New("Invalid attribute name") + ErrInvalidAttributeName = errors.New("invalid attribute name") ) type MatchAttribute struct { diff --git a/plumbing/format/gitattributes/pattern.go b/plumbing/format/gitattributes/pattern.go index d961aba9c..f101f4725 100644 --- a/plumbing/format/gitattributes/pattern.go +++ b/plumbing/format/gitattributes/pattern.go @@ -52,6 +52,11 @@ func (p *pattern) Match(path []string) bool { var match, doublestar bool var err error for _, part := range path { + // path is deeper than pattern + if len(pattern) == 0 { + return false + } + // skip empty if pattern[0] == "" { pattern = pattern[1:] diff --git a/plumbing/format/gitattributes/pattern_test.go b/plumbing/format/gitattributes/pattern_test.go index f95be6e7e..981d56f56 100644 --- a/plumbing/format/gitattributes/pattern_test.go +++ b/plumbing/format/gitattributes/pattern_test.go @@ -174,6 +174,12 @@ func (s *PatternSuite) TestGlobMatch_tailingAsterisks_single(c *C) { c.Assert(r, Equals, true) } +func (s *PatternSuite) TestGlobMatch_tailingAsterisk_single(c *C) { + p := ParsePattern("/*lue/*", nil) + r := p.Match([]string{"value", "volcano", "tail"}) + c.Assert(r, Equals, false) +} + func (s *PatternSuite) TestGlobMatch_tailingAsterisks_exactMatch(c *C) { p := ParsePattern("/*lue/vol?ano/**", nil) r := p.Match([]string{"value", "volcano"}) diff --git a/plumbing/format/gitignore/dir.go b/plumbing/format/gitignore/dir.go index 7cea50cd8..15bc9c779 100644 --- a/plumbing/format/gitignore/dir.go +++ b/plumbing/format/gitignore/dir.go @@ -13,13 +13,14 @@ import ( ) const ( - commentPrefix = "#" - coreSection = "core" - excludesfile = "excludesfile" - gitDir = ".git" - gitignoreFile = ".gitignore" - gitconfigFile = ".gitconfig" - systemFile = "/etc/gitconfig" + commentPrefix = "#" + coreSection = "core" + excludesfile = "excludesfile" + gitDir = ".git" + gitignoreFile = ".gitignore" + gitconfigFile = ".gitconfig" + systemFile = "/etc/gitconfig" + infoExcludeFile = gitDir + "/info/exclude" ) // readIgnoreFile reads a specific git ignore file. @@ -42,10 +43,14 @@ func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps [ return } -// ReadPatterns reads gitignore patterns recursively traversing through the directory -// structure. The result is in the ascending order of priority (last higher). +// ReadPatterns reads the .git/info/exclude and then the gitignore patterns +// recursively traversing through the directory structure. The result is in +// the ascending order of priority (last higher). func ReadPatterns(fs billy.Filesystem, path []string) (ps []Pattern, err error) { - ps, _ = readIgnoreFile(fs, path, gitignoreFile) + ps, _ = readIgnoreFile(fs, path, infoExcludeFile) + + subps, _ := readIgnoreFile(fs, path, gitignoreFile) + ps = append(ps, subps...) var fis []os.FileInfo fis, err = fs.ReadDir(fs.Join(path...)) diff --git a/plumbing/format/gitignore/dir_test.go b/plumbing/format/gitignore/dir_test.go index 94ed7befc..facc36d8e 100644 --- a/plumbing/format/gitignore/dir_test.go +++ b/plumbing/format/gitignore/dir_test.go @@ -24,7 +24,17 @@ var _ = Suite(&MatcherSuite{}) func (s *MatcherSuite) SetUpTest(c *C) { // setup generic git repository root fs := memfs.New() - f, err := fs.Create(".gitignore") + + err := fs.MkdirAll(".git/info", os.ModePerm) + c.Assert(err, IsNil) + f, err := fs.Create(".git/info/exclude") + c.Assert(err, IsNil) + _, err = f.Write([]byte("exclude.crlf\r\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + + f, err = fs.Create(".gitignore") c.Assert(err, IsNil) _, err = f.Write([]byte("vendor/g*/\n")) c.Assert(err, IsNil) @@ -44,6 +54,8 @@ func (s *MatcherSuite) SetUpTest(c *C) { err = fs.MkdirAll("another", os.ModePerm) c.Assert(err, IsNil) + err = fs.MkdirAll("exclude.crlf", os.ModePerm) + c.Assert(err, IsNil) err = fs.MkdirAll("ignore.crlf", os.ModePerm) c.Assert(err, IsNil) err = fs.MkdirAll("vendor/github.com", os.ModePerm) @@ -173,9 +185,10 @@ func (s *MatcherSuite) SetUpTest(c *C) { func (s *MatcherSuite) TestDir_ReadPatterns(c *C) { ps, err := ReadPatterns(s.GFS, nil) c.Assert(err, IsNil) - c.Assert(ps, HasLen, 3) + c.Assert(ps, HasLen, 4) m := NewMatcher(ps) + c.Assert(m.Match([]string{"exclude.crlf"}, true), Equals, true) c.Assert(m.Match([]string{"ignore.crlf"}, true), Equals, true) c.Assert(m.Match([]string{"vendor", "gopkg.in"}, true), Equals, true) c.Assert(m.Match([]string{"vendor", "github.com"}, true), Equals, false) diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index 7768bd650..51a390489 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -12,9 +12,9 @@ import ( var ( // ErrUnsupportedVersion is returned by Decode when the idx file version // is not supported. - ErrUnsupportedVersion = errors.New("Unsupported version") + ErrUnsupportedVersion = errors.New("unsupported version") // ErrMalformedIdxFile is returned by Decode when the idx file is corrupted. - ErrMalformedIdxFile = errors.New("Malformed IDX file") + ErrMalformedIdxFile = errors.New("malformed IDX file") ) const ( diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go index 26b2e4d6b..6ac445ff6 100644 --- a/plumbing/format/idxfile/encoder.go +++ b/plumbing/format/idxfile/encoder.go @@ -1,10 +1,10 @@ package idxfile import ( - "crypto/sha1" - "hash" + "crypto" "io" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -16,7 +16,7 @@ type Encoder struct { // NewEncoder returns a new stream encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := sha1.New() + h := hash.New(crypto.SHA1) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index 036b6365e..c4da20c08 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -3,15 +3,15 @@ package index import ( "bufio" "bytes" - "crypto/sha1" + "crypto" "errors" - "hash" "io" "io/ioutil" "strconv" "time" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -49,7 +49,7 @@ type Decoder struct { // NewDecoder returns a new decoder that reads from r. func NewDecoder(r io.Reader) *Decoder { - h := sha1.New() + h := hash.New(crypto.SHA1) return &Decoder{ r: io.TeeReader(r, h), hash: h, diff --git a/plumbing/format/index/encoder.go b/plumbing/format/index/encoder.go index 00d4e7a31..a91537870 100644 --- a/plumbing/format/index/encoder.go +++ b/plumbing/format/index/encoder.go @@ -2,19 +2,19 @@ package index import ( "bytes" - "crypto/sha1" + "crypto" "errors" - "hash" "io" "sort" "time" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) var ( // EncodeVersionSupported is the range of supported index versions - EncodeVersionSupported uint32 = 2 + EncodeVersionSupported uint32 = 3 // ErrInvalidTimestamp is returned by Encode if a Index with a Entry with // negative timestamp values @@ -29,16 +29,16 @@ type Encoder struct { // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := sha1.New() + h := hash.New(crypto.SHA1) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } // Encode writes the Index to the stream of the encoder. func (e *Encoder) Encode(idx *Index) error { - // TODO: support versions v3 and v4 + // TODO: support v4 // TODO: support extensions - if idx.Version != EncodeVersionSupported { + if idx.Version > EncodeVersionSupported { return ErrUnsupportedVersion } @@ -68,8 +68,12 @@ func (e *Encoder) encodeEntries(idx *Index) error { if err := e.encodeEntry(entry); err != nil { return err } + entryLength := entryHeaderLength + if entry.IntentToAdd || entry.SkipWorktree { + entryLength += 2 + } - wrote := entryHeaderLength + len(entry.Name) + wrote := entryLength + len(entry.Name) if err := e.padEntry(wrote); err != nil { return err } @@ -79,10 +83,6 @@ func (e *Encoder) encodeEntries(idx *Index) error { } func (e *Encoder) encodeEntry(entry *Entry) error { - if entry.IntentToAdd || entry.SkipWorktree { - return ErrUnsupportedVersion - } - sec, nsec, err := e.timeToUint32(&entry.CreatedAt) if err != nil { return err @@ -110,9 +110,25 @@ func (e *Encoder) encodeEntry(entry *Entry) error { entry.GID, entry.Size, entry.Hash[:], - flags, } + flagsFlow := []interface{}{flags} + + if entry.IntentToAdd || entry.SkipWorktree { + var extendedFlags uint16 + + if entry.IntentToAdd { + extendedFlags |= intentToAddMask + } + if entry.SkipWorktree { + extendedFlags |= skipWorkTreeMask + } + + flagsFlow = []interface{}{flags | entryExtended, extendedFlags} + } + + flow = append(flow, flagsFlow...) + if err := binary.Write(e.w, flow...); err != nil { return err } diff --git a/plumbing/format/index/encoder_test.go b/plumbing/format/index/encoder_test.go index b7a73cb14..25c24f14f 100644 --- a/plumbing/format/index/encoder_test.go +++ b/plumbing/format/index/encoder_test.go @@ -57,7 +57,7 @@ func (s *IndexSuite) TestEncode(c *C) { } func (s *IndexSuite) TestEncodeUnsupportedVersion(c *C) { - idx := &Index{Version: 3} + idx := &Index{Version: 4} buf := bytes.NewBuffer(nil) e := NewEncoder(buf) @@ -67,24 +67,40 @@ func (s *IndexSuite) TestEncodeUnsupportedVersion(c *C) { func (s *IndexSuite) TestEncodeWithIntentToAddUnsupportedVersion(c *C) { idx := &Index{ - Version: 2, + Version: 3, Entries: []*Entry{{IntentToAdd: true}}, } buf := bytes.NewBuffer(nil) e := NewEncoder(buf) err := e.Encode(idx) - c.Assert(err, Equals, ErrUnsupportedVersion) + c.Assert(err, IsNil) + + output := &Index{} + d := NewDecoder(buf) + err = d.Decode(output) + c.Assert(err, IsNil) + + c.Assert(cmp.Equal(idx, output), Equals, true) + c.Assert(output.Entries[0].IntentToAdd, Equals, true) } func (s *IndexSuite) TestEncodeWithSkipWorktreeUnsupportedVersion(c *C) { idx := &Index{ - Version: 2, + Version: 3, Entries: []*Entry{{SkipWorktree: true}}, } buf := bytes.NewBuffer(nil) e := NewEncoder(buf) err := e.Encode(idx) - c.Assert(err, Equals, ErrUnsupportedVersion) + c.Assert(err, IsNil) + + output := &Index{} + d := NewDecoder(buf) + err = d.Decode(output) + c.Assert(err, IsNil) + + c.Assert(cmp.Equal(idx, output), Equals, true) + c.Assert(output.Entries[0].SkipWorktree, Equals, true) } diff --git a/plumbing/format/index/index.go b/plumbing/format/index/index.go index 649416a2b..f4c7647d3 100644 --- a/plumbing/format/index/index.go +++ b/plumbing/format/index/index.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "time" "github.com/go-git/go-git/v5/plumbing" @@ -211,3 +212,20 @@ type EndOfIndexEntry struct { // their contents). Hash plumbing.Hash } + +// SkipUnless applies patterns in the form of A, A/B, A/B/C +// to the index to prevent the files from being checked out +func (i *Index) SkipUnless(patterns []string) { + for _, e := range i.Entries { + var include bool + for _, pattern := range patterns { + if strings.HasPrefix(e.Name, pattern) { + include = true + break + } + } + if !include { + e.SkipWorktree = true + } + } +} diff --git a/plumbing/format/objfile/reader.go b/plumbing/format/objfile/reader.go index b6b2ca06d..d7932f4ea 100644 --- a/plumbing/format/objfile/reader.go +++ b/plumbing/format/objfile/reader.go @@ -1,13 +1,13 @@ package objfile import ( - "compress/zlib" "errors" "io" "strconv" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -20,20 +20,22 @@ var ( // Reader implements io.ReadCloser. Close should be called when finished with // the Reader. Close will not close the underlying io.Reader. type Reader struct { - multi io.Reader - zlib io.ReadCloser - hasher plumbing.Hasher + multi io.Reader + zlib io.Reader + zlibref sync.ZLibReader + hasher plumbing.Hasher } // NewReader returns a new Reader reading from r. func NewReader(r io.Reader) (*Reader, error) { - zlib, err := zlib.NewReader(r) + zlib, err := sync.GetZlibReader(r) if err != nil { return nil, packfile.ErrZLib.AddDetails(err.Error()) } return &Reader{ - zlib: zlib, + zlib: zlib.Reader, + zlibref: zlib, }, nil } @@ -110,5 +112,6 @@ func (r *Reader) Hash() plumbing.Hash { // Close releases any resources consumed by the Reader. Calling Close does not // close the wrapped io.Reader originally passed to NewReader. func (r *Reader) Close() error { - return r.zlib.Close() + sync.PutZlibReader(r.zlibref) + return nil } diff --git a/plumbing/format/objfile/writer.go b/plumbing/format/objfile/writer.go index 2a96a4370..0d0f15492 100644 --- a/plumbing/format/objfile/writer.go +++ b/plumbing/format/objfile/writer.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -18,9 +19,9 @@ var ( // not close the underlying io.Writer. type Writer struct { raw io.Writer - zlib io.WriteCloser hasher plumbing.Hasher multi io.Writer + zlib *zlib.Writer closed bool pending int64 // number of unwritten bytes @@ -31,9 +32,10 @@ type Writer struct { // The returned Writer implements io.WriteCloser. Close should be called when // finished with the Writer. Close will not close the underlying io.Writer. func NewWriter(w io.Writer) *Writer { + zlib := sync.GetZlibWriter(w) return &Writer{ raw: w, - zlib: zlib.NewWriter(w), + zlib: zlib, } } @@ -100,6 +102,7 @@ func (w *Writer) Hash() plumbing.Hash { // Calling Close does not close the wrapped io.Writer originally passed to // NewWriter. func (w *Writer) Close() error { + defer sync.PutZlibWriter(w.zlib) if err := w.zlib.Close(); err != nil { return err } diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index df423ad50..36c5ef5b8 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -1,10 +1,7 @@ package packfile import ( - "bytes" - "compress/zlib" "io" - "sync" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" @@ -61,18 +58,3 @@ func WritePackfileToObjectStorage( return err } - -var bufPool = sync.Pool{ - New: func() interface{} { - return bytes.NewBuffer(nil) - }, -} - -var zlibInitBytes = []byte{0x78, 0x9c, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01} - -var zlibReaderPool = sync.Pool{ - New: func() interface{} { - r, _ := zlib.NewReader(bytes.NewReader(zlibInitBytes)) - return r - }, -} diff --git a/plumbing/format/packfile/delta_test.go b/plumbing/format/packfile/delta_test.go index 98f53f6d6..137e4859b 100644 --- a/plumbing/format/packfile/delta_test.go +++ b/plumbing/format/packfile/delta_test.go @@ -1,8 +1,11 @@ package packfile import ( + "bytes" + "io/ioutil" "math/rand" + "github.com/go-git/go-git/v5/plumbing" . "gopkg.in/check.v1" ) @@ -97,6 +100,32 @@ func (s *DeltaSuite) TestAddDelta(c *C) { } } +func (s *DeltaSuite) TestAddDeltaReader(c *C) { + for _, t := range s.testCases { + baseBuf := genBytes(t.base) + baseObj := &plumbing.MemoryObject{} + baseObj.Write(baseBuf) + + targetBuf := genBytes(t.target) + + delta := DiffDelta(baseBuf, targetBuf) + deltaRC := ioutil.NopCloser(bytes.NewReader(delta)) + + c.Log("Executing test case:", t.description) + + resultRC, err := ReaderFromDelta(baseObj, deltaRC) + c.Assert(err, IsNil) + + result, err := ioutil.ReadAll(resultRC) + c.Assert(err, IsNil) + + err = resultRC.Close() + c.Assert(err, IsNil) + + c.Assert(result, DeepEquals, targetBuf) + } +} + func (s *DeltaSuite) TestIncompleteDelta(c *C) { for _, t := range s.testCases { c.Log("Incomplete delta on:", t.description) @@ -125,3 +154,25 @@ func (s *DeltaSuite) TestMaxCopySizeDelta(c *C) { c.Assert(err, IsNil) c.Assert(result, DeepEquals, targetBuf) } + +func (s *DeltaSuite) TestMaxCopySizeDeltaReader(c *C) { + baseBuf := randBytes(maxCopySize) + baseObj := &plumbing.MemoryObject{} + baseObj.Write(baseBuf) + + targetBuf := baseBuf[0:] + targetBuf = append(targetBuf, byte(1)) + + delta := DiffDelta(baseBuf, targetBuf) + deltaRC := ioutil.NopCloser(bytes.NewReader(delta)) + + resultRC, err := ReaderFromDelta(baseObj, deltaRC) + c.Assert(err, IsNil) + + result, err := ioutil.ReadAll(resultRC) + c.Assert(err, IsNil) + + err = resultRC.Close() + c.Assert(err, IsNil) + c.Assert(result, DeepEquals, targetBuf) +} diff --git a/plumbing/format/packfile/diff_delta.go b/plumbing/format/packfile/diff_delta.go index 1951b34ef..2c7a33581 100644 --- a/plumbing/format/packfile/diff_delta.go +++ b/plumbing/format/packfile/diff_delta.go @@ -5,6 +5,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) // See https://github.com/jelmer/dulwich/blob/master/dulwich/pack.py and @@ -43,18 +44,16 @@ func getDelta(index *deltaIndex, base, target plumbing.EncodedObject) (o plumbin defer ioutil.CheckClose(tr, &err) - bb := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(bb) - bb.Reset() + bb := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(bb) _, err = bb.ReadFrom(br) if err != nil { return nil, err } - tb := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(tb) - tb.Reset() + tb := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(tb) _, err = tb.ReadFrom(tr) if err != nil { @@ -80,9 +79,8 @@ func DiffDelta(src, tgt []byte) []byte { } func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) buf.Write(deltaEncodeSize(len(src))) buf.Write(deltaEncodeSize(len(tgt))) @@ -90,9 +88,8 @@ func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { index.init(src) } - ibuf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(ibuf) - ibuf.Reset() + ibuf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(ibuf) for i := 0; i < len(tgt); i++ { offset, l := index.findMatch(src, tgt, i) diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index 5501f8861..a8a7e967b 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -2,11 +2,12 @@ package packfile import ( "compress/zlib" - "crypto/sha1" + "crypto" "fmt" "io" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/binary" "github.com/go-git/go-git/v5/utils/ioutil" @@ -28,7 +29,7 @@ type Encoder struct { // OFSDeltaObject. To use Reference deltas, set useRefDeltas to true. func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *Encoder { h := plumbing.Hasher{ - Hash: sha1.New(), + Hash: hash.New(crypto.SHA1), } mw := io.MultiWriter(w, h) ow := newOffsetWriter(mw) diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index 95db5c082..15c0fba40 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -105,7 +105,7 @@ func (s *EncoderAdvancedSuite) testEncodeDecode( _, err = f.Seek(0, io.SeekStart) c.Assert(err, IsNil) - p := NewPackfile(index, fs, f) + p := NewPackfile(index, fs, f, 0) decodeHash, err := p.ID() c.Assert(err, IsNil) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index d2db892a6..c9d49c3b5 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -318,7 +318,7 @@ func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) { index, err := w.Index() c.Assert(err, IsNil) - return NewPackfile(index, fs, file), func() { + return NewPackfile(index, fs, file, 0), func() { c.Assert(file.Close(), IsNil) } } diff --git a/plumbing/format/packfile/fsobject.go b/plumbing/format/packfile/fsobject.go index c5edaf52e..238339daf 100644 --- a/plumbing/format/packfile/fsobject.go +++ b/plumbing/format/packfile/fsobject.go @@ -7,19 +7,20 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/format/idxfile" + "github.com/go-git/go-git/v5/utils/ioutil" ) // FSObject is an object from the packfile on the filesystem. type FSObject struct { - hash plumbing.Hash - h *ObjectHeader - offset int64 - size int64 - typ plumbing.ObjectType - index idxfile.Index - fs billy.Filesystem - path string - cache cache.Object + hash plumbing.Hash + offset int64 + size int64 + typ plumbing.ObjectType + index idxfile.Index + fs billy.Filesystem + path string + cache cache.Object + largeObjectThreshold int64 } // NewFSObject creates a new filesystem object. @@ -32,16 +33,18 @@ func NewFSObject( fs billy.Filesystem, path string, cache cache.Object, + largeObjectThreshold int64, ) *FSObject { return &FSObject{ - hash: hash, - offset: offset, - size: contentSize, - typ: finalType, - index: index, - fs: fs, - path: path, - cache: cache, + hash: hash, + offset: offset, + size: contentSize, + typ: finalType, + index: index, + fs: fs, + path: path, + cache: cache, + largeObjectThreshold: largeObjectThreshold, } } @@ -62,7 +65,21 @@ func (o *FSObject) Reader() (io.ReadCloser, error) { return nil, err } - p := NewPackfileWithCache(o.index, nil, f, o.cache) + p := NewPackfileWithCache(o.index, nil, f, o.cache, o.largeObjectThreshold) + if o.largeObjectThreshold > 0 && o.size > o.largeObjectThreshold { + // We have a big object + h, err := p.objectHeaderAtOffset(o.offset) + if err != nil { + return nil, err + } + + r, err := p.getReaderDirect(h) + if err != nil { + _ = f.Close() + return nil, err + } + return ioutil.NewReadCloserWithCloser(r, f.Close), nil + } r, err := p.getObjectContent(o.offset) if err != nil { _ = f.Close() @@ -100,17 +117,3 @@ func (o *FSObject) Type() plumbing.ObjectType { func (o *FSObject) Writer() (io.WriteCloser, error) { return nil, nil } - -type objectReader struct { - io.ReadCloser - f billy.File -} - -func (r *objectReader) Close() error { - if err := r.ReadCloser.Close(); err != nil { - _ = r.f.Close() - return err - } - - return r.f.Close() -} diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index ddd7f62fc..685270225 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -2,6 +2,7 @@ package packfile import ( "bytes" + "fmt" "io" "os" @@ -11,6 +12,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/idxfile" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -35,11 +37,12 @@ const smallObjectThreshold = 16 * 1024 // Packfile allows retrieving information from inside a packfile. type Packfile struct { idxfile.Index - fs billy.Filesystem - file billy.File - s *Scanner - deltaBaseCache cache.Object - offsetToType map[int64]plumbing.ObjectType + fs billy.Filesystem + file billy.File + s *Scanner + deltaBaseCache cache.Object + offsetToType map[int64]plumbing.ObjectType + largeObjectThreshold int64 } // NewPackfileWithCache creates a new Packfile with the given object cache. @@ -50,6 +53,7 @@ func NewPackfileWithCache( fs billy.Filesystem, file billy.File, cache cache.Object, + largeObjectThreshold int64, ) *Packfile { s := NewScanner(file) return &Packfile{ @@ -59,6 +63,7 @@ func NewPackfileWithCache( s, cache, make(map[int64]plumbing.ObjectType), + largeObjectThreshold, } } @@ -66,8 +71,8 @@ func NewPackfileWithCache( // and packfile idx. // If the filesystem is provided, the packfile will return FSObjects, otherwise // it will return MemoryObjects. -func NewPackfile(index idxfile.Index, fs billy.Filesystem, file billy.File) *Packfile { - return NewPackfileWithCache(index, fs, file, cache.NewObjectLRUDefault()) +func NewPackfile(index idxfile.Index, fs billy.Filesystem, file billy.File, largeObjectThreshold int64) *Packfile { + return NewPackfileWithCache(index, fs, file, cache.NewObjectLRUDefault(), largeObjectThreshold) } // Get retrieves the encoded object in the packfile with the given hash. @@ -133,9 +138,8 @@ func (p *Packfile) getObjectSize(h *ObjectHeader) (int64, error) { case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: return h.Length, nil case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) if _, _, err := p.s.NextObject(buf); err != nil { return 0, err @@ -222,9 +226,9 @@ func (p *Packfile) getNextObject(h *ObjectHeader, hash plumbing.Hash) (plumbing. // For delta objects we read the delta data and apply the small object // optimization only if the expanded version of the object still meets // the small object threshold condition. - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + if _, _, err := p.s.NextObject(buf); err != nil { return nil, err } @@ -263,6 +267,7 @@ func (p *Packfile) getNextObject(h *ObjectHeader, hash plumbing.Hash) (plumbing. p.fs, p.file.Name(), p.deltaBaseCache, + p.largeObjectThreshold, ), nil } @@ -282,6 +287,49 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { return obj.Reader() } +func asyncReader(p *Packfile) (io.ReadCloser, error) { + reader := ioutil.NewReaderUsingReaderAt(p.file, p.s.r.offset) + zr, err := sync.GetZlibReader(reader) + if err != nil { + return nil, fmt.Errorf("zlib reset error: %s", err) + } + + return ioutil.NewReadCloserWithCloser(zr.Reader, func() error { + sync.PutZlibReader(zr) + return nil + }), nil + +} + +func (p *Packfile) getReaderDirect(h *ObjectHeader) (io.ReadCloser, error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + return asyncReader(p) + case plumbing.REFDeltaObject: + deltaRc, err := asyncReader(p) + if err != nil { + return nil, err + } + r, err := p.readREFDeltaObjectContent(h, deltaRc) + if err != nil { + return nil, err + } + return r, nil + case plumbing.OFSDeltaObject: + deltaRc, err := asyncReader(p) + if err != nil { + return nil, err + } + r, err := p.readOFSDeltaObjectContent(h, deltaRc) + if err != nil { + return nil, err + } + return r, nil + default: + return nil, ErrInvalidObject.AddDetails("type %q", h.Type) + } +} + func (p *Packfile) getNextMemoryObject(h *ObjectHeader) (plumbing.EncodedObject, error) { var obj = new(plumbing.MemoryObject) obj.SetSize(h.Length) @@ -323,9 +371,9 @@ func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) (err err } func (p *Packfile) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) error { - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + _, _, err := p.s.NextObject(buf) if err != nil { return err @@ -334,6 +382,20 @@ func (p *Packfile) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plu return p.fillREFDeltaObjectContentWithBuffer(obj, ref, buf) } +func (p *Packfile) readREFDeltaObjectContent(h *ObjectHeader, deltaRC io.Reader) (io.ReadCloser, error) { + var err error + + base, ok := p.cacheGet(h.Reference) + if !ok { + base, err = p.Get(h.Reference) + if err != nil { + return nil, err + } + } + + return ReaderFromDelta(base, deltaRC) +} + func (p *Packfile) fillREFDeltaObjectContentWithBuffer(obj plumbing.EncodedObject, ref plumbing.Hash, buf *bytes.Buffer) error { var err error @@ -353,9 +415,9 @@ func (p *Packfile) fillREFDeltaObjectContentWithBuffer(obj plumbing.EncodedObjec } func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset int64) error { - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + _, _, err := p.s.NextObject(buf) if err != nil { return err @@ -364,6 +426,20 @@ func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset return p.fillOFSDeltaObjectContentWithBuffer(obj, offset, buf) } +func (p *Packfile) readOFSDeltaObjectContent(h *ObjectHeader, deltaRC io.Reader) (io.ReadCloser, error) { + hash, err := p.FindHash(h.OffsetReference) + if err != nil { + return nil, err + } + + base, err := p.objectAtOffset(h.OffsetReference, hash) + if err != nil { + return nil, err + } + + return ReaderFromDelta(base, deltaRC) +} + func (p *Packfile) fillOFSDeltaObjectContentWithBuffer(obj plumbing.EncodedObject, offset int64, buf *bytes.Buffer) error { hash, err := p.FindHash(offset) if err != nil { diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go index 60c7c7341..2eb099df6 100644 --- a/plumbing/format/packfile/packfile_test.go +++ b/plumbing/format/packfile/packfile_test.go @@ -8,7 +8,6 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/idxfile" "github.com/go-git/go-git/v5/plumbing/format/packfile" - "github.com/go-git/go-git/v5/plumbing/storer" . "gopkg.in/check.v1" ) @@ -111,7 +110,7 @@ func (s *PackfileSuite) SetUpTest(c *C) { s.idx = idxfile.NewMemoryIndex() c.Assert(idxfile.NewDecoder(s.f.Idx()).Decode(s.idx), IsNil) - s.p = packfile.NewPackfile(s.idx, fixtures.Filesystem, s.f.Packfile()) + s.p = packfile.NewPackfile(s.idx, fixtures.Filesystem, s.f.Packfile(), 0) } func (s *PackfileSuite) TearDownTest(c *C) { @@ -122,7 +121,7 @@ func (s *PackfileSuite) TestDecode(c *C) { fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { index := getIndexFromIdxFile(f.Idx()) - p := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile()) + p := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile(), 0) defer p.Close() for _, h := range expectedHashes { @@ -138,7 +137,7 @@ func (s *PackfileSuite) TestDecodeByTypeRefDelta(c *C) { index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile()) + packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile(), 0) defer packfile.Close() iter, err := packfile.GetByType(plumbing.CommitObject) @@ -171,7 +170,7 @@ func (s *PackfileSuite) TestDecodeByType(c *C) { for _, t := range ts { index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile()) + packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile(), 0) defer packfile.Close() iter, err := packfile.GetByType(t) @@ -189,7 +188,7 @@ func (s *PackfileSuite) TestDecodeByTypeConstructor(c *C) { f := fixtures.Basic().ByTag("packfile").One() index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile()) + packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile(), 0) defer packfile.Close() _, err := packfile.GetByType(plumbing.OFSDeltaObject) @@ -236,22 +235,6 @@ var expectedHashes = []string{ "7e59600739c96546163833214c36459e324bad0a", } -func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { - i, err := s.IterEncodedObjects(plumbing.AnyObject) - c.Assert(err, IsNil) - - var count int - err = i.ForEach(func(plumbing.EncodedObject) error { count++; return nil }) - c.Assert(err, IsNil) - c.Assert(count, Equals, len(expects)) - - for _, exp := range expects { - obt, err := s.EncodedObject(plumbing.AnyObject, plumbing.NewHash(exp)) - c.Assert(err, IsNil) - c.Assert(obt.Hash().String(), Equals, exp) - } -} - func getIndexFromIdxFile(r io.Reader) idxfile.Index { idx := idxfile.NewMemoryIndex() if err := idxfile.NewDecoder(r).Decode(idx); err != nil { @@ -266,7 +249,7 @@ func (s *PackfileSuite) TestSize(c *C) { index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile()) + packfile := packfile.NewPackfile(index, fixtures.Filesystem, f.Packfile(), 0) defer packfile.Close() // Get the size of binary.jpg, which is not delta-encoded. diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 4b5a5708c..522c146f2 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -10,6 +10,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -46,7 +47,6 @@ type Parser struct { oi []*objectInfo oiByHash map[plumbing.Hash]*objectInfo oiByOffset map[int64]*objectInfo - hashOffset map[plumbing.Hash]int64 checksum plumbing.Hash cache *cache.BufferLRU @@ -176,7 +176,8 @@ func (p *Parser) init() error { } func (p *Parser) indexObjects() error { - buf := new(bytes.Buffer) + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) for i := uint32(0); i < p.count; i++ { buf.Reset() @@ -220,6 +221,7 @@ func (p *Parser) indexObjects() error { ota = newBaseObject(oh.Offset, oh.Length, t) } + buf.Grow(int(oh.Length)) _, crc, err := p.scanner.NextObject(buf) if err != nil { return err @@ -265,7 +267,9 @@ func (p *Parser) indexObjects() error { } func (p *Parser) resolveDeltas() error { - buf := &bytes.Buffer{} + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + for _, obj := range p.oi { buf.Reset() err := p.get(obj, buf) @@ -287,6 +291,7 @@ func (p *Parser) resolveDeltas() error { if err := p.resolveObject(stdioutil.Discard, child, content); err != nil { return err } + p.resolveExternalRef(child) } // Remove the delta from the cache. @@ -299,6 +304,16 @@ func (p *Parser) resolveDeltas() error { return nil } +func (p *Parser) resolveExternalRef(o *objectInfo) { + if ref, ok := p.oiByHash[o.SHA1]; ok && ref.ExternalRef { + p.oiByHash[o.SHA1] = o + o.Children = ref.Children + for _, c := range o.Children { + c.Parent = o + } + } +} + func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { if !o.ExternalRef { // skip cache check for placeholder parents b, ok := p.cache.Get(o.Offset) @@ -336,9 +351,8 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { } if o.DiskType.IsDelta() { - b := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(b) - b.Reset() + b := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(b) err := p.get(o.Parent, b) if err != nil { return err @@ -372,9 +386,8 @@ func (p *Parser) resolveObject( if !o.DiskType.IsDelta() { return nil } - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) err := p.readData(buf, o) if err != nil { return err diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index b0b4af82a..651d05f3d 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -10,8 +10,10 @@ import ( fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/format/packfile" "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/storage/filesystem" . "gopkg.in/check.v1" ) @@ -132,6 +134,19 @@ func (s *ParserSuite) TestThinPack(c *C) { } +func (s *ParserSuite) TestResolveExternalRefsInThinPack(c *C) { + extRefsThinPack := fixtures.ByTag("codecommit").One() + + scanner := packfile.NewScanner(extRefsThinPack.Packfile()) + + obs := new(testObserver) + parser, err := packfile.NewParser(scanner, obs) + c.Assert(err, IsNil) + + _, err = parser.Parse() + c.Assert(err, IsNil) +} + type observerObject struct { hash string otype plumbing.ObjectType @@ -235,3 +250,29 @@ func BenchmarkParseBasic(b *testing.B) { } } } + +func BenchmarkParser(b *testing.B) { + f := fixtures.Basic().One() + defer fixtures.Clean() + + b.ResetTimer() + for n := 0; n < b.N; n++ { + b.StopTimer() + scanner := packfile.NewScanner(f.Packfile()) + fs := osfs.New(os.TempDir()) + storage := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + + parser, err := packfile.NewParserWithStorage(scanner, storage) + if err != nil { + b.Error(err) + } + + b.StartTimer() + _, err = parser.Parse() + + b.StopTimer() + if err != nil { + b.Error(err) + } + } +} diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 9e90f30a7..f00562d63 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -1,12 +1,15 @@ package packfile import ( + "bufio" "bytes" "errors" "io" + "math" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) // See https://github.com/git/git/blob/49fa3dc76179e04b0833542fa52d0f287a4955ac/delta.h @@ -32,18 +35,16 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { defer ioutil.CheckClose(w, &err) - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) _, err = buf.ReadFrom(r) if err != nil { return err } src := buf.Bytes() - dst := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(dst) - dst.Reset() + dst := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(dst) err = patchDelta(dst, src, delta) if err != nil { return err @@ -51,9 +52,9 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { target.SetSize(int64(dst.Len())) - b := byteSlicePool.Get().([]byte) - _, err = io.CopyBuffer(w, dst, b) - byteSlicePool.Put(b) + b := sync.GetByteSlice() + _, err = io.CopyBuffer(w, dst, *b) + sync.PutByteSlice(b) return err } @@ -73,6 +74,131 @@ func PatchDelta(src, delta []byte) ([]byte, error) { return b.Bytes(), nil } +func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadCloser, error) { + deltaBuf := bufio.NewReaderSize(deltaRC, 1024) + srcSz, err := decodeLEB128ByteReader(deltaBuf) + if err != nil { + if err == io.EOF { + return nil, ErrInvalidDelta + } + return nil, err + } + if srcSz != uint(base.Size()) { + return nil, ErrInvalidDelta + } + + targetSz, err := decodeLEB128ByteReader(deltaBuf) + if err != nil { + if err == io.EOF { + return nil, ErrInvalidDelta + } + return nil, err + } + remainingTargetSz := targetSz + + dstRd, dstWr := io.Pipe() + + go func() { + baseRd, err := base.Reader() + if err != nil { + _ = dstWr.CloseWithError(ErrInvalidDelta) + return + } + defer baseRd.Close() + + baseBuf := bufio.NewReader(baseRd) + basePos := uint(0) + + for { + cmd, err := deltaBuf.ReadByte() + if err == io.EOF { + _ = dstWr.CloseWithError(ErrInvalidDelta) + return + } + if err != nil { + _ = dstWr.CloseWithError(err) + return + } + + if isCopyFromSrc(cmd) { + offset, err := decodeOffsetByteReader(cmd, deltaBuf) + if err != nil { + _ = dstWr.CloseWithError(err) + return + } + sz, err := decodeSizeByteReader(cmd, deltaBuf) + if err != nil { + _ = dstWr.CloseWithError(err) + return + } + + if invalidSize(sz, targetSz) || + invalidOffsetSize(offset, sz, srcSz) { + _ = dstWr.Close() + return + } + + discard := offset - basePos + if basePos > offset { + _ = baseRd.Close() + baseRd, err = base.Reader() + if err != nil { + _ = dstWr.CloseWithError(ErrInvalidDelta) + return + } + baseBuf.Reset(baseRd) + discard = offset + } + for discard > math.MaxInt32 { + n, err := baseBuf.Discard(math.MaxInt32) + if err != nil { + _ = dstWr.CloseWithError(err) + return + } + basePos += uint(n) + discard -= uint(n) + } + for discard > 0 { + n, err := baseBuf.Discard(int(discard)) + if err != nil { + _ = dstWr.CloseWithError(err) + return + } + basePos += uint(n) + discard -= uint(n) + } + if _, err := io.Copy(dstWr, io.LimitReader(baseBuf, int64(sz))); err != nil { + _ = dstWr.CloseWithError(err) + return + } + remainingTargetSz -= sz + basePos += sz + } else if isCopyFromDelta(cmd) { + sz := uint(cmd) // cmd is the size itself + if invalidSize(sz, targetSz) { + _ = dstWr.CloseWithError(ErrInvalidDelta) + return + } + if _, err := io.Copy(dstWr, io.LimitReader(deltaBuf, int64(sz))); err != nil { + _ = dstWr.CloseWithError(err) + return + } + + remainingTargetSz -= sz + } else { + _ = dstWr.CloseWithError(ErrDeltaCmd) + return + } + if remainingTargetSz <= 0 { + _ = dstWr.Close() + return + } + } + }() + + return dstRd, nil +} + func patchDelta(dst *bytes.Buffer, src, delta []byte) error { if len(delta) < deltaSizeMin { return ErrInvalidDelta @@ -161,6 +287,25 @@ func decodeLEB128(input []byte) (uint, []byte) { return num, input[sz:] } +func decodeLEB128ByteReader(input io.ByteReader) (uint, error) { + var num, sz uint + for { + b, err := input.ReadByte() + if err != nil { + return 0, err + } + + num |= (uint(b) & payload) << (sz * 7) // concats 7 bits chunks + sz++ + + if uint(b)&continuation == 0 { + break + } + } + + return num, nil +} + const ( payload = 0x7f // 0111 1111 continuation = 0x80 // 1000 0000 @@ -174,6 +319,40 @@ func isCopyFromDelta(cmd byte) bool { return (cmd&0x80) == 0 && cmd != 0 } +func decodeOffsetByteReader(cmd byte, delta io.ByteReader) (uint, error) { + var offset uint + if (cmd & 0x01) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + offset = uint(next) + } + if (cmd & 0x02) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + offset |= uint(next) << 8 + } + if (cmd & 0x04) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + offset |= uint(next) << 16 + } + if (cmd & 0x08) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + offset |= uint(next) << 24 + } + + return offset, nil +} + func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) { var offset uint if (cmd & 0x01) != 0 { @@ -208,6 +387,36 @@ func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) { return offset, delta, nil } +func decodeSizeByteReader(cmd byte, delta io.ByteReader) (uint, error) { + var sz uint + if (cmd & 0x10) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + sz = uint(next) + } + if (cmd & 0x20) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + sz |= uint(next) << 8 + } + if (cmd & 0x40) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + sz |= uint(next) << 16 + } + if sz == 0 { + sz = 0x10000 + } + + return sz, nil +} + func decodeSize(cmd byte, delta []byte) (uint, []byte, error) { var sz uint if (cmd & 0x10) != 0 { diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 6e6a68788..9ebb84a24 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -3,17 +3,16 @@ package packfile import ( "bufio" "bytes" - "compress/zlib" "fmt" "hash" "hash/crc32" "io" stdioutil "io/ioutil" - "sync" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/binary" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -114,7 +113,7 @@ func (s *Scanner) Header() (version, objects uint32, err error) { return } -// readSignature reads an returns the signature field in the packfile. +// readSignature reads a returns the signature field in the packfile. func (s *Scanner) readSignature() ([]byte, error) { var sig = make([]byte, 4) if _, err := io.ReadFull(s.r, sig); err != nil { @@ -320,29 +319,38 @@ func (s *Scanner) NextObject(w io.Writer) (written int64, crc32 uint32, err erro return } +// ReadObject returns a reader for the object content and an error +func (s *Scanner) ReadObject() (io.ReadCloser, error) { + s.pendingObject = nil + zr, err := sync.GetZlibReader(s.r) + + if err != nil { + return nil, fmt.Errorf("zlib reset error: %s", err) + } + + return ioutil.NewReadCloserWithCloser(zr.Reader, func() error { + sync.PutZlibReader(zr) + return nil + }), nil +} + // ReadRegularObject reads and write a non-deltified object // from it zlib stream in an object entry in the packfile. func (s *Scanner) copyObject(w io.Writer) (n int64, err error) { - zr := zlibReaderPool.Get().(io.ReadCloser) - defer zlibReaderPool.Put(zr) + zr, err := sync.GetZlibReader(s.r) + defer sync.PutZlibReader(zr) - if err = zr.(zlib.Resetter).Reset(s.r, nil); err != nil { + if err != nil { return 0, fmt.Errorf("zlib reset error: %s", err) } - defer ioutil.CheckClose(zr, &err) - buf := byteSlicePool.Get().([]byte) - n, err = io.CopyBuffer(w, zr, buf) - byteSlicePool.Put(buf) + defer ioutil.CheckClose(zr.Reader, &err) + buf := sync.GetByteSlice() + n, err = io.CopyBuffer(w, zr.Reader, *buf) + sync.PutByteSlice(buf) return } -var byteSlicePool = sync.Pool{ - New: func() interface{} { - return make([]byte, 32*1024) - }, -} - // SeekFromStart sets a new offset from start, returns the old position before // the change. func (s *Scanner) SeekFromStart(offset int64) (previous int64, err error) { @@ -372,9 +380,10 @@ func (s *Scanner) Checksum() (plumbing.Hash, error) { // Close reads the reader until io.EOF func (s *Scanner) Close() error { - buf := byteSlicePool.Get().([]byte) - _, err := io.CopyBuffer(stdioutil.Discard, s.r, buf) - byteSlicePool.Put(buf) + buf := sync.GetByteSlice() + _, err := io.CopyBuffer(stdioutil.Discard, s.r, *buf) + sync.PutByteSlice(buf) + return err } @@ -384,13 +393,13 @@ func (s *Scanner) Flush() error { } // scannerReader has the following characteristics: -// - Provides an io.SeekReader impl for bufio.Reader, when the underlying -// reader supports it. -// - Keeps track of the current read position, for when the underlying reader -// isn't an io.SeekReader, but we still want to know the current offset. -// - Writes to the hash writer what it reads, with the aid of a smaller buffer. -// The buffer helps avoid a performance penality for performing small writes -// to the crc32 hash writer. +// - Provides an io.SeekReader impl for bufio.Reader, when the underlying +// reader supports it. +// - Keeps track of the current read position, for when the underlying reader +// isn't an io.SeekReader, but we still want to know the current offset. +// - Writes to the hash writer what it reads, with the aid of a smaller buffer. +// The buffer helps avoid a performance penalty for performing small writes +// to the crc32 hash writer. type scannerReader struct { reader io.Reader crc io.Writer diff --git a/plumbing/hash.go b/plumbing/hash.go index afc602a9e..2fab7593b 100644 --- a/plumbing/hash.go +++ b/plumbing/hash.go @@ -2,11 +2,12 @@ package plumbing import ( "bytes" - "crypto/sha1" + "crypto" "encoding/hex" - "hash" "sort" "strconv" + + "github.com/go-git/go-git/v5/plumbing/hash" ) // Hash SHA1 hashed content @@ -46,7 +47,7 @@ type Hasher struct { } func NewHasher(t ObjectType, size int64) Hasher { - h := Hasher{sha1.New()} + h := Hasher{hash.New(crypto.SHA1)} h.Write(t.Bytes()) h.Write([]byte(" ")) h.Write([]byte(strconv.FormatInt(size, 10))) diff --git a/plumbing/hash/hash.go b/plumbing/hash/hash.go new file mode 100644 index 000000000..fe3bf7655 --- /dev/null +++ b/plumbing/hash/hash.go @@ -0,0 +1,59 @@ +// package hash provides a way for managing the +// underlying hash implementations used across go-git. +package hash + +import ( + "crypto" + "fmt" + "hash" + + "github.com/pjbgf/sha1cd/cgo" +) + +// algos is a map of hash algorithms. +var algos = map[crypto.Hash]func() hash.Hash{} + +func init() { + reset() +} + +// reset resets the default algos value. Can be used after running tests +// that registers new algorithms to avoid side effects. +func reset() { + // For performance reasons the cgo version of the collision + // detection algorithm is being used. + algos[crypto.SHA1] = cgo.New +} + +// RegisterHash allows for the hash algorithm used to be overriden. +// This ensures the hash selection for go-git must be explicit, when +// overriding the default value. +func RegisterHash(h crypto.Hash, f func() hash.Hash) error { + if f == nil { + return fmt.Errorf("cannot register hash: f is nil") + } + + switch h { + case crypto.SHA1: + algos[h] = f + default: + return fmt.Errorf("unsupported hash function: %v", h) + } + return nil +} + +// Hash is the same as hash.Hash. This allows consumers +// to not having to import this package alongside "hash". +type Hash interface { + hash.Hash +} + +// New returns a new Hash for the given hash function. +// It panics if the hash function is not registered. +func New(h crypto.Hash) Hash { + hh, ok := algos[h] + if !ok { + panic(fmt.Sprintf("hash algorithm not registered: %v", h)) + } + return hh() +} diff --git a/plumbing/hash/hash_test.go b/plumbing/hash/hash_test.go new file mode 100644 index 000000000..f70ad117e --- /dev/null +++ b/plumbing/hash/hash_test.go @@ -0,0 +1,103 @@ +package hash + +import ( + "crypto" + "crypto/sha1" + "crypto/sha512" + "encoding/hex" + "hash" + "strings" + "testing" +) + +func TestRegisterHash(t *testing.T) { + // Reset default hash to avoid side effects. + defer reset() + + tests := []struct { + name string + hash crypto.Hash + new func() hash.Hash + wantErr string + }{ + { + name: "sha1", + hash: crypto.SHA1, + new: sha1.New, + }, + { + name: "sha1", + hash: crypto.SHA1, + wantErr: "cannot register hash: f is nil", + }, + { + name: "sha512", + hash: crypto.SHA512, + new: sha512.New, + wantErr: "unsupported hash function", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RegisterHash(tt.hash, tt.new) + if tt.wantErr == "" && err != nil { + t.Errorf("unexpected error: %v", err) + } else if tt.wantErr != "" && err == nil { + t.Errorf("expected error: %v got: nil", tt.wantErr) + } else if err != nil && !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("expected error: %v got: %v", tt.wantErr, err) + } + }) + } +} + +// Verifies that the SHA1 implementation used is collision-resistant +// by default. +func TestSha1Collision(t *testing.T) { + defer reset() + + tests := []struct { + name string + content string + hash string + before func() + }{ + { + name: "sha-mbles-1: with collision detection", + content: "99040d047fe81780012000ff4b65792069732070617274206f66206120636f6c6c6973696f6e212049742773206120747261702179c61af0afcc054515d9274e7307624b1dc7fb23988bb8de8b575dba7b9eab31c1674b6d974378a827732ff5851c76a2e60772b5a47ce1eac40bb993c12d8c70e24a4f8d5fcdedc1b32c9cf19e31af2429759d42e4dfdb31719f587623ee552939b6dcdc459fca53553b70f87ede30a247ea3af6c759a2f20b320d760db64ff479084fd3ccb3cdd48362d96a9c430617caff6c36c637e53fde28417f626fec54ed7943a46e5f5730f2bb38fb1df6e0090010d00e24ad78bf92641993608e8d158a789f34c46fe1e6027f35a4cbfb827076c50eca0e8b7cca69bb2c2b790259f9bf9570dd8d4437a3115faff7c3cac09ad25266055c27104755178eaeff825a2caa2acfb5de64ce7641dc59a541a9fc9c756756e2e23dc713c8c24c9790aa6b0e38a7f55f14452a1ca2850ddd9562fd9a18ad42496aa97008f74672f68ef461eb88b09933d626b4f918749cc027fddd6c425fc4216835d0134d15285bab2cb784a4f7cbb4fb514d4bf0f6237cf00a9e9f132b9a066e6fd17f6c42987478586ff651af96747fb426b9872b9a88e4063f59bb334cc00650f83a80c42751b71974d300fc2819a2e8f1e32c1b51cb18e6bfc4db9baef675d4aaf5b1574a047f8f6dd2ec153a93412293974d928f88ced9363cfef97ce2e742bf34c96b8ef3875676fea5cca8e5f7dea0bab2413d4de00ee71ee01f162bdb6d1eafd925e6aebaae6a354ef17cf205a404fbdb12fc454d41fdd95cf2459664a2ad032d1da60a73264075d7f1e0d6c1403ae7a0d861df3fe5707188dd5e07d1589b9f8b6630553f8fc352b3e0c27da80bddba4c64020d", + hash: "4f3d9be4a472c4dae83c6314aa6c36a064c1fd14", + }, + { + name: "sha-mbles-1: with default SHA1", + content: "99040d047fe81780012000ff4b65792069732070617274206f66206120636f6c6c6973696f6e212049742773206120747261702179c61af0afcc054515d9274e7307624b1dc7fb23988bb8de8b575dba7b9eab31c1674b6d974378a827732ff5851c76a2e60772b5a47ce1eac40bb993c12d8c70e24a4f8d5fcdedc1b32c9cf19e31af2429759d42e4dfdb31719f587623ee552939b6dcdc459fca53553b70f87ede30a247ea3af6c759a2f20b320d760db64ff479084fd3ccb3cdd48362d96a9c430617caff6c36c637e53fde28417f626fec54ed7943a46e5f5730f2bb38fb1df6e0090010d00e24ad78bf92641993608e8d158a789f34c46fe1e6027f35a4cbfb827076c50eca0e8b7cca69bb2c2b790259f9bf9570dd8d4437a3115faff7c3cac09ad25266055c27104755178eaeff825a2caa2acfb5de64ce7641dc59a541a9fc9c756756e2e23dc713c8c24c9790aa6b0e38a7f55f14452a1ca2850ddd9562fd9a18ad42496aa97008f74672f68ef461eb88b09933d626b4f918749cc027fddd6c425fc4216835d0134d15285bab2cb784a4f7cbb4fb514d4bf0f6237cf00a9e9f132b9a066e6fd17f6c42987478586ff651af96747fb426b9872b9a88e4063f59bb334cc00650f83a80c42751b71974d300fc2819a2e8f1e32c1b51cb18e6bfc4db9baef675d4aaf5b1574a047f8f6dd2ec153a93412293974d928f88ced9363cfef97ce2e742bf34c96b8ef3875676fea5cca8e5f7dea0bab2413d4de00ee71ee01f162bdb6d1eafd925e6aebaae6a354ef17cf205a404fbdb12fc454d41fdd95cf2459664a2ad032d1da60a73264075d7f1e0d6c1403ae7a0d861df3fe5707188dd5e07d1589b9f8b6630553f8fc352b3e0c27da80bddba4c64020d", + hash: "8ac60ba76f1999a1ab70223f225aefdc78d4ddc0", + before: func() { + RegisterHash(crypto.SHA1, sha1.New) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.before != nil { + tt.before() + } + + h := New(crypto.SHA1) + data, err := hex.DecodeString(tt.content) + if err != nil { + t.Fatal(err) + } + + h.Reset() + h.Write(data) + sum := h.Sum(nil) + got := hex.EncodeToString(sum) + + if tt.hash != got { + t.Errorf("\n got: %q\nwanted: %q", got, tt.hash) + } + }) + } +} diff --git a/plumbing/memory.go b/plumbing/memory.go index 21337cc0d..6d11271dd 100644 --- a/plumbing/memory.go +++ b/plumbing/memory.go @@ -25,13 +25,13 @@ func (o *MemoryObject) Hash() Hash { return o.h } -// Type return the ObjectType +// Type returns the ObjectType func (o *MemoryObject) Type() ObjectType { return o.t } // SetType sets the ObjectType func (o *MemoryObject) SetType(t ObjectType) { o.t = t } -// Size return the size of the object +// Size returns the size of the object func (o *MemoryObject) Size() int64 { return o.sz } // SetSize set the object size, a content of the given size should be written diff --git a/plumbing/object/change.go b/plumbing/object/change.go index 8b119bc9c..3c619df86 100644 --- a/plumbing/object/change.go +++ b/plumbing/object/change.go @@ -39,7 +39,7 @@ func (c *Change) Action() (merkletrie.Action, error) { return merkletrie.Modify, nil } -// Files return the files before and after a change. +// Files returns the files before and after a change. // For insertions from will be nil. For deletions to will be nil. func (c *Change) Files() (from, to *File, err error) { action, err := c.Action() diff --git a/plumbing/object/change_adaptor.go b/plumbing/object/change_adaptor.go index f70118828..b96ee84d9 100644 --- a/plumbing/object/change_adaptor.go +++ b/plumbing/object/change_adaptor.go @@ -16,11 +16,11 @@ func newChange(c merkletrie.Change) (*Change, error) { var err error if ret.From, err = newChangeEntry(c.From); err != nil { - return nil, fmt.Errorf("From field: %s", err) + return nil, fmt.Errorf("from field: %s", err) } if ret.To, err = newChangeEntry(c.To); err != nil { - return nil, fmt.Errorf("To field: %s", err) + return nil, fmt.Errorf("to field: %s", err) } return ret, nil diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index 7a1b8e5ae..d2f718408 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -1,7 +1,6 @@ package object import ( - "bufio" "bytes" "context" "errors" @@ -14,6 +13,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) const ( @@ -180,9 +180,8 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } defer ioutil.CheckClose(reader, &err) - r := bufPool.Get().(*bufio.Reader) - defer bufPool.Put(r) - r.Reset(reader) + r := sync.GetBufioReader(reader) + defer sync.PutBufioReader(r) var message bool var pgpsig bool diff --git a/plumbing/object/common.go b/plumbing/object/common.go deleted file mode 100644 index 3591f5f0a..000000000 --- a/plumbing/object/common.go +++ /dev/null @@ -1,12 +0,0 @@ -package object - -import ( - "bufio" - "sync" -) - -var bufPool = sync.Pool{ - New: func() interface{} { - return bufio.NewReader(nil) - }, -} diff --git a/plumbing/object/patch.go b/plumbing/object/patch.go index 56b62c191..06bc35bbc 100644 --- a/plumbing/object/patch.go +++ b/plumbing/object/patch.go @@ -96,10 +96,6 @@ func filePatchWithContext(ctx context.Context, c *Change) (fdiff.FilePatch, erro } -func filePatch(c *Change) (fdiff.FilePatch, error) { - return filePatchWithContext(context.Background(), c) -} - func fileContent(f *File) (content string, isBinary bool, err error) { if f == nil { return diff --git a/plumbing/object/rename.go b/plumbing/object/rename.go index 7fed72c2f..0394613ff 100644 --- a/plumbing/object/rename.go +++ b/plumbing/object/rename.go @@ -403,10 +403,16 @@ func min(a, b int) int { return b } +const maxMatrixSize = 10000 + func buildSimilarityMatrix(srcs, dsts []*Change, renameScore int) (similarityMatrix, error) { // Allocate for the worst-case scenario where every pair has a score // that we need to consider. We might not need that many. - matrix := make(similarityMatrix, 0, len(srcs)*len(dsts)) + matrixSize := len(srcs) * len(dsts) + if matrixSize > maxMatrixSize { + matrixSize = maxMatrixSize + } + matrix := make(similarityMatrix, 0, matrixSize) srcSizes := make([]int64, len(srcs)) dstSizes := make([]int64, len(dsts)) dstTooLarge := make(map[int]bool) diff --git a/plumbing/object/tag.go b/plumbing/object/tag.go index 216010d91..84066f768 100644 --- a/plumbing/object/tag.go +++ b/plumbing/object/tag.go @@ -1,7 +1,6 @@ package object import ( - "bufio" "bytes" "fmt" "io" @@ -13,6 +12,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) // Tag represents an annotated tag object. It points to a single git object of @@ -93,9 +93,9 @@ func (t *Tag) Decode(o plumbing.EncodedObject) (err error) { } defer ioutil.CheckClose(reader, &err) - r := bufPool.Get().(*bufio.Reader) - defer bufPool.Put(r) - r.Reset(reader) + r := sync.GetBufioReader(reader) + defer sync.PutBufioReader(r) + for { var line []byte line, err = r.ReadBytes('\n') diff --git a/plumbing/object/tree.go b/plumbing/object/tree.go index 5e6378ca4..e9f7666b8 100644 --- a/plumbing/object/tree.go +++ b/plumbing/object/tree.go @@ -1,7 +1,6 @@ package object import ( - "bufio" "context" "errors" "fmt" @@ -14,6 +13,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) const ( @@ -230,9 +230,9 @@ func (t *Tree) Decode(o plumbing.EncodedObject) (err error) { } defer ioutil.CheckClose(reader, &err) - r := bufPool.Get().(*bufio.Reader) - defer bufPool.Put(r) - r.Reset(reader) + r := sync.GetBufioReader(reader) + defer sync.PutBufioReader(r) + for { str, err := r.ReadString(' ') if err != nil { diff --git a/plumbing/object/treenoder.go b/plumbing/object/treenoder.go index b4891b957..6e7b334cb 100644 --- a/plumbing/object/treenoder.go +++ b/plumbing/object/treenoder.go @@ -38,6 +38,10 @@ func NewTreeRootNode(t *Tree) noder.Noder { } } +func (t *treeNoder) Skip() bool { + return false +} + func (t *treeNoder) isRoot() bool { return t.name == "" } diff --git a/plumbing/protocol/packp/capability/capability.go b/plumbing/protocol/packp/capability/capability.go index 8d6a56f53..b52e8a49d 100644 --- a/plumbing/protocol/packp/capability/capability.go +++ b/plumbing/protocol/packp/capability/capability.go @@ -1,6 +1,11 @@ // Package capability defines the server and client capabilities. package capability +import ( + "fmt" + "os" +) + // Capability describes a server or client capability. type Capability string @@ -238,7 +243,15 @@ const ( Filter Capability = "filter" ) -const DefaultAgent = "go-git/4.x" +const userAgent = "go-git/5.x" + +// DefaultAgent provides the user agent string. +func DefaultAgent() string { + if envUserAgent, ok := os.LookupEnv("GO_GIT_USER_AGENT_EXTRA"); ok { + return fmt.Sprintf("%s %s", userAgent, envUserAgent) + } + return userAgent +} var known = map[Capability]bool{ MultiACK: true, MultiACKDetailed: true, NoDone: true, ThinPack: true, diff --git a/plumbing/protocol/packp/capability/capability_test.go b/plumbing/protocol/packp/capability/capability_test.go new file mode 100644 index 000000000..f1fd0282a --- /dev/null +++ b/plumbing/protocol/packp/capability/capability_test.go @@ -0,0 +1,22 @@ +package capability + +import ( + "fmt" + "os" + + check "gopkg.in/check.v1" +) + +var _ = check.Suite(&SuiteCapabilities{}) + +func (s *SuiteCapabilities) TestDefaultAgent(c *check.C) { + os.Unsetenv("GO_GIT_USER_AGENT_EXTRA") + ua := DefaultAgent() + c.Assert(ua, check.Equals, userAgent) +} + +func (s *SuiteCapabilities) TestEnvAgent(c *check.C) { + os.Setenv("GO_GIT_USER_AGENT_EXTRA", "abc xyz") + ua := DefaultAgent() + c.Assert(ua, check.Equals, fmt.Sprintf("%s %s", userAgent, "abc xyz")) +} diff --git a/plumbing/protocol/packp/capability/list.go b/plumbing/protocol/packp/capability/list.go index f41ec799c..553d81cbe 100644 --- a/plumbing/protocol/packp/capability/list.go +++ b/plumbing/protocol/packp/capability/list.go @@ -86,7 +86,9 @@ func (l *List) Get(capability Capability) []string { // Set sets a capability removing the previous values func (l *List) Set(capability Capability, values ...string) error { - delete(l.m, capability) + if _, ok := l.m[capability]; ok { + l.m[capability].Values = l.m[capability].Values[:0] + } return l.Add(capability, values...) } diff --git a/plumbing/protocol/packp/capability/list_test.go b/plumbing/protocol/packp/capability/list_test.go index 61b0b13be..71181cbc9 100644 --- a/plumbing/protocol/packp/capability/list_test.go +++ b/plumbing/protocol/packp/capability/list_test.go @@ -122,6 +122,17 @@ func (s *SuiteCapabilities) TestSetEmpty(c *check.C) { c.Assert(cap.Get(Agent), check.HasLen, 1) } +func (s *SuiteCapabilities) TestSetDuplicate(c *check.C) { + cap := NewList() + err := cap.Set(Agent, "baz") + c.Assert(err, check.IsNil) + + err = cap.Set(Agent, "bar") + c.Assert(err, check.IsNil) + + c.Assert(cap.String(), check.Equals, "agent=bar") +} + func (s *SuiteCapabilities) TestGetEmpty(c *check.C) { cap := NewList() c.Assert(cap.Get(Agent), check.HasLen, 0) diff --git a/plumbing/protocol/packp/common.go b/plumbing/protocol/packp/common.go index ab07ac8f7..fef50a450 100644 --- a/plumbing/protocol/packp/common.go +++ b/plumbing/protocol/packp/common.go @@ -19,7 +19,6 @@ var ( // common sp = []byte(" ") eol = []byte("\n") - eq = []byte{'='} // advertised-refs null = []byte("\x00") diff --git a/plumbing/protocol/packp/srvresp.go b/plumbing/protocol/packp/srvresp.go index b3a7ee804..8cd0a7247 100644 --- a/plumbing/protocol/packp/srvresp.go +++ b/plumbing/protocol/packp/srvresp.go @@ -21,11 +21,6 @@ type ServerResponse struct { // Decode decodes the response into the struct, isMultiACK should be true, if // the request was done with multi_ack or multi_ack_detailed capabilities. func (r *ServerResponse) Decode(reader *bufio.Reader, isMultiACK bool) error { - // TODO: implement support for multi_ack or multi_ack_detailed responses - if isMultiACK { - return errors.New("multi_ack and multi_ack_detailed are not supported") - } - s := pktline.NewScanner(reader) for s.Scan() { @@ -48,7 +43,23 @@ func (r *ServerResponse) Decode(reader *bufio.Reader, isMultiACK bool) error { } } - return s.Err() + // isMultiACK is true when the remote server advertises the related + // capabilities when they are not in transport.UnsupportedCapabilities. + // + // Users may decide to remove multi_ack and multi_ack_detailed from the + // unsupported capabilities list, which allows them to do initial clones + // from Azure DevOps. + // + // Follow-up fetches may error, therefore errors are wrapped with additional + // information highlighting that this capabilities are not supported by go-git. + // + // TODO: Implement support for multi_ack or multi_ack_detailed responses. + err := s.Err() + if err != nil && isMultiACK { + return fmt.Errorf("multi_ack and multi_ack_detailed are not supported: %w", err) + } + + return err } // stopReading detects when a valid command such as ACK or NAK is found to be @@ -113,8 +124,9 @@ func (r *ServerResponse) decodeACKLine(line []byte) error { } // Encode encodes the ServerResponse into a writer. -func (r *ServerResponse) Encode(w io.Writer) error { - if len(r.ACKs) > 1 { +func (r *ServerResponse) Encode(w io.Writer, isMultiACK bool) error { + if len(r.ACKs) > 1 && !isMultiACK { + // For further information, refer to comments in the Decode func above. return errors.New("multi_ack and multi_ack_detailed are not supported") } diff --git a/plumbing/protocol/packp/srvresp_test.go b/plumbing/protocol/packp/srvresp_test.go index 02fab424e..aa0af528a 100644 --- a/plumbing/protocol/packp/srvresp_test.go +++ b/plumbing/protocol/packp/srvresp_test.go @@ -72,8 +72,21 @@ func (s *ServerResponseSuite) TestDecodeMalformed(c *C) { c.Assert(err, NotNil) } +// multi_ack isn't fully implemented, this ensures that Decode ignores that fact, +// as in some circumstances that's OK to assume so. +// +// TODO: Review as part of multi_ack implementation. func (s *ServerResponseSuite) TestDecodeMultiACK(c *C) { + raw := "" + + "0031ACK 1111111111111111111111111111111111111111\n" + + "0031ACK 6ecf0ef2c2dffb796033e5a02219af86ec6584e5\n" + + "00080PACK\n" + sr := &ServerResponse{} - err := sr.Decode(bufio.NewReader(bytes.NewBuffer(nil)), true) - c.Assert(err, NotNil) + err := sr.Decode(bufio.NewReader(bytes.NewBufferString(raw)), true) + c.Assert(err, IsNil) + + c.Assert(sr.ACKs, HasLen, 2) + c.Assert(sr.ACKs[0], Equals, plumbing.NewHash("1111111111111111111111111111111111111111")) + c.Assert(sr.ACKs[1], Equals, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) } diff --git a/plumbing/protocol/packp/ulreq.go b/plumbing/protocol/packp/ulreq.go index ddec06e99..344f8c7e3 100644 --- a/plumbing/protocol/packp/ulreq.go +++ b/plumbing/protocol/packp/ulreq.go @@ -95,7 +95,7 @@ func NewUploadRequestFromCapabilities(adv *capability.List) *UploadRequest { } if adv.Supports(capability.Agent) { - r.Capabilities.Set(capability.Agent, capability.DefaultAgent) + r.Capabilities.Set(capability.Agent, capability.DefaultAgent()) } return r diff --git a/plumbing/protocol/packp/ulreq_test.go b/plumbing/protocol/packp/ulreq_test.go index 0b3b61653..2797a4ea5 100644 --- a/plumbing/protocol/packp/ulreq_test.go +++ b/plumbing/protocol/packp/ulreq_test.go @@ -25,7 +25,7 @@ func (s *UlReqSuite) TestNewUploadRequestFromCapabilities(c *C) { r := NewUploadRequestFromCapabilities(cap) c.Assert(r.Capabilities.String(), Equals, - "multi_ack_detailed side-band-64k thin-pack ofs-delta agent=go-git/4.x", + "multi_ack_detailed side-band-64k thin-pack ofs-delta agent=go-git/5.x", ) } diff --git a/plumbing/protocol/packp/updreq.go b/plumbing/protocol/packp/updreq.go index 4d927d8b8..8f39b39cb 100644 --- a/plumbing/protocol/packp/updreq.go +++ b/plumbing/protocol/packp/updreq.go @@ -19,6 +19,7 @@ var ( type ReferenceUpdateRequest struct { Capabilities *capability.List Commands []*Command + Options []*Option Shallow *plumbing.Hash // Packfile contains an optional packfile reader. Packfile io.ReadCloser @@ -58,7 +59,7 @@ func NewReferenceUpdateRequestFromCapabilities(adv *capability.List) *ReferenceU r := NewReferenceUpdateRequest() if adv.Supports(capability.Agent) { - r.Capabilities.Set(capability.Agent, capability.DefaultAgent) + r.Capabilities.Set(capability.Agent, capability.DefaultAgent()) } if adv.Supports(capability.ReportStatus) { @@ -86,9 +87,9 @@ type Action string const ( Create Action = "create" - Update = "update" - Delete = "delete" - Invalid = "invalid" + Update Action = "update" + Delete Action = "delete" + Invalid Action = "invalid" ) type Command struct { @@ -120,3 +121,8 @@ func (c *Command) validate() error { return nil } + +type Option struct { + Key string + Value string +} diff --git a/plumbing/protocol/packp/updreq_encode.go b/plumbing/protocol/packp/updreq_encode.go index 2545e935e..1205cfaf1 100644 --- a/plumbing/protocol/packp/updreq_encode.go +++ b/plumbing/protocol/packp/updreq_encode.go @@ -9,10 +9,6 @@ import ( "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" ) -var ( - zeroHashString = plumbing.ZeroHash.String() -) - // Encode writes the ReferenceUpdateRequest encoding to the stream. func (req *ReferenceUpdateRequest) Encode(w io.Writer) error { if err := req.validate(); err != nil { @@ -29,6 +25,12 @@ func (req *ReferenceUpdateRequest) Encode(w io.Writer) error { return err } + if req.Capabilities.Supports(capability.PushOptions) { + if err := req.encodeOptions(e, req.Options); err != nil { + return err + } + } + if req.Packfile != nil { if _, err := io.Copy(w, req.Packfile); err != nil { return err @@ -73,3 +75,15 @@ func formatCommand(cmd *Command) string { n := cmd.New.String() return fmt.Sprintf("%s %s %s", o, n, cmd.Name) } + +func (req *ReferenceUpdateRequest) encodeOptions(e *pktline.Encoder, + opts []*Option) error { + + for _, opt := range opts { + if err := e.Encodef("%s=%s", opt.Key, opt.Value); err != nil { + return err + } + } + + return e.Flush() +} diff --git a/plumbing/protocol/packp/updreq_encode_test.go b/plumbing/protocol/packp/updreq_encode_test.go index 5ad2b1bb7..4370b794f 100644 --- a/plumbing/protocol/packp/updreq_encode_test.go +++ b/plumbing/protocol/packp/updreq_encode_test.go @@ -5,9 +5,11 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/pktline" + "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" - . "gopkg.in/check.v1" "io/ioutil" + + . "gopkg.in/check.v1" ) type UpdReqEncodeSuite struct{} @@ -142,3 +144,48 @@ func (s *UpdReqEncodeSuite) TestWithPackfile(c *C) { s.testEncode(c, r, expected) } + +func (s *UpdReqEncodeSuite) TestPushOptions(c *C) { + hash1 := plumbing.NewHash("1ecf0ef2c2dffb796033e5a02219af86ec6584e5") + hash2 := plumbing.NewHash("2ecf0ef2c2dffb796033e5a02219af86ec6584e5") + name := plumbing.ReferenceName("myref") + + r := NewReferenceUpdateRequest() + r.Capabilities.Set(capability.PushOptions) + r.Commands = []*Command{ + {Name: name, Old: hash1, New: hash2}, + } + r.Options = []*Option{ + {Key: "SomeKey", Value: "SomeValue"}, + {Key: "AnotherKey", Value: "AnotherValue"}, + } + + expected := pktlines(c, + "1ecf0ef2c2dffb796033e5a02219af86ec6584e5 2ecf0ef2c2dffb796033e5a02219af86ec6584e5 myref\x00push-options", + pktline.FlushString, + "SomeKey=SomeValue", + "AnotherKey=AnotherValue", + pktline.FlushString, + ) + + s.testEncode(c, r, expected) +} + +func (s *UpdReqEncodeSuite) TestPushAtomic(c *C) { + hash1 := plumbing.NewHash("1ecf0ef2c2dffb796033e5a02219af86ec6584e5") + hash2 := plumbing.NewHash("2ecf0ef2c2dffb796033e5a02219af86ec6584e5") + name := plumbing.ReferenceName("myref") + + r := NewReferenceUpdateRequest() + r.Capabilities.Set(capability.Atomic) + r.Commands = []*Command{ + {Name: name, Old: hash1, New: hash2}, + } + + expected := pktlines(c, + "1ecf0ef2c2dffb796033e5a02219af86ec6584e5 2ecf0ef2c2dffb796033e5a02219af86ec6584e5 myref\x00atomic", + pktline.FlushString, + ) + + s.testEncode(c, r, expected) +} diff --git a/plumbing/protocol/packp/updreq_test.go b/plumbing/protocol/packp/updreq_test.go index c4ccbaf64..80e03fbe7 100644 --- a/plumbing/protocol/packp/updreq_test.go +++ b/plumbing/protocol/packp/updreq_test.go @@ -23,14 +23,14 @@ func (s *UpdReqSuite) TestNewReferenceUpdateRequestFromCapabilities(c *C) { r := NewReferenceUpdateRequestFromCapabilities(cap) c.Assert(r.Capabilities.String(), Equals, - "agent=go-git/4.x report-status", + "agent=go-git/5.x report-status", ) cap = capability.NewList() cap.Set(capability.Agent, "foo") r = NewReferenceUpdateRequestFromCapabilities(cap) - c.Assert(r.Capabilities.String(), Equals, "agent=go-git/4.x") + c.Assert(r.Capabilities.String(), Equals, "agent=go-git/5.x") cap = capability.NewList() diff --git a/plumbing/protocol/packp/uppackreq_test.go b/plumbing/protocol/packp/uppackreq_test.go index f723e3cc7..5a6eb2cc8 100644 --- a/plumbing/protocol/packp/uppackreq_test.go +++ b/plumbing/protocol/packp/uppackreq_test.go @@ -18,7 +18,7 @@ func (s *UploadPackRequestSuite) TestNewUploadPackRequestFromCapabilities(c *C) cap.Set(capability.Agent, "foo") r := NewUploadPackRequestFromCapabilities(cap) - c.Assert(r.Capabilities.String(), Equals, "agent=go-git/4.x") + c.Assert(r.Capabilities.String(), Equals, "agent=go-git/5.x") } func (s *UploadPackRequestSuite) TestIsEmpty(c *C) { diff --git a/plumbing/protocol/packp/uppackresp.go b/plumbing/protocol/packp/uppackresp.go index a9a7192ea..a485cb7b2 100644 --- a/plumbing/protocol/packp/uppackresp.go +++ b/plumbing/protocol/packp/uppackresp.go @@ -24,7 +24,6 @@ type UploadPackResponse struct { r io.ReadCloser isShallow bool isMultiACK bool - isOk bool } // NewUploadPackResponse create a new UploadPackResponse instance, the request @@ -79,7 +78,7 @@ func (r *UploadPackResponse) Encode(w io.Writer) (err error) { } } - if err := r.ServerResponse.Encode(w); err != nil { + if err := r.ServerResponse.Encode(w, r.isMultiACK); err != nil { return err } diff --git a/plumbing/protocol/packp/uppackresp_test.go b/plumbing/protocol/packp/uppackresp_test.go index 260dc5748..3f87804a5 100644 --- a/plumbing/protocol/packp/uppackresp_test.go +++ b/plumbing/protocol/packp/uppackresp_test.go @@ -59,6 +59,10 @@ func (s *UploadPackResponseSuite) TestDecodeMalformed(c *C) { c.Assert(err, NotNil) } +// multi_ack isn't fully implemented, this ensures that Decode ignores that fact, +// as in some circumstances that's OK to assume so. +// +// TODO: Review as part of multi_ack implementation. func (s *UploadPackResponseSuite) TestDecodeMultiACK(c *C) { req := NewUploadPackRequest() req.Capabilities.Set(capability.MultiACK) @@ -67,7 +71,7 @@ func (s *UploadPackResponseSuite) TestDecodeMultiACK(c *C) { defer res.Close() err := res.Decode(ioutil.NopCloser(bytes.NewBuffer(nil))) - c.Assert(err, NotNil) + c.Assert(err, IsNil) } func (s *UploadPackResponseSuite) TestReadNoDecode(c *C) { diff --git a/plumbing/reference.go b/plumbing/reference.go index 08e908f1f..eef11e827 100644 --- a/plumbing/reference.go +++ b/plumbing/reference.go @@ -168,22 +168,22 @@ func NewHashReference(n ReferenceName, h Hash) *Reference { } } -// Type return the type of a reference +// Type returns the type of a reference func (r *Reference) Type() ReferenceType { return r.t } -// Name return the name of a reference +// Name returns the name of a reference func (r *Reference) Name() ReferenceName { return r.n } -// Hash return the hash of a hash reference +// Hash returns the hash of a hash reference func (r *Reference) Hash() Hash { return r.h } -// Target return the target of a symbolic reference +// Target returns the target of a symbolic reference func (r *Reference) Target() ReferenceName { return r.target } @@ -204,6 +204,21 @@ func (r *Reference) Strings() [2]string { } func (r *Reference) String() string { - s := r.Strings() - return fmt.Sprintf("%s %s", s[1], s[0]) + ref := "" + switch r.Type() { + case HashReference: + ref = r.Hash().String() + case SymbolicReference: + ref = symrefPrefix + r.Target().String() + default: + return "" + } + + name := r.Name().String() + var v strings.Builder + v.Grow(len(ref) + len(name) + 1) + v.WriteString(ref) + v.WriteString(" ") + v.WriteString(name) + return v.String() } diff --git a/plumbing/reference_test.go b/plumbing/reference_test.go index b3ccf5340..e69076ff6 100644 --- a/plumbing/reference_test.go +++ b/plumbing/reference_test.go @@ -1,6 +1,10 @@ package plumbing -import . "gopkg.in/check.v1" +import ( + "testing" + + . "gopkg.in/check.v1" +) type ReferenceSuite struct{} @@ -98,3 +102,21 @@ func (s *ReferenceSuite) TestIsTag(c *C) { r := ReferenceName("refs/tags/v3.1.") c.Assert(r.IsTag(), Equals, true) } + +func benchMarkReferenceString(r *Reference, b *testing.B) { + for n := 0; n < b.N; n++ { + r.String() + } +} + +func BenchmarkReferenceStringSymbolic(b *testing.B) { + benchMarkReferenceString(NewSymbolicReference("v3.1.1", "refs/tags/v3.1.1"), b) +} + +func BenchmarkReferenceStringHash(b *testing.B) { + benchMarkReferenceString(NewHashReference("v3.1.1", NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")), b) +} + +func BenchmarkReferenceStringInvalid(b *testing.B) { + benchMarkReferenceString(&Reference{}, b) +} diff --git a/plumbing/revlist/revlist_test.go b/plumbing/revlist/revlist_test.go index a1ee504e8..9f2f93b53 100644 --- a/plumbing/revlist/revlist_test.go +++ b/plumbing/revlist/revlist_test.go @@ -55,12 +55,6 @@ func (s *RevListSuite) SetUpTest(c *C) { s.Storer = sto } -func (s *RevListSuite) commit(c *C, h plumbing.Hash) *object.Commit { - commit, err := object.GetCommit(s.Storer, h) - c.Assert(err, IsNil) - return commit -} - func (s *RevListSuite) TestRevListObjects_Submodules(c *C) { submodules := map[string]bool{ "6ecf0ef2c2dffb796033e5a02219af86ec6584e5": true, diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index dfe309db1..d8a9c27a6 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -52,8 +52,8 @@ type DeltaObjectStorer interface { DeltaObject(plumbing.ObjectType, plumbing.Hash) (plumbing.EncodedObject, error) } -// Transactioner is a optional method for ObjectStorer, it enable transaction -// base write and read operations in the storage +// Transactioner is a optional method for ObjectStorer, it enables transactional read and write +// operations. type Transactioner interface { // Begin starts a transaction. Begin() Transaction @@ -87,8 +87,8 @@ type PackedObjectStorer interface { DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error } -// PackfileWriter is a optional method for ObjectStorer, it enable direct write -// of packfile to the storage +// PackfileWriter is an optional method for ObjectStorer, it enables directly writing +// a packfile to storage. type PackfileWriter interface { // PackfileWriter returns a writer for writing a packfile to the storage // diff --git a/plumbing/transport/client/client_test.go b/plumbing/transport/client/client_test.go index 9ebe113b1..92db525a5 100644 --- a/plumbing/transport/client/client_test.go +++ b/plumbing/transport/client/client_test.go @@ -1,7 +1,6 @@ package client import ( - "fmt" "net/http" "testing" @@ -68,7 +67,3 @@ func (*dummyClient) NewReceivePackSession(*transport.Endpoint, transport.AuthMet transport.ReceivePackSession, error) { return nil, nil } - -func typeAsString(v interface{}) string { - return fmt.Sprintf("%T", v) -} diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index a9ee2caee..a2a78f028 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -112,7 +112,7 @@ type Endpoint struct { Port int // Path is the repository path. Path string - // InsecureSkipTLS skips ssl verify if protocal is https + // InsecureSkipTLS skips ssl verify if protocol is https InsecureSkipTLS bool // CaBundle specify additional ca bundle with system cert pool CaBundle []byte diff --git a/plumbing/transport/git/common.go b/plumbing/transport/git/common.go index 306aae261..c18d600c2 100644 --- a/plumbing/transport/git/common.go +++ b/plumbing/transport/git/common.go @@ -77,14 +77,14 @@ func (c *command) StderrPipe() (io.Reader, error) { return nil, nil } -// StdinPipe return the underlying connection as WriteCloser, wrapped to prevent +// StdinPipe returns the underlying connection as WriteCloser, wrapped to prevent // call to the Close function from the connection, a command execution in git // protocol can't be closed or killed func (c *command) StdinPipe() (io.WriteCloser, error) { return ioutil.WriteNopCloser(c.conn), nil } -// StdoutPipe return the underlying connection as Reader +// StdoutPipe returns the underlying connection as Reader func (c *command) StdoutPipe() (io.Reader, error) { return c.conn, nil } diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index fdb148f59..d0e9a2974 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -428,11 +428,6 @@ func isRepoNotFoundError(s string) bool { return false } -var ( - nak = []byte("NAK") - eol = []byte("\n") -) - // uploadPack implements the git-upload-pack protocol. func uploadPack(w io.WriteCloser, r io.Reader, req *packp.UploadPackRequest) error { // TODO support multi_ack mode diff --git a/plumbing/transport/server/server.go b/plumbing/transport/server/server.go index 8ab70fe70..11fa0c801 100644 --- a/plumbing/transport/server/server.go +++ b/plumbing/transport/server/server.go @@ -189,7 +189,7 @@ func (s *upSession) objectsToUpload(req *packp.UploadPackRequest) ([]plumbing.Ha } func (*upSession) setSupportedCapabilities(c *capability.List) error { - if err := c.Set(capability.Agent, capability.DefaultAgent); err != nil { + if err := c.Set(capability.Agent, capability.DefaultAgent()); err != nil { return err } @@ -355,7 +355,7 @@ func (s *rpSession) reportStatus() *packp.ReportStatus { } func (*rpSession) setSupportedCapabilities(c *capability.List) error { - if err := c.Set(capability.Agent, capability.DefaultAgent); err != nil { + if err := c.Set(capability.Agent, capability.DefaultAgent()); err != nil { return err } diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index 351466954..9d3bcd359 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -10,10 +10,9 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/mitchellh/go-homedir" + "github.com/skeema/knownhosts" sshagent "github.com/xanzy/ssh-agent" "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/knownhosts" ) const DefaultUsername = "git" @@ -44,7 +43,6 @@ const ( type KeyboardInteractive struct { User string Challenge ssh.KeyboardInteractiveChallenge - HostKeyCallbackHelper } func (a *KeyboardInteractive) Name() string { @@ -56,19 +54,18 @@ func (a *KeyboardInteractive) String() string { } func (a *KeyboardInteractive) ClientConfig() (*ssh.ClientConfig, error) { - return a.SetHostKeyCallback(&ssh.ClientConfig{ + return &ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ a.Challenge, }, - }) + }, nil } // Password implements AuthMethod by using the given password. type Password struct { User string Password string - HostKeyCallbackHelper } func (a *Password) Name() string { @@ -80,10 +77,10 @@ func (a *Password) String() string { } func (a *Password) ClientConfig() (*ssh.ClientConfig, error) { - return a.SetHostKeyCallback(&ssh.ClientConfig{ + return &ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.Password(a.Password)}, - }) + }, nil } // PasswordCallback implements AuthMethod by using a callback @@ -91,7 +88,6 @@ func (a *Password) ClientConfig() (*ssh.ClientConfig, error) { type PasswordCallback struct { User string Callback func() (pass string, err error) - HostKeyCallbackHelper } func (a *PasswordCallback) Name() string { @@ -103,17 +99,16 @@ func (a *PasswordCallback) String() string { } func (a *PasswordCallback) ClientConfig() (*ssh.ClientConfig, error) { - return a.SetHostKeyCallback(&ssh.ClientConfig{ + return &ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.PasswordCallback(a.Callback)}, - }) + }, nil } // PublicKeys implements AuthMethod by using the given key pairs. type PublicKeys struct { User string Signer ssh.Signer - HostKeyCallbackHelper } // NewPublicKeys returns a PublicKeys from a PEM encoded private key. An @@ -152,10 +147,10 @@ func (a *PublicKeys) String() string { } func (a *PublicKeys) ClientConfig() (*ssh.ClientConfig, error) { - return a.SetHostKeyCallback(&ssh.ClientConfig{ + return &ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.PublicKeys(a.Signer)}, - }) + }, nil } func username() (string, error) { @@ -178,7 +173,6 @@ func username() (string, error) { type PublicKeysCallback struct { User string Callback func() (signers []ssh.Signer, err error) - HostKeyCallbackHelper } // NewSSHAgentAuth returns a PublicKeysCallback based on a SSH agent, it opens @@ -213,10 +207,10 @@ func (a *PublicKeysCallback) String() string { } func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { - return a.SetHostKeyCallback(&ssh.ClientConfig{ + return &ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.PublicKeysCallback(a.Callback)}, - }) + }, nil } // NewKnownHostsCallback returns ssh.HostKeyCallback based on a file based on a @@ -224,12 +218,19 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // // If list of files is empty, then it will be read from the SSH_KNOWN_HOSTS // environment variable, example: -// /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file +// +// /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file // // If SSH_KNOWN_HOSTS is not set the following file locations will be used: -// ~/.ssh/known_hosts -// /etc/ssh/ssh_known_hosts +// +// ~/.ssh/known_hosts +// /etc/ssh/ssh_known_hosts func NewKnownHostsCallback(files ...string) (ssh.HostKeyCallback, error) { + kh, err := newKnownHosts(files...) + return ssh.HostKeyCallback(kh), err +} + +func newKnownHosts(files ...string) (knownhosts.HostKeyCallback, error) { var err error if len(files) == 0 { @@ -251,7 +252,7 @@ func getDefaultKnownHostsFiles() ([]string, error) { return files, nil } - homeDirPath, err := homedir.Dir() + homeDirPath, err := os.UserHomeDir() if err != nil { return nil, err } @@ -285,6 +286,9 @@ func filterKnownHostsFiles(files ...string) ([]string, error) { // HostKeyCallbackHelper is a helper that provides common functionality to // configure HostKeyCallback into a ssh.ClientConfig. +// Deprecated in favor of SetConfigHostKeyFields (see common.go) which provides +// a mechanism for also setting ClientConfig.HostKeyAlgorithms for a specific +// host. type HostKeyCallbackHelper struct { // HostKeyCallback is the function type used for verifying server keys. // If nil default callback will be create using NewKnownHostsCallback diff --git a/plumbing/transport/ssh/common.go b/plumbing/transport/ssh/common.go index 46e79134c..4b9ac0797 100644 --- a/plumbing/transport/ssh/common.go +++ b/plumbing/transport/ssh/common.go @@ -121,10 +121,15 @@ func (c *command) connect() error { if err != nil { return err } + hostWithPort := c.getHostWithPort() + config, err = SetConfigHostKeyFields(config, hostWithPort) + if err != nil { + return err + } overrideConfig(c.config, config) - c.client, err = dial("tcp", c.getHostWithPort(), config) + c.client, err = dial("tcp", hostWithPort, config) if err != nil { return err } @@ -162,6 +167,23 @@ func dial(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { return ssh.NewClient(c, chans, reqs), nil } +// SetConfigHostKeyFields sets cfg.HostKeyCallback and cfg.HostKeyAlgorithms +// based on OpenSSH known_hosts. cfg is modified in-place. hostWithPort must be +// supplied, since the algorithms will be set based on the known host keys for +// that specific host. Otherwise, golang.org/x/crypto/ssh can return an error +// upon connecting to a host whose *first* key is not known, even though other +// keys (of different types) are known and match properly. +// For background see https://github.com/go-git/go-git/issues/411 as well as +// https://github.com/golang/go/issues/29286 for root cause. +func SetConfigHostKeyFields(cfg *ssh.ClientConfig, hostWithPort string) (*ssh.ClientConfig, error) { + kh, err := newKnownHosts() + if err == nil { + cfg.HostKeyCallback = kh.HostKeyCallback() + cfg.HostKeyAlgorithms = kh.HostKeyAlgorithms(hostWithPort) + } + return cfg, err +} + func (c *command) getHostWithPort() string { if addr, found := c.doGetHostWithPortFromSSHConfig(); found { return addr diff --git a/plumbing/transport/ssh/common_test.go b/plumbing/transport/ssh/common_test.go index e04a9c5dc..6d634d532 100644 --- a/plumbing/transport/ssh/common_test.go +++ b/plumbing/transport/ssh/common_test.go @@ -7,7 +7,6 @@ import ( "github.com/kevinburke/ssh_config" "golang.org/x/crypto/ssh" - stdssh "golang.org/x/crypto/ssh" . "gopkg.in/check.v1" ) @@ -99,7 +98,7 @@ func (s *SuiteCommon) TestIssue70(c *C) { uploadPack.SetUpSuite(c) config := &ssh.ClientConfig{ - HostKeyCallback: stdssh.InsecureIgnoreHostKey(), + HostKeyCallback: ssh.InsecureIgnoreHostKey(), } r := &runner{ config: config, diff --git a/prune.go b/prune.go index cc5907a14..8e35b994e 100644 --- a/prune.go +++ b/prune.go @@ -17,7 +17,7 @@ type PruneOptions struct { Handler PruneHandler } -var ErrLooseObjectsNotSupported = errors.New("Loose objects not supported") +var ErrLooseObjectsNotSupported = errors.New("loose objects not supported") // DeleteObject deletes an object from a repository. // The type conveniently matches PruneHandler. diff --git a/remote.go b/remote.go index 4a06106c5..db78ae718 100644 --- a/remote.go +++ b/remote.go @@ -5,10 +5,12 @@ import ( "errors" "fmt" "io" + "strings" "time" "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/internal/url" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/format/packfile" @@ -103,7 +105,11 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) (err error) { return fmt.Errorf("remote names don't match: %s != %s", o.RemoteName, r.c.Name) } - s, err := newSendPackSession(r.c.URLs[0], o.Auth, o.InsecureSkipTLS, o.CABundle) + if o.RemoteURL == "" { + o.RemoteURL = r.c.URLs[0] + } + + s, err := newSendPackSession(o.RemoteURL, o.Auth, o.InsecureSkipTLS, o.CABundle) if err != nil { return err } @@ -183,12 +189,12 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) (err error) { var hashesToPush []plumbing.Hash // Avoid the expensive revlist operation if we're only doing deletes. if !allDelete { - if r.c.IsFirstURLLocal() { + if url.IsLocalEndpoint(o.RemoteURL) { // If we're are pushing to a local repo, it might be much // faster to use a local storage layer to get the commits // to ignore, when calculating the object revlist. localStorer := filesystem.NewStorage( - osfs.New(r.c.URLs[0]), cache.NewObjectLRUDefault()) + osfs.New(o.RemoteURL), cache.NewObjectLRUDefault()) hashesToPush, err = revlist.ObjectsWithStorageForIgnores( r.s, localStorer, objects, haves) } else { @@ -225,6 +231,74 @@ func (r *Remote) useRefDeltas(ar *packp.AdvRefs) bool { return !ar.Capabilities.Supports(capability.OFSDelta) } +func (r *Remote) addReachableTags(localRefs []*plumbing.Reference, remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest) error { + tags := make(map[plumbing.Reference]struct{}) + // get a list of all tags locally + for _, ref := range localRefs { + if strings.HasPrefix(string(ref.Name()), "refs/tags") { + tags[*ref] = struct{}{} + } + } + + remoteRefIter, err := remoteRefs.IterReferences() + if err != nil { + return err + } + + // remove any that are already on the remote + if err := remoteRefIter.ForEach(func(reference *plumbing.Reference) error { + delete(tags, *reference) + return nil + }); err != nil { + return err + } + + for tag := range tags { + tagObject, err := object.GetObject(r.s, tag.Hash()) + var tagCommit *object.Commit + if err != nil { + return fmt.Errorf("get tag object: %w", err) + } + + if tagObject.Type() != plumbing.TagObject { + continue + } + + annotatedTag, ok := tagObject.(*object.Tag) + if !ok { + return errors.New("could not get annotated tag object") + } + + tagCommit, err = object.GetCommit(r.s, annotatedTag.Target) + if err != nil { + return fmt.Errorf("get annotated tag commit: %w", err) + } + + // only include tags that are reachable from one of the refs + // already being pushed + for _, cmd := range req.Commands { + if tag.Name() == cmd.Name { + continue + } + + if strings.HasPrefix(cmd.Name.String(), "refs/tags") { + continue + } + + c, err := object.GetCommit(r.s, cmd.New) + if err != nil { + return fmt.Errorf("get commit %v: %w", cmd.Name, err) + } + + if isAncestor, err := tagCommit.IsAncestor(c); err == nil && isAncestor { + req.Commands = append(req.Commands, &packp.Command{Name: tag.Name(), New: tag.Hash()}) + } + } + } + + return nil +} + func (r *Remote) newReferenceUpdateRequest( o *PushOptions, localRefs []*plumbing.Reference, @@ -242,10 +316,28 @@ func (r *Remote) newReferenceUpdateRequest( } } - if err := r.addReferencesToUpdate(o.RefSpecs, localRefs, remoteRefs, req, o.Prune); err != nil { + if ar.Capabilities.Supports(capability.PushOptions) { + _ = req.Capabilities.Set(capability.PushOptions) + for k, v := range o.Options { + req.Options = append(req.Options, &packp.Option{Key: k, Value: v}) + } + } + + if o.Atomic && ar.Capabilities.Supports(capability.Atomic) { + _ = req.Capabilities.Set(capability.Atomic) + } + + if err := r.addReferencesToUpdate(o.RefSpecs, localRefs, remoteRefs, req, o.Prune, o.ForceWithLease); err != nil { + return nil, err } + if o.FollowTags { + if err := r.addReachableTags(localRefs, remoteRefs, req); err != nil { + return nil, err + } + } + return req, nil } @@ -314,7 +406,11 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (sto storer.Referen o.RefSpecs = r.c.Fetch } - s, err := newUploadPackSession(r.c.URLs[0], o.Auth, o.InsecureSkipTLS, o.CABundle) + if o.RemoteURL == "" { + o.RemoteURL = r.c.URLs[0] + } + + s, err := newUploadPackSession(o.RemoteURL, o.Auth, o.InsecureSkipTLS, o.CABundle) if err != nil { return nil, err } @@ -474,6 +570,7 @@ func (r *Remote) addReferencesToUpdate( remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest, prune bool, + forceWithLease *ForceWithLease, ) error { // This references dictionary will be used to search references by name. refsDict := make(map[string]*plumbing.Reference) @@ -487,7 +584,7 @@ func (r *Remote) addReferencesToUpdate( return err } } else { - err := r.addOrUpdateReferences(rs, localRefs, refsDict, remoteRefs, req) + err := r.addOrUpdateReferences(rs, localRefs, refsDict, remoteRefs, req, forceWithLease) if err != nil { return err } @@ -509,20 +606,25 @@ func (r *Remote) addOrUpdateReferences( refsDict map[string]*plumbing.Reference, remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest, + forceWithLease *ForceWithLease, ) error { // If it is not a wilcard refspec we can directly search for the reference // in the references dictionary. if !rs.IsWildcard() { ref, ok := refsDict[rs.Src()] if !ok { + commit, err := object.GetCommit(r.s, plumbing.NewHash(rs.Src())) + if err == nil { + return r.addCommit(rs, remoteRefs, commit.Hash, req) + } return nil } - return r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req) + return r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req, forceWithLease) } for _, ref := range localRefs { - err := r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req) + err := r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req, forceWithLease) if err != nil { return err } @@ -569,9 +671,46 @@ func (r *Remote) deleteReferences(rs config.RefSpec, }) } +func (r *Remote) addCommit(rs config.RefSpec, + remoteRefs storer.ReferenceStorer, localCommit plumbing.Hash, + req *packp.ReferenceUpdateRequest) error { + + if rs.IsWildcard() { + return errors.New("can't use wildcard together with hash refspecs") + } + + cmd := &packp.Command{ + Name: rs.Dst(""), + Old: plumbing.ZeroHash, + New: localCommit, + } + remoteRef, err := remoteRefs.Reference(cmd.Name) + if err == nil { + if remoteRef.Type() != plumbing.HashReference { + //TODO: check actual git behavior here + return nil + } + + cmd.Old = remoteRef.Hash() + } else if err != plumbing.ErrReferenceNotFound { + return err + } + if cmd.Old == cmd.New { + return nil + } + if !rs.IsForceUpdate() { + if err := checkFastForwardUpdate(r.s, remoteRefs, cmd); err != nil { + return err + } + } + + req.Commands = append(req.Commands, cmd) + return nil +} + func (r *Remote) addReferenceIfRefSpecMatches(rs config.RefSpec, remoteRefs storer.ReferenceStorer, localRef *plumbing.Reference, - req *packp.ReferenceUpdateRequest) error { + req *packp.ReferenceUpdateRequest, forceWithLease *ForceWithLease) error { if localRef.Type() != plumbing.HashReference { return nil @@ -603,7 +742,11 @@ func (r *Remote) addReferenceIfRefSpecMatches(rs config.RefSpec, return nil } - if !rs.IsForceUpdate() { + if forceWithLease != nil { + if err = r.checkForceWithLease(localRef, cmd, forceWithLease); err != nil { + return err + } + } else if !rs.IsForceUpdate() { if err := checkFastForwardUpdate(r.s, remoteRefs, cmd); err != nil { return err } @@ -613,6 +756,31 @@ func (r *Remote) addReferenceIfRefSpecMatches(rs config.RefSpec, return nil } +func (r *Remote) checkForceWithLease(localRef *plumbing.Reference, cmd *packp.Command, forceWithLease *ForceWithLease) error { + remotePrefix := fmt.Sprintf("refs/remotes/%s/", r.Config().Name) + + ref, err := storer.ResolveReference( + r.s, + plumbing.ReferenceName(remotePrefix+strings.Replace(localRef.Name().String(), "refs/heads/", "", -1))) + if err != nil { + return err + } + + if forceWithLease.RefName.String() == "" || (forceWithLease.RefName == cmd.Name) { + expectedOID := ref.Hash() + + if !forceWithLease.Hash.IsZero() { + expectedOID = forceWithLease.Hash + } + + if cmd.Old != expectedOID { + return fmt.Errorf("non-fast-forward update: %s", cmd.Name.String()) + } + } + + return nil +} + func (r *Remote) references() ([]*plumbing.Reference, error) { var localRefs []*plumbing.Reference diff --git a/remote_test.go b/remote_test.go index 1efc9da70..751c89a1b 100644 --- a/remote_test.go +++ b/remote_test.go @@ -5,12 +5,16 @@ import ( "context" "errors" "io" + "io/ioutil" + "os" + "path/filepath" "runtime" "time" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/protocol/packp" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" "github.com/go-git/go-git/v5/plumbing/storer" @@ -46,6 +50,12 @@ func (s *RemoteSuite) TestFetchInvalidSchemaEndpoint(c *C) { c.Assert(err, ErrorMatches, ".*unsupported scheme.*") } +func (s *RemoteSuite) TestFetchOverriddenEndpoint(c *C) { + r := NewRemote(nil, &config.RemoteConfig{Name: "foo", URLs: []string{"http://perfectly-valid-url.example.com"}}) + err := r.Fetch(&FetchOptions{RemoteURL: "http://\\"}) + c.Assert(err, ErrorMatches, ".*invalid character.*") +} + func (s *RemoteSuite) TestFetchInvalidFetchOptions(c *C) { r := NewRemote(nil, &config.RemoteConfig{Name: "foo", URLs: []string{"qux://foo"}}) invalid := config.RefSpec("^*$ñ") @@ -525,10 +535,22 @@ func (s *RemoteSuite) TestPushContext(c *C) { }) c.Assert(err, IsNil) - // let the goroutine from pushHashes finish and check that the number of - // goroutines is the same as before - time.Sleep(100 * time.Millisecond) - c.Assert(runtime.NumGoroutine(), Equals, numGoroutines) + eventually(c, func() bool { + return runtime.NumGoroutine() <= numGoroutines + }) +} + +func eventually(c *C, condition func() bool) { + select { + case <-time.After(5 * time.Second): + default: + if condition() { + break + } + time.Sleep(100 * time.Millisecond) + } + + c.Assert(condition(), Equals, true) } func (s *RemoteSuite) TestPushContextCanceled(c *C) { @@ -556,10 +578,9 @@ func (s *RemoteSuite) TestPushContextCanceled(c *C) { }) c.Assert(err, Equals, context.Canceled) - // let the goroutine from pushHashes finish and check that the number of - // goroutines is the same as before - time.Sleep(100 * time.Millisecond) - c.Assert(runtime.NumGoroutine(), Equals, numGoroutines) + eventually(c, func() bool { + return runtime.NumGoroutine() <= numGoroutines + }) } func (s *RemoteSuite) TestPushTags(c *C) { @@ -591,6 +612,66 @@ func (s *RemoteSuite) TestPushTags(c *C) { }) } +func (s *RemoteSuite) TestPushFollowTags(c *C) { + url, clean := s.TemporalDir() + defer clean() + + server, err := PlainInit(url, true) + c.Assert(err, IsNil) + + fs := fixtures.ByURL("https://github.com/git-fixtures/basic.git").One().DotGit() + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + + r := NewRemote(sto, &config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{url}, + }) + + localRepo := newRepository(sto, fs) + tipTag, err := localRepo.CreateTag( + "tip", + plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"), + &CreateTagOptions{ + Message: "an annotated tag", + }, + ) + c.Assert(err, IsNil) + + initialTag, err := localRepo.CreateTag( + "initial-commit", + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), + &CreateTagOptions{ + Message: "a tag for the initial commit", + }, + ) + c.Assert(err, IsNil) + + _, err = localRepo.CreateTag( + "master-tag", + plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + &CreateTagOptions{ + Message: "a tag with a commit not reachable from branch", + }, + ) + c.Assert(err, IsNil) + + err = r.Push(&PushOptions{ + RefSpecs: []config.RefSpec{"+refs/heads/branch:refs/heads/branch"}, + FollowTags: true, + }) + c.Assert(err, IsNil) + + AssertReferences(c, server, map[string]string{ + "refs/heads/branch": "e8d3ffab552895c19b9fcf7aa264d277cde33881", + "refs/tags/tip": tipTag.Hash().String(), + "refs/tags/initial-commit": initialTag.Hash().String(), + }) + + AssertReferencesMissing(c, server, []string{ + "refs/tags/master-tag", + }) +} + func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { fs := fixtures.Basic().One().DotGit() sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) @@ -746,6 +827,133 @@ func (s *RemoteSuite) TestPushForceWithOption(c *C) { c.Assert(newRef, Not(DeepEquals), oldRef) } +func (s *RemoteSuite) TestPushForceWithLease_success(c *C) { + testCases := []struct { + desc string + forceWithLease ForceWithLease + }{ + { + desc: "no arguments", + forceWithLease: ForceWithLease{}, + }, + { + desc: "ref name", + forceWithLease: ForceWithLease{ + RefName: plumbing.ReferenceName("refs/heads/branch"), + }, + }, + { + desc: "ref name and sha", + forceWithLease: ForceWithLease{ + RefName: plumbing.ReferenceName("refs/heads/branch"), + Hash: plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"), + }, + }, + } + + for _, tc := range testCases { + c.Log("Executing test cases:", tc.desc) + + f := fixtures.Basic().One() + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) + dstFs := f.DotGit() + dstSto := filesystem.NewStorage(dstFs, cache.NewObjectLRUDefault()) + + newCommit := plumbing.NewHashReference( + "refs/heads/branch", plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), + ) + c.Assert(sto.SetReference(newCommit), IsNil) + + ref, err := sto.Reference("refs/heads/branch") + c.Log(ref.String()) + + url := dstFs.Root() + r := NewRemote(sto, &config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{url}, + }) + + oldRef, err := dstSto.Reference("refs/heads/branch") + c.Assert(err, IsNil) + c.Assert(oldRef, NotNil) + + c.Assert(r.Push(&PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/branch:refs/heads/branch"}, + ForceWithLease: &ForceWithLease{}, + }), IsNil) + + newRef, err := dstSto.Reference("refs/heads/branch") + c.Assert(err, IsNil) + c.Assert(newRef, DeepEquals, newCommit) + } +} + +func (s *RemoteSuite) TestPushForceWithLease_failure(c *C) { + testCases := []struct { + desc string + forceWithLease ForceWithLease + }{ + { + desc: "no arguments", + forceWithLease: ForceWithLease{}, + }, + { + desc: "ref name", + forceWithLease: ForceWithLease{ + RefName: plumbing.ReferenceName("refs/heads/branch"), + }, + }, + { + desc: "ref name and sha", + forceWithLease: ForceWithLease{ + RefName: plumbing.ReferenceName("refs/heads/branch"), + Hash: plumbing.NewHash("152175bf7e5580299fa1f0ba41ef6474cc043b70"), + }, + }, + } + + for _, tc := range testCases { + c.Log("Executing test cases:", tc.desc) + + f := fixtures.Basic().One() + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) + c.Assert(sto.SetReference( + plumbing.NewHashReference( + "refs/heads/branch", plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), + ), + ), IsNil) + + dstFs := f.DotGit() + dstSto := filesystem.NewStorage(dstFs, cache.NewObjectLRUDefault()) + c.Assert(dstSto.SetReference( + plumbing.NewHashReference( + "refs/heads/branch", plumbing.NewHash("ad7897c0fb8e7d9a9ba41fa66072cf06095a6cfc"), + ), + ), IsNil) + + url := dstFs.Root() + r := NewRemote(sto, &config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{url}, + }) + + oldRef, err := dstSto.Reference("refs/heads/branch") + c.Assert(err, IsNil) + c.Assert(oldRef, NotNil) + + err = r.Push(&PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/branch:refs/heads/branch"}, + ForceWithLease: &ForceWithLease{}, + }) + + c.Assert(err, DeepEquals, errors.New("non-fast-forward update: refs/heads/branch")) + + newRef, err := dstSto.Reference("refs/heads/branch") + c.Assert(err, IsNil) + c.Assert(newRef, Not(DeepEquals), plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9")) + } +} + func (s *RemoteSuite) TestPushPrune(c *C) { fs := fixtures.Basic().One().DotGit() @@ -805,7 +1013,7 @@ func (s *RemoteSuite) TestPushPrune(c *C) { "refs/remotes/origin/master": ref.Hash().String(), }) - ref, err = server.Reference(plumbing.ReferenceName("refs/tags/v1.0.0"), true) + _, err = server.Reference(plumbing.ReferenceName("refs/tags/v1.0.0"), true) c.Assert(err, Equals, plumbing.ErrReferenceNotFound) } @@ -903,6 +1111,12 @@ func (s *RemoteSuite) TestPushNonExistentEndpoint(c *C) { c.Assert(err, NotNil) } +func (s *RemoteSuite) TestPushOverriddenEndpoint(c *C) { + r := NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{"http://perfectly-valid-url.example.com"}}) + err := r.Push(&PushOptions{RemoteURL: "http://\\"}) + c.Assert(err, ErrorMatches, ".*invalid character.*") +} + func (s *RemoteSuite) TestPushInvalidSchemaEndpoint(c *C) { r := NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{"qux://foo"}}) err := r.Push(&PushOptions{}) @@ -1134,3 +1348,91 @@ func (s *RemoteSuite) TestPushRequireRemoteRefs(c *C) { c.Assert(err, IsNil) c.Assert(newRef, Not(DeepEquals), oldRef) } + +func (s *RemoteSuite) TestCanPushShasToReference(c *C) { + d, err := ioutil.TempDir("", "TestCanPushShasToReference") + c.Assert(err, IsNil) + if err != nil { + return + } + defer os.RemoveAll(d) + + // remote currently forces a plain path for path based remotes inside the PushContext function. + // This makes it impossible, in the current state to use memfs. + // For the sake of readability, use the same osFS everywhere and use plain git repositories on temporary files + remote, err := PlainInit(filepath.Join(d, "remote"), true) + c.Assert(err, IsNil) + c.Assert(remote, NotNil) + + repo, err := PlainInit(filepath.Join(d, "repo"), false) + c.Assert(err, IsNil) + c.Assert(repo, NotNil) + + fd, err := os.Create(filepath.Join(d, "repo", "README.md")) + c.Assert(err, IsNil) + if err != nil { + return + } + _, err = fd.WriteString("# test repo") + c.Assert(err, IsNil) + if err != nil { + return + } + err = fd.Close() + c.Assert(err, IsNil) + if err != nil { + return + } + + wt, err := repo.Worktree() + c.Assert(err, IsNil) + if err != nil { + return + } + + wt.Add("README.md") + sha, err := wt.Commit("test commit", &CommitOptions{ + Author: &object.Signature{ + Name: "test", + Email: "test@example.com", + When: time.Now(), + }, + Committer: &object.Signature{ + Name: "test", + Email: "test@example.com", + When: time.Now(), + }, + }) + c.Assert(err, IsNil) + if err != nil { + return + } + + gitremote, err := repo.CreateRemote(&config.RemoteConfig{ + Name: "local", + URLs: []string{filepath.Join(d, "remote")}, + }) + c.Assert(err, IsNil) + if err != nil { + return + } + + err = gitremote.Push(&PushOptions{ + RemoteName: "local", + RefSpecs: []config.RefSpec{ + // TODO: check with short hashes that this is still respected + config.RefSpec(sha.String() + ":refs/heads/branch"), + }, + }) + c.Assert(err, IsNil) + if err != nil { + return + } + + ref, err := remote.Reference(plumbing.ReferenceName("refs/heads/branch"), false) + c.Assert(err, IsNil) + if err != nil { + return + } + c.Assert(ref.Hash().String(), Equals, sha.String()) +} diff --git a/repository.go b/repository.go index d3fbf9759..7292df627 100644 --- a/repository.go +++ b/repository.go @@ -56,7 +56,7 @@ var ( ErrWorktreeNotProvided = errors.New("worktree should be provided") ErrIsBareRepository = errors.New("worktree not available in a bare repository") ErrUnableToResolveCommit = errors.New("unable to resolve commit") - ErrPackedObjectsNotSupported = errors.New("Packed objects not supported") + ErrPackedObjectsNotSupported = errors.New("packed objects not supported") ) // Repository represents a git repository @@ -280,6 +280,9 @@ func dotGitToOSFilesystems(path string, detect bool) (dot, wt billy.Filesystem, pathinfo, err := fs.Stat("/") if !os.IsNotExist(err) { + if pathinfo == nil { + return nil, nil, err + } if !pathinfo.IsDir() && detect { fs = osfs.New(filepath.Dir(path)) } @@ -1547,7 +1550,7 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err } if c == nil { - return &plumbing.ZeroHash, fmt.Errorf(`No commit message match regexp : "%s"`, re.String()) + return &plumbing.ZeroHash, fmt.Errorf("no commit message match regexp: %q", re.String()) } commit = c diff --git a/repository_test.go b/repository_test.go index 2bc5c902c..7a9db151d 100644 --- a/repository_test.go +++ b/repository_test.go @@ -210,6 +210,37 @@ func (s *RepositorySuite) TestCloneWithTags(c *C) { c.Assert(count, Equals, 3) } +func (s *RepositorySuite) TestCloneSparse(c *C) { + fs := memfs.New() + r, err := Clone(memory.NewStorage(), fs, &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + sparseCheckoutDirectories := []string{"go", "json", "php"} + c.Assert(w.Checkout(&CheckoutOptions{ + Branch: "refs/heads/master", + SparseCheckoutDirectories: sparseCheckoutDirectories, + }), IsNil) + + fis, err := fs.ReadDir(".") + c.Assert(err, IsNil) + for _, fi := range fis { + c.Assert(fi.IsDir(), Equals, true) + var oneOfSparseCheckoutDirs bool + + for _, sparseCheckoutDirectory := range sparseCheckoutDirectories { + if strings.HasPrefix(fi.Name(), sparseCheckoutDirectory) { + oneOfSparseCheckoutDirs = true + } + } + c.Assert(oneOfSparseCheckoutDirs, Equals, true) + } +} + func (s *RepositorySuite) TestCreateRemoteAndRemote(c *C) { r, _ := Init(memory.NewStorage(), nil) remote, err := r.CreateRemote(&config.RemoteConfig{ @@ -2756,7 +2787,7 @@ func (s *RepositorySuite) TestResolveRevisionWithErrors(c *C) { datas := map[string]string{ "efs/heads/master~": "reference not found", "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, - "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, + "HEAD^{/whatever}": `no commit message match regexp: "whatever"`, "4e1243bd22c66e76c2ba9eddc1f91394e57f9f83": "reference not found", } @@ -2917,6 +2948,11 @@ func (s *RepositorySuite) TestBrokenMultipleShallowFetch(c *C) { c.Assert(err, IsNil) } +func (s *RepositorySuite) TestDotGitToOSFilesystemsInvalidPath(c *C) { + _, _, err := dotGitToOSFilesystems("\000", false) + c.Assert(err, NotNil) +} + func BenchmarkObjects(b *testing.B) { defer fixtures.Clean() diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 4c2ae941c..a8f0eb754 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -3,6 +3,7 @@ package dotgit import ( "bufio" "encoding/hex" + "io" "io/ioutil" "os" "path/filepath" @@ -510,13 +511,13 @@ func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") // Move to an specific offset - pack.Seek(42, os.SEEK_SET) + pack.Seek(42, io.SeekStart) pack2, err := dir.ObjectPack(plumbing.NewHash(f.PackfileHash)) c.Assert(err, IsNil) // If the file is the same the offset should be the same - offset, err := pack2.Seek(0, os.SEEK_CUR) + offset, err := pack2.Seek(0, io.SeekCurrent) c.Assert(err, IsNil) c.Assert(offset, Equals, int64(42)) @@ -527,7 +528,7 @@ func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { c.Assert(err, IsNil) // If the file is opened again its offset should be 0 - offset, err = pack2.Seek(0, os.SEEK_CUR) + offset, err = pack2.Seek(0, io.SeekCurrent) c.Assert(err, IsNil) c.Assert(offset, Equals, int64(0)) @@ -653,7 +654,7 @@ func (s *SuiteDotGit) TestObject(c *C) { fs.MkdirAll(incomingDirPath, os.FileMode(0755)) fs.Create(incomingFilePath) - file, err = dir.Object(plumbing.NewHash(incomingHash)) + _, err = dir.Object(plumbing.NewHash(incomingHash)) c.Assert(err, IsNil) } diff --git a/storage/filesystem/dotgit/reader.go b/storage/filesystem/dotgit/reader.go new file mode 100644 index 000000000..975f92ac6 --- /dev/null +++ b/storage/filesystem/dotgit/reader.go @@ -0,0 +1,79 @@ +package dotgit + +import ( + "fmt" + "io" + "os" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/format/objfile" + "github.com/go-git/go-git/v5/utils/ioutil" +) + +var _ (plumbing.EncodedObject) = &EncodedObject{} + +type EncodedObject struct { + dir *DotGit + h plumbing.Hash + t plumbing.ObjectType + sz int64 +} + +func (e *EncodedObject) Hash() plumbing.Hash { + return e.h +} + +func (e *EncodedObject) Reader() (io.ReadCloser, error) { + f, err := e.dir.Object(e.h) + if err != nil { + if os.IsNotExist(err) { + return nil, plumbing.ErrObjectNotFound + } + + return nil, err + } + r, err := objfile.NewReader(f) + if err != nil { + return nil, err + } + + t, size, err := r.Header() + if err != nil { + _ = r.Close() + return nil, err + } + if t != e.t { + _ = r.Close() + return nil, objfile.ErrHeader + } + if size != e.sz { + _ = r.Close() + return nil, objfile.ErrHeader + } + return ioutil.NewReadCloserWithCloser(r, f.Close), nil +} + +func (e *EncodedObject) SetType(plumbing.ObjectType) {} + +func (e *EncodedObject) Type() plumbing.ObjectType { + return e.t +} + +func (e *EncodedObject) Size() int64 { + return e.sz +} + +func (e *EncodedObject) SetSize(int64) {} + +func (e *EncodedObject) Writer() (io.WriteCloser, error) { + return nil, fmt.Errorf("not supported") +} + +func NewEncodedObject(dir *DotGit, h plumbing.Hash, t plumbing.ObjectType, size int64) *EncodedObject { + return &EncodedObject{ + dir: dir, + h: h, + t: t, + sz: size, + } +} diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 0c25dad61..21667fa5a 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "os" + "sync" "time" "github.com/go-git/go-git/v5/plumbing" @@ -204,9 +205,9 @@ func (s *ObjectStorage) packfile(idx idxfile.Index, pack plumbing.Hash) (*packfi var p *packfile.Packfile if s.objectCache != nil { - p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache, s.options.LargeObjectThreshold) } else { - p = packfile.NewPackfile(idx, s.dir.Fs(), f) + p = packfile.NewPackfile(idx, s.dir.Fs(), f, s.options.LargeObjectThreshold) } return p, s.storePackfileInCache(pack, p) @@ -389,7 +390,6 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return cacheObj, nil } - obj = s.NewEncodedObject() r, err := objfile.NewReader(f) if err != nil { return nil, err @@ -402,6 +402,13 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return nil, err } + if s.options.LargeObjectThreshold > 0 && size > s.options.LargeObjectThreshold { + obj = dotgit.NewEncodedObject(s.dir, h, t, size) + return obj, nil + } + + obj = s.NewEncodedObject() + obj.SetType(t) obj.SetSize(size) w, err := obj.Writer() @@ -413,10 +420,21 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb s.objectCache.Put(obj) - _, err = io.Copy(w, r) + bufp := copyBufferPool.Get().(*[]byte) + buf := *bufp + _, err = io.CopyBuffer(w, r, buf) + copyBufferPool.Put(bufp) + return obj, err } +var copyBufferPool = sync.Pool{ + New: func() interface{} { + b := make([]byte, 32*1024) + return &b + }, +} + // Get returns the object with the given hash, by searching for it in // the packfile. func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( @@ -595,6 +613,7 @@ func (s *ObjectStorage) buildPackfileIters( return newPackfileIter( s.dir.Fs(), pack, t, seen, s.index[h], s.objectCache, s.options.KeepDescriptors, + s.options.LargeObjectThreshold, ) }, }, nil @@ -684,6 +703,7 @@ func NewPackfileIter( idxFile billy.File, t plumbing.ObjectType, keepPack bool, + largeObjectThreshold int64, ) (storer.EncodedObjectIter, error) { idx := idxfile.NewMemoryIndex() if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { @@ -695,7 +715,7 @@ func NewPackfileIter( } seen := make(map[plumbing.Hash]struct{}) - return newPackfileIter(fs, f, t, seen, idx, nil, keepPack) + return newPackfileIter(fs, f, t, seen, idx, nil, keepPack, largeObjectThreshold) } func newPackfileIter( @@ -706,12 +726,13 @@ func newPackfileIter( index idxfile.Index, cache cache.Object, keepPack bool, + largeObjectThreshold int64, ) (storer.EncodedObjectIter, error) { var p *packfile.Packfile if cache != nil { - p = packfile.NewPackfileWithCache(index, fs, f, cache) + p = packfile.NewPackfileWithCache(index, fs, f, cache, largeObjectThreshold) } else { - p = packfile.NewPackfile(index, fs, f) + p = packfile.NewPackfile(index, fs, f, largeObjectThreshold) } iter, err := p.GetByType(t) diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 22f5b0c2b..19a791464 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -71,7 +71,7 @@ func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { pack1, err := dg.ObjectPack(packfiles[0]) c.Assert(err, IsNil) - pack1.Seek(42, os.SEEK_SET) + pack1.Seek(42, io.SeekStart) err = o.Close() c.Assert(err, IsNil) @@ -79,7 +79,7 @@ func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { pack2, err := dg.ObjectPack(packfiles[0]) c.Assert(err, IsNil) - offset, err := pack2.Seek(0, os.SEEK_CUR) + offset, err := pack2.Seek(0, io.SeekCurrent) c.Assert(err, IsNil) c.Assert(offset, Equals, int64(0)) @@ -107,6 +107,27 @@ func (s *FsSuite) TestGetFromPackfileMaxOpenDescriptors(c *C) { c.Assert(err, IsNil) } +func (s *FsSuite) TestGetFromPackfileMaxOpenDescriptorsLargeObjectThreshold(c *C) { + fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() + o := NewObjectStorageWithOptions(dotgit.New(fs), cache.NewObjectLRUDefault(), Options{ + MaxOpenDescriptors: 1, + LargeObjectThreshold: 1, + }) + + expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") + obj, err := o.getFromPackfile(expected, false) + c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + + expected = plumbing.NewHash("e9cfa4c9ca160546efd7e8582ec77952a27b17db") + obj, err = o.getFromPackfile(expected, false) + c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + + err = o.Close() + c.Assert(err, IsNil) +} + func (s *FsSuite) TestGetSizeOfObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) @@ -160,6 +181,21 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { c.Assert(obj.Hash(), Equals, expected) } +func (s *FsSuite) TestGetFromPackfileMultiplePackfilesLargeObjectThreshold(c *C) { + fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() + o := NewObjectStorageWithOptions(dotgit.New(fs), cache.NewObjectLRUDefault(), Options{LargeObjectThreshold: 1}) + + expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") + obj, err := o.getFromPackfile(expected, false) + c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + + expected = plumbing.NewHash("e9cfa4c9ca160546efd7e8582ec77952a27b17db") + obj, err = o.getFromPackfile(expected, false) + c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) +} + func (s *FsSuite) TestIter(c *C) { fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() @@ -179,6 +215,25 @@ func (s *FsSuite) TestIter(c *C) { }) } +func (s *FsSuite) TestIterLargeObjectThreshold(c *C) { + fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + o := NewObjectStorageWithOptions(dotgit.New(fs), cache.NewObjectLRUDefault(), Options{LargeObjectThreshold: 1}) + + iter, err := o.IterEncodedObjects(plumbing.AnyObject) + c.Assert(err, IsNil) + + var count int32 + err = iter.ForEach(func(o plumbing.EncodedObject) error { + count++ + return nil + }) + + c.Assert(err, IsNil) + c.Assert(count, Equals, f.ObjectsCount) + }) +} + func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { for _, t := range objectTypes { @@ -215,7 +270,7 @@ func (s *FsSuite) TestPackfileIter(c *C) { idxf, err := dg.ObjectPackIdx(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(fs, f, idxf, t, false) + iter, err := NewPackfileIter(fs, f, idxf, t, false, 0) c.Assert(err, IsNil) err = iter.ForEach(func(o plumbing.EncodedObject) error { @@ -298,7 +353,7 @@ func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) { idxf, err := dg.ObjectPackIdx(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(fs, f, idxf, t, true) + iter, err := NewPackfileIter(fs, f, idxf, t, true, 0) c.Assert(err, IsNil) err = iter.ForEach(func(o plumbing.EncodedObject) error { @@ -331,7 +386,7 @@ func (s *FsSuite) TestGetFromObjectFileSharedCache(c *C) { c.Assert(err, IsNil) c.Assert(obj.Hash(), Equals, expected) - obj, err = o2.EncodedObject(plumbing.CommitObject, expected) + _, err = o2.EncodedObject(plumbing.CommitObject, expected) c.Assert(err, Equals, plumbing.ErrObjectNotFound) } @@ -377,7 +432,7 @@ func BenchmarkPackfileIter(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(fs, f, idxf, t, false) + iter, err := NewPackfileIter(fs, f, idxf, t, false, 0) if err != nil { b.Fatal(err) } @@ -425,7 +480,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(fs, f, idxf, t, false) + iter, err := NewPackfileIter(fs, f, idxf, t, false, 0) if err != nil { b.Fatal(err) } diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go index afb600cf2..ac48fdfbb 100644 --- a/storage/filesystem/shallow.go +++ b/storage/filesystem/shallow.go @@ -34,7 +34,7 @@ func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error { return err } -// Shallow return the shallow commits reading from shallo file from .git +// Shallow returns the shallow commits reading from shallo file from .git func (s *ShallowStorage) Shallow() ([]plumbing.Hash, error) { f, err := s.dir.Shallow() if f == nil || err != nil { diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 8b69b27b0..7e7a2c50f 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -34,6 +34,9 @@ type Options struct { // MaxOpenDescriptors is the max number of file descriptors to keep // open. If KeepDescriptors is true, all file descriptors will remain open. MaxOpenDescriptors int + // LargeObjectThreshold maximum object size (in bytes) that will be read in to memory. + // If left unset or set to 0 there is no limit + LargeObjectThreshold int64 } // NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. diff --git a/storage/memory/storage.go b/storage/memory/storage.go index a8e56697b..ef6a44551 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -193,7 +193,7 @@ func (o *ObjectStorage) DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) er return nil } -var errNotSupported = fmt.Errorf("Not supported") +var errNotSupported = fmt.Errorf("not supported") func (o *ObjectStorage) LooseObjectTime(hash plumbing.Hash) (time.Time, error) { return time.Time{}, errNotSupported diff --git a/storage/transactional/config_test.go b/storage/transactional/config_test.go index 1f3a572f4..34d7763f6 100644 --- a/storage/transactional/config_test.go +++ b/storage/transactional/config_test.go @@ -54,7 +54,7 @@ func (s *ConfigSuite) TestSetConfigTemporal(c *C) { cfg, err = cs.Config() c.Assert(err, IsNil) - c.Assert(temporalCfg.Core.Worktree, Equals, "bar") + c.Assert(cfg.Core.Worktree, Equals, "bar") } func (s *ConfigSuite) TestCommit(c *C) { diff --git a/storage/transactional/reference.go b/storage/transactional/reference.go index 3b009e2e6..1c0930755 100644 --- a/storage/transactional/reference.go +++ b/storage/transactional/reference.go @@ -15,9 +15,6 @@ type ReferenceStorage struct { // commit is requested, the entries are added when RemoveReference is called // and deleted if SetReference is called. deleted map[plumbing.ReferenceName]struct{} - // packRefs if true PackRefs is going to be called in the based storer when - // commit is called. - packRefs bool } // NewReferenceStorage returns a new ReferenceStorer based on a base storer and @@ -108,7 +105,6 @@ func (r ReferenceStorage) CountLooseRefs() (int, error) { // PackRefs honors the storer.ReferenceStorer interface. func (r ReferenceStorage) PackRefs() error { - r.packRefs = true return nil } diff --git a/utils/ioutil/common.go b/utils/ioutil/common.go index b52e85a38..b0ace4e62 100644 --- a/utils/ioutil/common.go +++ b/utils/ioutil/common.go @@ -55,6 +55,28 @@ func NewReadCloser(r io.Reader, c io.Closer) io.ReadCloser { return &readCloser{Reader: r, closer: c} } +type readCloserCloser struct { + io.ReadCloser + closer func() error +} + +func (r *readCloserCloser) Close() (err error) { + defer func() { + if err == nil { + err = r.closer() + return + } + _ = r.closer() + }() + return r.ReadCloser.Close() +} + +// NewReadCloserWithCloser creates an `io.ReadCloser` with the given `io.ReaderCloser` and +// `io.Closer` that ensures that the closer is closed on close +func NewReadCloserWithCloser(r io.ReadCloser, c func() error) io.ReadCloser { + return &readCloserCloser{ReadCloser: r, closer: c} +} + type writeCloser struct { io.Writer closer io.Closer @@ -82,6 +104,24 @@ func WriteNopCloser(w io.Writer) io.WriteCloser { return writeNopCloser{w} } +type readerAtAsReader struct { + io.ReaderAt + offset int64 +} + +func (r *readerAtAsReader) Read(bs []byte) (int, error) { + n, err := r.ReaderAt.ReadAt(bs, r.offset) + r.offset += int64(n) + return n, err +} + +func NewReaderUsingReaderAt(r io.ReaderAt, offset int64) io.Reader { + return &readerAtAsReader{ + ReaderAt: r, + offset: offset, + } +} + // CheckClose calls Close on the given io.Closer. If the given *error points to // nil, it will be assigned the error returned by Close. Otherwise, any error // returned by Close will be ignored. CheckClose is usually called with defer. diff --git a/utils/merkletrie/difftree.go b/utils/merkletrie/difftree.go index bd084b2ab..9f5145a26 100644 --- a/utils/merkletrie/difftree.go +++ b/utils/merkletrie/difftree.go @@ -304,13 +304,38 @@ func DiffTreeContext(ctx context.Context, fromTree, toTree noder.Noder, return nil, err } case onlyToRemains: - if err = ret.AddRecursiveInsert(to); err != nil { - return nil, err + if to.Skip() { + if err = ret.AddRecursiveDelete(to); err != nil { + return nil, err + } + } else { + if err = ret.AddRecursiveInsert(to); err != nil { + return nil, err + } } if err = ii.nextTo(); err != nil { return nil, err } case bothHaveNodes: + if from.Skip() { + if err = ret.AddRecursiveDelete(from); err != nil { + return nil, err + } + if err := ii.nextBoth(); err != nil { + return nil, err + } + break + } + if to.Skip() { + if err = ret.AddRecursiveDelete(to); err != nil { + return nil, err + } + if err := ii.nextBoth(); err != nil { + return nil, err + } + break + } + if err = diffNodes(&ret, ii); err != nil { return nil, err } diff --git a/utils/merkletrie/filesystem/node.go b/utils/merkletrie/filesystem/node.go index 2fc3d7a63..ad169ff4a 100644 --- a/utils/merkletrie/filesystem/node.go +++ b/utils/merkletrie/filesystem/node.go @@ -61,6 +61,10 @@ func (n *node) IsDir() bool { return n.isDir } +func (n *node) Skip() bool { + return false +} + func (n *node) Children() ([]noder.Noder, error) { if err := n.calculateChildren(); err != nil { return nil, err diff --git a/utils/merkletrie/index/node.go b/utils/merkletrie/index/node.go index d05b0c694..c1809f7ec 100644 --- a/utils/merkletrie/index/node.go +++ b/utils/merkletrie/index/node.go @@ -19,6 +19,7 @@ type node struct { entry *index.Entry children []noder.Noder isDir bool + skip bool } // NewRootNode returns the root node of a computed tree from a index.Index, @@ -39,7 +40,7 @@ func NewRootNode(idx *index.Index) noder.Noder { continue } - n := &node{path: fullpath} + n := &node{path: fullpath, skip: e.SkipWorktree} if fullpath == e.Name { n.entry = e } else { @@ -58,6 +59,10 @@ func (n *node) String() string { return n.path } +func (n *node) Skip() bool { + return n.skip +} + // Hash the hash of a filesystem is a 24-byte slice, is the result of // concatenating the computed plumbing.Hash of the file as a Blob and its // plumbing.FileMode; that way the difftree algorithm will detect changes in the diff --git a/utils/merkletrie/internal/fsnoder/dir.go b/utils/merkletrie/internal/fsnoder/dir.go index 20a2aeebb..3a4c2424e 100644 --- a/utils/merkletrie/internal/fsnoder/dir.go +++ b/utils/merkletrie/internal/fsnoder/dir.go @@ -112,6 +112,10 @@ func (d *dir) NumChildren() (int, error) { return len(d.children), nil } +func (d *dir) Skip() bool { + return false +} + const ( dirStartMark = '(' dirEndMark = ')' diff --git a/utils/merkletrie/internal/fsnoder/file.go b/utils/merkletrie/internal/fsnoder/file.go index d53643f1a..0bb908b7a 100644 --- a/utils/merkletrie/internal/fsnoder/file.go +++ b/utils/merkletrie/internal/fsnoder/file.go @@ -55,6 +55,10 @@ func (f *file) NumChildren() (int, error) { return 0, nil } +func (f *file) Skip() bool { + return false +} + const ( fileStartMark = '<' fileEndMark = '>' diff --git a/utils/merkletrie/noder/noder.go b/utils/merkletrie/noder/noder.go index d6b3de4ad..6d22b8c14 100644 --- a/utils/merkletrie/noder/noder.go +++ b/utils/merkletrie/noder/noder.go @@ -53,6 +53,7 @@ type Noder interface { // implement NumChildren in O(1) while Children is usually more // complex. NumChildren() (int, error) + Skip() bool } // NoChildren represents the children of a noder without children. diff --git a/utils/merkletrie/noder/noder_test.go b/utils/merkletrie/noder/noder_test.go index 5e014fe9b..c1af99843 100644 --- a/utils/merkletrie/noder/noder_test.go +++ b/utils/merkletrie/noder/noder_test.go @@ -25,6 +25,7 @@ func (n noderMock) Name() string { return n.name } func (n noderMock) IsDir() bool { return n.isDir } func (n noderMock) Children() ([]Noder, error) { return n.children, nil } func (n noderMock) NumChildren() (int, error) { return len(n.children), nil } +func (n noderMock) Skip() bool { return false } // Returns a sequence with the noders 3, 2, and 1 from the // following diagram: @@ -57,20 +58,6 @@ func childrenFixture() []Noder { return []Noder{c1, c2} } -// Returns the same as nodersFixture but sorted by name, this is: "1", -// "2" and then "3". -func sortedNodersFixture() []Noder { - n1 := &noderMock{ - name: "1", - hash: []byte{0x00, 0x01, 0x02}, - isDir: true, - children: childrenFixture(), - } - n2 := &noderMock{name: "2"} - n3 := &noderMock{name: "3"} - return []Noder{n1, n2, n3} // the same as nodersFixture but sorted by name -} - // returns nodersFixture as the path of "1". func pathFixture() Path { return Path(nodersFixture()) diff --git a/utils/merkletrie/noder/path.go b/utils/merkletrie/noder/path.go index 1c7ef54ee..6c1d36332 100644 --- a/utils/merkletrie/noder/path.go +++ b/utils/merkletrie/noder/path.go @@ -15,6 +15,14 @@ import ( // not be used. type Path []Noder +func (p Path) Skip() bool { + if len(p) > 0 { + return p.Last().Skip() + } + + return false +} + // String returns the full path of the final noder as a string, using // "/" as the separator. func (p Path) String() string { diff --git a/utils/sync/bufio.go b/utils/sync/bufio.go new file mode 100644 index 000000000..5009ea804 --- /dev/null +++ b/utils/sync/bufio.go @@ -0,0 +1,29 @@ +package sync + +import ( + "bufio" + "io" + "sync" +) + +var bufioReader = sync.Pool{ + New: func() interface{} { + return bufio.NewReader(nil) + }, +} + +// GetBufioReader returns a *bufio.Reader that is managed by a sync.Pool. +// Returns a bufio.Reader that is resetted with reader and ready for use. +// +// After use, the *bufio.Reader should be put back into the sync.Pool +// by calling PutBufioReader. +func GetBufioReader(reader io.Reader) *bufio.Reader { + r := bufioReader.Get().(*bufio.Reader) + r.Reset(reader) + return r +} + +// PutBufioReader puts reader back into its sync.Pool. +func PutBufioReader(reader *bufio.Reader) { + bufioReader.Put(reader) +} diff --git a/utils/sync/bufio_test.go b/utils/sync/bufio_test.go new file mode 100644 index 000000000..e70f3d803 --- /dev/null +++ b/utils/sync/bufio_test.go @@ -0,0 +1,26 @@ +package sync + +import ( + "io" + "strings" + "testing" +) + +func TestGetAndPutBufioReader(t *testing.T) { + wanted := "someinput" + r := GetBufioReader(strings.NewReader(wanted)) + if r == nil { + t.Error("nil was not expected") + } + + got, err := r.ReadString(0) + if err != nil && err != io.EOF { + t.Errorf("unexpected error reading string: %v", err) + } + + if wanted != got { + t.Errorf("wanted %q got %q", wanted, got) + } + + PutBufioReader(r) +} diff --git a/utils/sync/bytes.go b/utils/sync/bytes.go new file mode 100644 index 000000000..dd06fc0bc --- /dev/null +++ b/utils/sync/bytes.go @@ -0,0 +1,51 @@ +package sync + +import ( + "bytes" + "sync" +) + +var ( + byteSlice = sync.Pool{ + New: func() interface{} { + b := make([]byte, 16*1024) + return &b + }, + } + bytesBuffer = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(nil) + }, + } +) + +// GetByteSlice returns a *[]byte that is managed by a sync.Pool. +// The initial slice length will be 16384 (16kb). +// +// After use, the *[]byte should be put back into the sync.Pool +// by calling PutByteSlice. +func GetByteSlice() *[]byte { + buf := byteSlice.Get().(*[]byte) + return buf +} + +// PutByteSlice puts buf back into its sync.Pool. +func PutByteSlice(buf *[]byte) { + byteSlice.Put(buf) +} + +// GetBytesBuffer returns a *bytes.Buffer that is managed by a sync.Pool. +// Returns a buffer that is resetted and ready for use. +// +// After use, the *bytes.Buffer should be put back into the sync.Pool +// by calling PutBytesBuffer. +func GetBytesBuffer() *bytes.Buffer { + buf := bytesBuffer.Get().(*bytes.Buffer) + buf.Reset() + return buf +} + +// PutBytesBuffer puts buf back into its sync.Pool. +func PutBytesBuffer(buf *bytes.Buffer) { + bytesBuffer.Put(buf) +} diff --git a/utils/sync/bytes_test.go b/utils/sync/bytes_test.go new file mode 100644 index 000000000..b233429ce --- /dev/null +++ b/utils/sync/bytes_test.go @@ -0,0 +1,49 @@ +package sync + +import ( + "testing" +) + +func TestGetAndPutBytesBuffer(t *testing.T) { + buf := GetBytesBuffer() + if buf == nil { + t.Error("nil was not expected") + } + + initialLen := buf.Len() + buf.Grow(initialLen * 2) + grownLen := buf.Len() + + PutBytesBuffer(buf) + + buf = GetBytesBuffer() + if buf.Len() != grownLen { + t.Error("bytes buffer was not reused") + } + + buf2 := GetBytesBuffer() + if buf2.Len() != initialLen { + t.Errorf("new bytes buffer length: wanted %d got %d", initialLen, buf2.Len()) + } +} + +func TestGetAndPutByteSlice(t *testing.T) { + slice := GetByteSlice() + if slice == nil { + t.Error("nil was not expected") + } + + wanted := 16 * 1024 + got := len(*slice) + if wanted != got { + t.Errorf("byte slice length: wanted %d got %d", wanted, got) + } + + newByteSlice := make([]byte, wanted*2) + PutByteSlice(&newByteSlice) + + newSlice := GetByteSlice() + if len(*newSlice) != len(newByteSlice) { + t.Error("byte slice was not reused") + } +} diff --git a/utils/sync/zlib.go b/utils/sync/zlib.go new file mode 100644 index 000000000..c61388595 --- /dev/null +++ b/utils/sync/zlib.go @@ -0,0 +1,74 @@ +package sync + +import ( + "bytes" + "compress/zlib" + "io" + "sync" +) + +var ( + zlibInitBytes = []byte{0x78, 0x9c, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01} + zlibReader = sync.Pool{ + New: func() interface{} { + r, _ := zlib.NewReader(bytes.NewReader(zlibInitBytes)) + return ZLibReader{ + Reader: r.(zlibReadCloser), + } + }, + } + zlibWriter = sync.Pool{ + New: func() interface{} { + return zlib.NewWriter(nil) + }, + } +) + +type zlibReadCloser interface { + io.ReadCloser + zlib.Resetter +} + +type ZLibReader struct { + dict *[]byte + Reader zlibReadCloser +} + +// GetZlibReader returns a ZLibReader that is managed by a sync.Pool. +// Returns a ZLibReader that is resetted using a dictionary that is +// also managed by a sync.Pool. +// +// After use, the ZLibReader should be put back into the sync.Pool +// by calling PutZlibReader. +func GetZlibReader(r io.Reader) (ZLibReader, error) { + z := zlibReader.Get().(ZLibReader) + z.dict = GetByteSlice() + + err := z.Reader.Reset(r, *z.dict) + + return z, err +} + +// PutZlibReader puts z back into its sync.Pool, first closing the reader. +// The Byte slice dictionary is also put back into its sync.Pool. +func PutZlibReader(z ZLibReader) { + z.Reader.Close() + PutByteSlice(z.dict) + zlibReader.Put(z) +} + +// GetZlibWriter returns a *zlib.Writer that is managed by a sync.Pool. +// Returns a writer that is resetted with w and ready for use. +// +// After use, the *zlib.Writer should be put back into the sync.Pool +// by calling PutZlibWriter. +func GetZlibWriter(w io.Writer) *zlib.Writer { + z := zlibWriter.Get().(*zlib.Writer) + z.Reset(w) + return z +} + +// PutZlibWriter puts w back into its sync.Pool. +func PutZlibWriter(w *zlib.Writer) { + zlibWriter.Put(w) +} diff --git a/utils/sync/zlib_test.go b/utils/sync/zlib_test.go new file mode 100644 index 000000000..b736fb221 --- /dev/null +++ b/utils/sync/zlib_test.go @@ -0,0 +1,74 @@ +package sync + +import ( + "bytes" + "compress/zlib" + "io" + "testing" +) + +func TestGetAndPutZlibReader(t *testing.T) { + _, err := GetZlibReader(bytes.NewReader(zlibInitBytes)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + dict := &[]byte{} + reader := FakeZLibReader{} + PutZlibReader(ZLibReader{dict: dict, Reader: &reader}) + + if !reader.wasClosed { + t.Errorf("reader was not closed") + } + + z2, err := GetZlibReader(bytes.NewReader(zlibInitBytes)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if dict != z2.dict { + t.Errorf("zlib dictionary was not reused") + } + + if &reader != z2.Reader { + t.Errorf("zlib reader was not reused") + } + + if !reader.wasReset { + t.Errorf("reader was not reset") + } +} + +func TestGetAndPutZlibWriter(t *testing.T) { + w := GetZlibWriter(nil) + if w == nil { + t.Errorf("nil was not expected") + } + + newW := zlib.NewWriter(nil) + PutZlibWriter(newW) + + w2 := GetZlibWriter(nil) + if w2 != newW { + t.Errorf("zlib writer was not reused") + } +} + +type FakeZLibReader struct { + wasClosed bool + wasReset bool +} + +func (f *FakeZLibReader) Reset(r io.Reader, dict []byte) error { + f.wasReset = true + return nil +} + +func (f *FakeZLibReader) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (f *FakeZLibReader) Close() error { + f.wasClosed = true + return nil +} diff --git a/worktree.go b/worktree.go index f23d9f170..02f90a9e6 100644 --- a/worktree.go +++ b/worktree.go @@ -9,8 +9,9 @@ import ( "os" "path/filepath" "strings" - "sync" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/util" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" @@ -20,9 +21,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" "github.com/go-git/go-git/v5/utils/merkletrie" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/util" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -73,6 +72,7 @@ func (w *Worktree) PullContext(ctx context.Context, o *PullOptions) error { fetchHead, err := remote.fetch(ctx, &FetchOptions{ RemoteName: o.RemoteName, + RemoteURL: o.RemoteURL, Depth: o.Depth, Auth: o.Auth, Progress: o.Progress, @@ -182,6 +182,10 @@ func (w *Worktree) Checkout(opts *CheckoutOptions) error { return err } + if len(opts.SparseCheckoutDirectories) > 0 { + return w.ResetSparsely(ro, opts.SparseCheckoutDirectories) + } + return w.Reset(ro) } func (w *Worktree) createBranch(opts *CheckoutOptions) error { @@ -262,8 +266,7 @@ func (w *Worktree) setHEADToBranch(branch plumbing.ReferenceName, commit plumbin return w.r.Storer.SetReference(head) } -// Reset the worktree to a specified state. -func (w *Worktree) Reset(opts *ResetOptions) error { +func (w *Worktree) ResetSparsely(opts *ResetOptions, dirs []string) error { if err := opts.Validate(w.r); err != nil { return err } @@ -293,7 +296,7 @@ func (w *Worktree) Reset(opts *ResetOptions) error { } if opts.Mode == MixedReset || opts.Mode == MergeReset || opts.Mode == HardReset { - if err := w.resetIndex(t); err != nil { + if err := w.resetIndex(t, dirs); err != nil { return err } } @@ -307,8 +310,17 @@ func (w *Worktree) Reset(opts *ResetOptions) error { return nil } -func (w *Worktree) resetIndex(t *object.Tree) error { +// Reset the worktree to a specified state. +func (w *Worktree) Reset(opts *ResetOptions) error { + return w.ResetSparsely(opts, nil) +} + +func (w *Worktree) resetIndex(t *object.Tree, dirs []string) error { idx, err := w.r.Storer.Index() + if len(dirs) > 0 { + idx.SkipUnless(dirs) + } + if err != nil { return err } @@ -520,12 +532,6 @@ func (w *Worktree) checkoutChangeRegularFile(name string, return nil } -var copyBufferPool = sync.Pool{ - New: func() interface{} { - return make([]byte, 32*1024) - }, -} - func (w *Worktree) checkoutFile(f *object.File) (err error) { mode, err := f.Mode.ToOSFileMode() if err != nil { @@ -549,9 +555,9 @@ func (w *Worktree) checkoutFile(f *object.File) (err error) { } defer ioutil.CheckClose(to, &err) - buf := copyBufferPool.Get().([]byte) - _, err = io.CopyBuffer(to, from, buf) - copyBufferPool.Put(buf) + buf := sync.GetByteSlice() + _, err = io.CopyBuffer(to, from, *buf) + sync.PutByteSlice(buf) return } diff --git a/worktree_bsd.go b/worktree_bsd.go index d4ea32758..d4682eb83 100644 --- a/worktree_bsd.go +++ b/worktree_bsd.go @@ -12,7 +12,7 @@ import ( func init() { fillSystemInfo = func(e *index.Entry, sys interface{}) { if os, ok := sys.(*syscall.Stat_t); ok { - e.CreatedAt = time.Unix(int64(os.Atimespec.Sec), int64(os.Atimespec.Nsec)) + e.CreatedAt = time.Unix(os.Atimespec.Unix()) e.Dev = uint32(os.Dev) e.Inode = uint32(os.Ino) e.GID = os.Gid diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 65d4b695d..097c6e5d3 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -212,10 +212,10 @@ func (s *WorktreeSuite) TestCommitTreeSort(c *C) { defer clean() st := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) - r, err := Init(st, nil) + _, err := Init(st, nil) c.Assert(err, IsNil) - r, _ = Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ + r, _ := Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ URL: fs.Root(), }) @@ -296,6 +296,7 @@ func (s *WorktreeSuite) TestJustStoreObjectsNotAlreadyStored(c *C) { All: true, Author: defaultSignature(), }) + c.Assert(err, IsNil) c.Assert(hash, Equals, plumbing.NewHash("97c0c5177e6ac57d10e8ea0017f2d39b91e2b364")) // Step 3: Check diff --git a/worktree_linux.go b/worktree_linux.go index cf0db2524..6fcace2f9 100644 --- a/worktree_linux.go +++ b/worktree_linux.go @@ -12,7 +12,7 @@ import ( func init() { fillSystemInfo = func(e *index.Entry, sys interface{}) { if os, ok := sys.(*syscall.Stat_t); ok { - e.CreatedAt = time.Unix(int64(os.Ctim.Sec), int64(os.Ctim.Nsec)) + e.CreatedAt = time.Unix(os.Ctim.Unix()) e.Dev = uint32(os.Dev) e.Inode = uint32(os.Ino) e.GID = os.Gid diff --git a/worktree_test.go b/worktree_test.go index 79cbefd77..4a1412693 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "runtime" + "strings" "testing" "time" @@ -183,7 +184,7 @@ func (s *WorktreeSuite) TestPullInSingleBranch(c *C) { c.Assert(err, IsNil) c.Assert(branch.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") - branch, err = r.Reference("refs/remotes/foo/branch", false) + _, err = r.Reference("refs/remotes/foo/branch", false) c.Assert(err, NotNil) storage := r.Storer.(*memory.Storage) @@ -417,6 +418,37 @@ func (s *WorktreeSuite) TestCheckoutSymlink(c *C) { c.Assert(err, IsNil) } +func (s *WorktreeSuite) TestCheckoutSparse(c *C) { + fs := memfs.New() + r, err := Clone(memory.NewStorage(), fs, &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + sparseCheckoutDirectories := []string{"go", "json", "php"} + c.Assert(w.Checkout(&CheckoutOptions{ + SparseCheckoutDirectories: sparseCheckoutDirectories, + }), IsNil) + + fis, err := fs.ReadDir("/") + c.Assert(err, IsNil) + + for _, fi := range fis { + c.Assert(fi.IsDir(), Equals, true) + var oneOfSparseCheckoutDirs bool + + for _, sparseCheckoutDirectory := range sparseCheckoutDirectories { + if strings.HasPrefix(fi.Name(), sparseCheckoutDirectory) { + oneOfSparseCheckoutDirs = true + } + } + c.Assert(oneOfSparseCheckoutDirs, Equals, true) + } +} + func (s *WorktreeSuite) TestFilenameNormalization(c *C) { if runtime.GOOS == "windows" { c.Skip("windows paths may contain non utf-8 sequences") @@ -555,6 +587,7 @@ func (s *WorktreeSuite) TestCheckoutRelativePathSubmoduleInitialized(c *C) { // test submodule path modules, err := w.readGitmodulesFile() + c.Assert(err, IsNil) c.Assert(modules.Submodules["basic"].URL, Equals, "../basic.git") c.Assert(modules.Submodules["itself"].URL, Equals, "../submodule.git") diff --git a/worktree_unix_other.go b/worktree_unix_other.go index f45966be9..5b16e70b7 100644 --- a/worktree_unix_other.go +++ b/worktree_unix_other.go @@ -12,7 +12,7 @@ import ( func init() { fillSystemInfo = func(e *index.Entry, sys interface{}) { if os, ok := sys.(*syscall.Stat_t); ok { - e.CreatedAt = time.Unix(int64(os.Atim.Sec), int64(os.Atim.Nsec)) + e.CreatedAt = time.Unix(os.Atim.Unix()) e.Dev = uint32(os.Dev) e.Inode = uint32(os.Ino) e.GID = os.Gid