From 32975ee2c72d60121c37aa969d70b3b9953cd76a Mon Sep 17 00:00:00 2001 From: Zhizhen He Date: Fri, 8 Sep 2023 09:30:43 +0800 Subject: [PATCH 01/56] *: fix some typos Signed-off-by: Zhizhen He --- _examples/sha256/main.go | 2 +- config/config.go | 12 +- plumbing/filemode/filemode.go | 2 +- plumbing/hash/hash.go | 2 +- plumbing/object.go | 2 +- plumbing/object/commitgraph/commitnode.go | 196 ++++++------ .../object/commitgraph/commitnode_test.go | 296 +++++++++--------- plumbing/protocol/packp/ulreq_decode.go | 2 +- plumbing/protocol/packp/ulreq_decode_test.go | 2 +- plumbing/transport/common.go | 2 +- remote.go | 6 +- remote_test.go | 2 +- utils/binary/read.go | 2 +- utils/merkletrie/difftree.go | 2 +- utils/merkletrie/internal/fsnoder/file.go | 2 +- worktree_test.go | 2 +- 16 files changed, 267 insertions(+), 267 deletions(-) diff --git a/_examples/sha256/main.go b/_examples/sha256/main.go index e1772d274..03927724d 100644 --- a/_examples/sha256/main.go +++ b/_examples/sha256/main.go @@ -15,7 +15,7 @@ import ( // This example requires building with the sha256 tag for it to work: // go run -tags sha256 main.go /tmp/repository -// Basic example of how to initialise a repository using sha256 as the hashing algorithmn. +// Basic example of how to initialise a repository using sha256 as the hashing algorithm. func main() { CheckArgs("") directory := os.Args[1] diff --git a/config/config.go b/config/config.go index 82af12d28..da425a784 100644 --- a/config/config.go +++ b/config/config.go @@ -63,9 +63,9 @@ type Config struct { } User struct { - // Name is the personal name of the author and the commiter of a commit. + // Name is the personal name of the author and the committer of a commit. Name string - // Email is the email of the author and the commiter of a commit. + // Email is the email of the author and the committer of a commit. Email string } @@ -77,9 +77,9 @@ type Config struct { } Committer struct { - // Name is the personal name of the commiter of a commit. + // Name is the personal name of the committer of a commit. Name string - // Email is the email of the the commiter of a commit. + // Email is the email of the committer of a commit. Email string } @@ -157,8 +157,8 @@ func ReadConfig(r io.Reader) (*Config, error) { } // LoadConfig loads a config file from a given scope. The returned Config, -// contains exclusively information fom the given scope. If couldn't find a -// config file to the given scope, a empty one is returned. +// contains exclusively information from the given scope. If it couldn't find a +// config file to the given scope, an empty one is returned. func LoadConfig(scope Scope) (*Config, error) { if scope == LocalScope { return nil, fmt.Errorf("LocalScope should be read from the a ConfigStorer") diff --git a/plumbing/filemode/filemode.go b/plumbing/filemode/filemode.go index b848a9796..ea1a45755 100644 --- a/plumbing/filemode/filemode.go +++ b/plumbing/filemode/filemode.go @@ -133,7 +133,7 @@ func (m FileMode) IsMalformed() bool { m != Submodule } -// String returns the FileMode as a string in the standatd git format, +// String returns the FileMode as a string in the standard git format, // this is, an octal number padded with ceros to 7 digits. Malformed // modes are printed in that same format, for easier debugging. // diff --git a/plumbing/hash/hash.go b/plumbing/hash/hash.go index 82d185616..8609848f6 100644 --- a/plumbing/hash/hash.go +++ b/plumbing/hash/hash.go @@ -24,7 +24,7 @@ func reset() { algos[crypto.SHA256] = crypto.SHA256.New } -// RegisterHash allows for the hash algorithm used to be overriden. +// RegisterHash allows for the hash algorithm used to be overridden. // This ensures the hash selection for go-git must be explicit, when // overriding the default value. func RegisterHash(h crypto.Hash, f func() hash.Hash) error { diff --git a/plumbing/object.go b/plumbing/object.go index 2655dee43..3ee9de9f3 100644 --- a/plumbing/object.go +++ b/plumbing/object.go @@ -82,7 +82,7 @@ func (t ObjectType) Valid() bool { return t >= CommitObject && t <= REFDeltaObject } -// IsDelta returns true for any ObjectTyoe that represents a delta (i.e. +// IsDelta returns true for any ObjectType that represents a delta (i.e. // REFDeltaObject or OFSDeltaObject). func (t ObjectType) IsDelta() bool { return t == REFDeltaObject || t == OFSDeltaObject diff --git a/plumbing/object/commitgraph/commitnode.go b/plumbing/object/commitgraph/commitnode.go index 7abc58b80..d92c9064f 100644 --- a/plumbing/object/commitgraph/commitnode.go +++ b/plumbing/object/commitgraph/commitnode.go @@ -1,98 +1,98 @@ -package commitgraph - -import ( - "io" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -// CommitNode is generic interface encapsulating a lightweight commit object retrieved -// from CommitNodeIndex -type CommitNode interface { - // ID returns the Commit object id referenced by the commit graph node. - ID() plumbing.Hash - // Tree returns the Tree referenced by the commit graph node. - Tree() (*object.Tree, error) - // CommitTime returns the Commiter.When time of the Commit referenced by the commit graph node. - CommitTime() time.Time - // NumParents returns the number of parents in a commit. - NumParents() int - // ParentNodes return a CommitNodeIter for parents of specified node. - ParentNodes() CommitNodeIter - // ParentNode returns the ith parent of a commit. - ParentNode(i int) (CommitNode, error) - // ParentHashes returns hashes of the parent commits for a specified node - ParentHashes() []plumbing.Hash - // Generation returns the generation of the commit for reachability analysis. - // Objects with newer generation are not reachable from objects of older generation. - Generation() uint64 - // Commit returns the full commit object from the node - Commit() (*object.Commit, error) -} - -// CommitNodeIndex is generic interface encapsulating an index of CommitNode objects -type CommitNodeIndex interface { - // Get returns a commit node from a commit hash - Get(hash plumbing.Hash) (CommitNode, error) -} - -// CommitNodeIter is a generic closable interface for iterating over commit nodes. -type CommitNodeIter interface { - Next() (CommitNode, error) - ForEach(func(CommitNode) error) error - Close() -} - -// parentCommitNodeIter provides an iterator for parent commits from associated CommitNodeIndex. -type parentCommitNodeIter struct { - node CommitNode - i int -} - -func newParentgraphCommitNodeIter(node CommitNode) CommitNodeIter { - return &parentCommitNodeIter{node, 0} -} - -// Next moves the iterator to the next commit and returns a pointer to it. If -// there are no more commits, it returns io.EOF. -func (iter *parentCommitNodeIter) Next() (CommitNode, error) { - obj, err := iter.node.ParentNode(iter.i) - if err == object.ErrParentNotFound { - return nil, io.EOF - } - if err == nil { - iter.i++ - } - - return obj, err -} - -// ForEach call the cb function for each commit contained on this iter until -// an error appends or the end of the iter is reached. If ErrStop is sent -// the iteration is stopped but no error is returned. The iterator is closed. -func (iter *parentCommitNodeIter) ForEach(cb func(CommitNode) error) error { - for { - obj, err := iter.Next() - if err != nil { - if err == io.EOF { - return nil - } - - return err - } - - if err := cb(obj); err != nil { - if err == storer.ErrStop { - return nil - } - - return err - } - } -} - -func (iter *parentCommitNodeIter) Close() { -} +package commitgraph + +import ( + "io" + "time" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" +) + +// CommitNode is generic interface encapsulating a lightweight commit object retrieved +// from CommitNodeIndex +type CommitNode interface { + // ID returns the Commit object id referenced by the commit graph node. + ID() plumbing.Hash + // Tree returns the Tree referenced by the commit graph node. + Tree() (*object.Tree, error) + // CommitTime returns the Committer.When time of the Commit referenced by the commit graph node. + CommitTime() time.Time + // NumParents returns the number of parents in a commit. + NumParents() int + // ParentNodes return a CommitNodeIter for parents of specified node. + ParentNodes() CommitNodeIter + // ParentNode returns the ith parent of a commit. + ParentNode(i int) (CommitNode, error) + // ParentHashes returns hashes of the parent commits for a specified node + ParentHashes() []plumbing.Hash + // Generation returns the generation of the commit for reachability analysis. + // Objects with newer generation are not reachable from objects of older generation. + Generation() uint64 + // Commit returns the full commit object from the node + Commit() (*object.Commit, error) +} + +// CommitNodeIndex is generic interface encapsulating an index of CommitNode objects +type CommitNodeIndex interface { + // Get returns a commit node from a commit hash + Get(hash plumbing.Hash) (CommitNode, error) +} + +// CommitNodeIter is a generic closable interface for iterating over commit nodes. +type CommitNodeIter interface { + Next() (CommitNode, error) + ForEach(func(CommitNode) error) error + Close() +} + +// parentCommitNodeIter provides an iterator for parent commits from associated CommitNodeIndex. +type parentCommitNodeIter struct { + node CommitNode + i int +} + +func newParentgraphCommitNodeIter(node CommitNode) CommitNodeIter { + return &parentCommitNodeIter{node, 0} +} + +// Next moves the iterator to the next commit and returns a pointer to it. If +// there are no more commits, it returns io.EOF. +func (iter *parentCommitNodeIter) Next() (CommitNode, error) { + obj, err := iter.node.ParentNode(iter.i) + if err == object.ErrParentNotFound { + return nil, io.EOF + } + if err == nil { + iter.i++ + } + + return obj, err +} + +// ForEach call the cb function for each commit contained on this iter until +// an error appends or the end of the iter is reached. If ErrStop is sent +// the iteration is stopped but no error is returned. The iterator is closed. +func (iter *parentCommitNodeIter) ForEach(cb func(CommitNode) error) error { + for { + obj, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + + return err + } + + if err := cb(obj); err != nil { + if err == storer.ErrStop { + return nil + } + + return err + } + } +} + +func (iter *parentCommitNodeIter) Close() { +} diff --git a/plumbing/object/commitgraph/commitnode_test.go b/plumbing/object/commitgraph/commitnode_test.go index 6c9a64333..91fb21117 100644 --- a/plumbing/object/commitgraph/commitnode_test.go +++ b/plumbing/object/commitgraph/commitnode_test.go @@ -1,148 +1,148 @@ -package commitgraph - -import ( - "path" - "testing" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/format/commitgraph" - "github.com/go-git/go-git/v5/plumbing/format/packfile" - "github.com/go-git/go-git/v5/storage/filesystem" - - fixtures "github.com/go-git/go-git-fixtures/v4" - . "gopkg.in/check.v1" -) - -func Test(t *testing.T) { TestingT(t) } - -type CommitNodeSuite struct { - fixtures.Suite -} - -var _ = Suite(&CommitNodeSuite{}) - -func unpackRepositry(f *fixtures.Fixture) *filesystem.Storage { - storer := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) - p := f.Packfile() - defer p.Close() - packfile.UpdateObjectStorage(storer, p) - return storer -} - -func testWalker(c *C, nodeIndex CommitNodeIndex) { - head, err := nodeIndex.Get(plumbing.NewHash("b9d69064b190e7aedccf84731ca1d917871f8a1c")) - c.Assert(err, IsNil) - - iter := NewCommitNodeIterCTime( - head, - nil, - nil, - ) - - var commits []CommitNode - iter.ForEach(func(c CommitNode) error { - commits = append(commits, c) - return nil - }) - - c.Assert(commits, HasLen, 9) - - expected := []string{ - "b9d69064b190e7aedccf84731ca1d917871f8a1c", - "6f6c5d2be7852c782be1dd13e36496dd7ad39560", - "a45273fe2d63300e1962a9e26a6b15c276cd7082", - "c0edf780dd0da6a65a7a49a86032fcf8a0c2d467", - "bb13916df33ed23004c3ce9ed3b8487528e655c1", - "03d2c021ff68954cf3ef0a36825e194a4b98f981", - "ce275064ad67d51e99f026084e20827901a8361c", - "e713b52d7e13807e87a002e812041f248db3f643", - "347c91919944a68e9413581a1bc15519550a3afe", - } - for i, commit := range commits { - c.Assert(commit.ID().String(), Equals, expected[i]) - } -} - -func testParents(c *C, nodeIndex CommitNodeIndex) { - merge3, err := nodeIndex.Get(plumbing.NewHash("6f6c5d2be7852c782be1dd13e36496dd7ad39560")) - c.Assert(err, IsNil) - - var parents []CommitNode - merge3.ParentNodes().ForEach(func(c CommitNode) error { - parents = append(parents, c) - return nil - }) - - c.Assert(parents, HasLen, 3) - - expected := []string{ - "ce275064ad67d51e99f026084e20827901a8361c", - "bb13916df33ed23004c3ce9ed3b8487528e655c1", - "a45273fe2d63300e1962a9e26a6b15c276cd7082", - } - for i, parent := range parents { - c.Assert(parent.ID().String(), Equals, expected[i]) - } -} - -func testCommitAndTree(c *C, nodeIndex CommitNodeIndex) { - merge3node, err := nodeIndex.Get(plumbing.NewHash("6f6c5d2be7852c782be1dd13e36496dd7ad39560")) - c.Assert(err, IsNil) - merge3commit, err := merge3node.Commit() - c.Assert(err, IsNil) - c.Assert(merge3node.ID().String(), Equals, merge3commit.ID().String()) - tree, err := merge3node.Tree() - c.Assert(err, IsNil) - c.Assert(tree.ID().String(), Equals, merge3commit.TreeHash.String()) -} - -func (s *CommitNodeSuite) TestObjectGraph(c *C) { - f := fixtures.ByTag("commit-graph").One() - storer := unpackRepositry(f) - - nodeIndex := NewObjectCommitNodeIndex(storer) - testWalker(c, nodeIndex) - testParents(c, nodeIndex) - testCommitAndTree(c, nodeIndex) -} - -func (s *CommitNodeSuite) TestCommitGraph(c *C) { - f := fixtures.ByTag("commit-graph").One() - storer := unpackRepositry(f) - reader, err := storer.Filesystem().Open(path.Join("objects", "info", "commit-graph")) - c.Assert(err, IsNil) - defer reader.Close() - index, err := commitgraph.OpenFileIndex(reader) - c.Assert(err, IsNil) - - nodeIndex := NewGraphCommitNodeIndex(index, storer) - testWalker(c, nodeIndex) - testParents(c, nodeIndex) - testCommitAndTree(c, nodeIndex) -} - -func (s *CommitNodeSuite) TestMixedGraph(c *C) { - f := fixtures.ByTag("commit-graph").One() - storer := unpackRepositry(f) - - // Take the commit-graph file and copy it to memory index without the last commit - reader, err := storer.Filesystem().Open(path.Join("objects", "info", "commit-graph")) - c.Assert(err, IsNil) - defer reader.Close() - fileIndex, err := commitgraph.OpenFileIndex(reader) - c.Assert(err, IsNil) - memoryIndex := commitgraph.NewMemoryIndex() - for i, hash := range fileIndex.Hashes() { - if hash.String() != "b9d69064b190e7aedccf84731ca1d917871f8a1c" { - node, err := fileIndex.GetCommitDataByIndex(i) - c.Assert(err, IsNil) - memoryIndex.Add(hash, node) - } - } - - nodeIndex := NewGraphCommitNodeIndex(memoryIndex, storer) - testWalker(c, nodeIndex) - testParents(c, nodeIndex) - testCommitAndTree(c, nodeIndex) -} +package commitgraph + +import ( + "path" + "testing" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/format/commitgraph" + "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/storage/filesystem" + + fixtures "github.com/go-git/go-git-fixtures/v4" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type CommitNodeSuite struct { + fixtures.Suite +} + +var _ = Suite(&CommitNodeSuite{}) + +func unpackRepository(f *fixtures.Fixture) *filesystem.Storage { + storer := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) + p := f.Packfile() + defer p.Close() + packfile.UpdateObjectStorage(storer, p) + return storer +} + +func testWalker(c *C, nodeIndex CommitNodeIndex) { + head, err := nodeIndex.Get(plumbing.NewHash("b9d69064b190e7aedccf84731ca1d917871f8a1c")) + c.Assert(err, IsNil) + + iter := NewCommitNodeIterCTime( + head, + nil, + nil, + ) + + var commits []CommitNode + iter.ForEach(func(c CommitNode) error { + commits = append(commits, c) + return nil + }) + + c.Assert(commits, HasLen, 9) + + expected := []string{ + "b9d69064b190e7aedccf84731ca1d917871f8a1c", + "6f6c5d2be7852c782be1dd13e36496dd7ad39560", + "a45273fe2d63300e1962a9e26a6b15c276cd7082", + "c0edf780dd0da6a65a7a49a86032fcf8a0c2d467", + "bb13916df33ed23004c3ce9ed3b8487528e655c1", + "03d2c021ff68954cf3ef0a36825e194a4b98f981", + "ce275064ad67d51e99f026084e20827901a8361c", + "e713b52d7e13807e87a002e812041f248db3f643", + "347c91919944a68e9413581a1bc15519550a3afe", + } + for i, commit := range commits { + c.Assert(commit.ID().String(), Equals, expected[i]) + } +} + +func testParents(c *C, nodeIndex CommitNodeIndex) { + merge3, err := nodeIndex.Get(plumbing.NewHash("6f6c5d2be7852c782be1dd13e36496dd7ad39560")) + c.Assert(err, IsNil) + + var parents []CommitNode + merge3.ParentNodes().ForEach(func(c CommitNode) error { + parents = append(parents, c) + return nil + }) + + c.Assert(parents, HasLen, 3) + + expected := []string{ + "ce275064ad67d51e99f026084e20827901a8361c", + "bb13916df33ed23004c3ce9ed3b8487528e655c1", + "a45273fe2d63300e1962a9e26a6b15c276cd7082", + } + for i, parent := range parents { + c.Assert(parent.ID().String(), Equals, expected[i]) + } +} + +func testCommitAndTree(c *C, nodeIndex CommitNodeIndex) { + merge3node, err := nodeIndex.Get(plumbing.NewHash("6f6c5d2be7852c782be1dd13e36496dd7ad39560")) + c.Assert(err, IsNil) + merge3commit, err := merge3node.Commit() + c.Assert(err, IsNil) + c.Assert(merge3node.ID().String(), Equals, merge3commit.ID().String()) + tree, err := merge3node.Tree() + c.Assert(err, IsNil) + c.Assert(tree.ID().String(), Equals, merge3commit.TreeHash.String()) +} + +func (s *CommitNodeSuite) TestObjectGraph(c *C) { + f := fixtures.ByTag("commit-graph").One() + storer := unpackRepository(f) + + nodeIndex := NewObjectCommitNodeIndex(storer) + testWalker(c, nodeIndex) + testParents(c, nodeIndex) + testCommitAndTree(c, nodeIndex) +} + +func (s *CommitNodeSuite) TestCommitGraph(c *C) { + f := fixtures.ByTag("commit-graph").One() + storer := unpackRepository(f) + reader, err := storer.Filesystem().Open(path.Join("objects", "info", "commit-graph")) + c.Assert(err, IsNil) + defer reader.Close() + index, err := commitgraph.OpenFileIndex(reader) + c.Assert(err, IsNil) + + nodeIndex := NewGraphCommitNodeIndex(index, storer) + testWalker(c, nodeIndex) + testParents(c, nodeIndex) + testCommitAndTree(c, nodeIndex) +} + +func (s *CommitNodeSuite) TestMixedGraph(c *C) { + f := fixtures.ByTag("commit-graph").One() + storer := unpackRepository(f) + + // Take the commit-graph file and copy it to memory index without the last commit + reader, err := storer.Filesystem().Open(path.Join("objects", "info", "commit-graph")) + c.Assert(err, IsNil) + defer reader.Close() + fileIndex, err := commitgraph.OpenFileIndex(reader) + c.Assert(err, IsNil) + memoryIndex := commitgraph.NewMemoryIndex() + for i, hash := range fileIndex.Hashes() { + if hash.String() != "b9d69064b190e7aedccf84731ca1d917871f8a1c" { + node, err := fileIndex.GetCommitDataByIndex(i) + c.Assert(err, IsNil) + memoryIndex.Add(hash, node) + } + } + + nodeIndex := NewGraphCommitNodeIndex(memoryIndex, storer) + testWalker(c, nodeIndex) + testParents(c, nodeIndex) + testCommitAndTree(c, nodeIndex) +} diff --git a/plumbing/protocol/packp/ulreq_decode.go b/plumbing/protocol/packp/ulreq_decode.go index 895a3bf6d..3da29985e 100644 --- a/plumbing/protocol/packp/ulreq_decode.go +++ b/plumbing/protocol/packp/ulreq_decode.go @@ -43,7 +43,7 @@ func (d *ulReqDecoder) Decode(v *UploadRequest) error { return d.err } -// fills out the parser stiky error +// fills out the parser sticky error func (d *ulReqDecoder) error(format string, a ...interface{}) { msg := fmt.Sprintf( "pkt-line %d: %s", d.nLine, diff --git a/plumbing/protocol/packp/ulreq_decode_test.go b/plumbing/protocol/packp/ulreq_decode_test.go index efcc7b456..7658922de 100644 --- a/plumbing/protocol/packp/ulreq_decode_test.go +++ b/plumbing/protocol/packp/ulreq_decode_test.go @@ -398,7 +398,7 @@ func (s *UlReqDecodeSuite) TestDeepenCommits(c *C) { c.Assert(int(commits), Equals, 1234) } -func (s *UlReqDecodeSuite) TestDeepenCommitsInfiniteInplicit(c *C) { +func (s *UlReqDecodeSuite) TestDeepenCommitsInfiniteImplicit(c *C) { payloads := []string{ "want 3333333333333333333333333333333333333333 ofs-delta multi_ack", "deepen 0", diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index c6a054a65..b05437fbf 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -108,7 +108,7 @@ type Endpoint struct { // Host is the host. Host string // Port is the port to connect, if 0 the default port for the given protocol - // wil be used. + // will be used. Port int // Path is the repository path. Path string diff --git a/remote.go b/remote.go index 679e0af21..2ffffe7b6 100644 --- a/remote.go +++ b/remote.go @@ -614,7 +614,7 @@ func (r *Remote) addOrUpdateReferences( req *packp.ReferenceUpdateRequest, forceWithLease *ForceWithLease, ) error { - // If it is not a wilcard refspec we can directly search for the reference + // If it is not a wildcard refspec we can directly search for the reference // in the references dictionary. if !rs.IsWildcard() { ref, ok := refsDict[rs.Src()] @@ -693,7 +693,7 @@ func (r *Remote) addCommit(rs config.RefSpec, remoteRef, err := remoteRefs.Reference(cmd.Name) if err == nil { if remoteRef.Type() != plumbing.HashReference { - //TODO: check actual git behavior here + // TODO: check actual git behavior here return nil } @@ -735,7 +735,7 @@ func (r *Remote) addReferenceIfRefSpecMatches(rs config.RefSpec, remoteRef, err := remoteRefs.Reference(cmd.Name) if err == nil { if remoteRef.Type() != plumbing.HashReference { - //TODO: check actual git behavior here + // TODO: check actual git behavior here return nil } diff --git a/remote_test.go b/remote_test.go index ca5f261c7..e0c333294 100644 --- a/remote_test.go +++ b/remote_test.go @@ -196,7 +196,7 @@ func (s *RemoteSuite) TestFetchToNewBranchWithAllTags(c *C) { }) } -func (s *RemoteSuite) TestFetchNonExistantReference(c *C) { +func (s *RemoteSuite) TestFetchNonExistentReference(c *C) { r := NewRemote(memory.NewStorage(), &config.RemoteConfig{ URLs: []string{s.GetLocalRepositoryURL(fixtures.ByTag("tags").One())}, }) diff --git a/utils/binary/read.go b/utils/binary/read.go index a14d48db9..b8f9df1a2 100644 --- a/utils/binary/read.go +++ b/utils/binary/read.go @@ -1,4 +1,4 @@ -// Package binary implements sintax-sugar functions on top of the standard +// Package binary implements syntax-sugar functions on top of the standard // library binary package package binary diff --git a/utils/merkletrie/difftree.go b/utils/merkletrie/difftree.go index 9f5145a26..8090942dd 100644 --- a/utils/merkletrie/difftree.go +++ b/utils/merkletrie/difftree.go @@ -55,7 +55,7 @@ package merkletrie // Here is a full list of all the cases that are similar and how to // merge them together into more general cases. Each general case // is labeled with an uppercase letter for further reference, and it -// is followed by the pseudocode of the checks you have to perfrom +// is followed by the pseudocode of the checks you have to perform // on both noders to see if you are in such a case, the actions to // perform (i.e. what changes to output) and how to advance the // iterators of each tree to continue the comparison process. diff --git a/utils/merkletrie/internal/fsnoder/file.go b/utils/merkletrie/internal/fsnoder/file.go index 0bb908b7a..453efee04 100644 --- a/utils/merkletrie/internal/fsnoder/file.go +++ b/utils/merkletrie/internal/fsnoder/file.go @@ -32,7 +32,7 @@ func newFile(name, contents string) (*file, error) { func (f *file) Hash() []byte { if f.hash == nil { h := fnv.New64a() - h.Write([]byte(f.contents)) // it nevers returns an error. + h.Write([]byte(f.contents)) // it never returns an error. f.hash = h.Sum(nil) } diff --git a/worktree_test.go b/worktree_test.go index c69c61717..712695ae2 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -903,7 +903,7 @@ func (s *WorktreeSuite) TestStatusCheckedInBeforeIgnored(c *C) { c.Assert(status.IsClean(), Equals, true) c.Assert(status, NotNil) - err = util.WriteFile(fs, "secondIgnoredFile", []byte("Should be completly ignored"), 0755) + err = util.WriteFile(fs, "secondIgnoredFile", []byte("Should be completely ignored"), 0755) c.Assert(err, IsNil) status = nil status, err = w.Status() From c67eceb3c2b90ce392f577f8425d066197132860 Mon Sep 17 00:00:00 2001 From: nodivbyzero Date: Thu, 14 Sep 2023 17:03:16 -0700 Subject: [PATCH 02/56] git: worktree:: apply ProxyOption on Pull. Fixes #840 --- worktree.go | 1 + 1 file changed, 1 insertion(+) diff --git a/worktree.go b/worktree.go index f9c01af28..f8b854dda 100644 --- a/worktree.go +++ b/worktree.go @@ -78,6 +78,7 @@ func (w *Worktree) PullContext(ctx context.Context, o *PullOptions) error { Force: o.Force, InsecureSkipTLS: o.InsecureSkipTLS, CABundle: o.CABundle, + ProxyOptions: o.ProxyOptions, }) updated := true From 1ad7d8dd024abca82779e2dfdff69d9712161ab8 Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Wed, 31 May 2023 11:58:44 -0400 Subject: [PATCH 03/56] git: add `PlainInitOptions.Bare` Refactor `PlainInit` to call `PlainInitWithOptions` --- options.go | 2 ++ repository.go | 23 ++++++++++++----------- repository_test.go | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/options.go b/options.go index 757bdc84e..00d9b9776 100644 --- a/options.go +++ b/options.go @@ -737,6 +737,8 @@ type PlainOpenOptions struct { func (o *PlainOpenOptions) Validate() error { return nil } type PlainInitOptions struct { + // Determines if the repository will have a worktree (non-bare) or not (bare). + Bare bool ObjectFormat formatcfg.ObjectFormat } diff --git a/repository.go b/repository.go index 3154ac019..e94fc4893 100644 --- a/repository.go +++ b/repository.go @@ -235,9 +235,19 @@ func CloneContext( // if the repository will have worktree (non-bare) or not (bare), if the path // is not empty ErrRepositoryAlreadyExists is returned. func PlainInit(path string, isBare bool) (*Repository, error) { + return PlainInitWithOptions(path, &PlainInitOptions{ + Bare: isBare, + }) +} + +func PlainInitWithOptions(path string, opts *PlainInitOptions) (*Repository, error) { + if opts == nil { + opts = &PlainInitOptions{} + } + var wt, dot billy.Filesystem - if isBare { + if opts.Bare { dot = osfs.New(path) } else { wt = osfs.New(path) @@ -246,15 +256,6 @@ func PlainInit(path string, isBare bool) (*Repository, error) { s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - return Init(s, wt) -} - -func PlainInitWithOptions(path string, opts *PlainInitOptions) (*Repository, error) { - wt := osfs.New(path) - dot, _ := wt.Chroot(GitDirName) - - s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - r, err := Init(s, wt) if err != nil { return nil, err @@ -265,7 +266,7 @@ func PlainInitWithOptions(path string, opts *PlainInitOptions) (*Repository, err return nil, err } - if opts != nil { + if opts.ObjectFormat != "" { if opts.ObjectFormat == formatcfg.SHA256 && hash.CryptoType != crypto.SHA256 { return nil, ErrSHA256NotSupported } diff --git a/repository_test.go b/repository_test.go index 9e000a3da..bc6b18853 100644 --- a/repository_test.go +++ b/repository_test.go @@ -518,6 +518,21 @@ func (s *RepositorySuite) TestPlainInit(c *C) { c.Assert(cfg.Core.IsBare, Equals, true) } +func (s *RepositorySuite) TestPlainInitWithOptions(c *C) { + dir, clean := s.TemporalDir() + defer clean() + + r, err := PlainInitWithOptions(dir, &PlainInitOptions{ + Bare: true, + }) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + cfg, err := r.Config() + c.Assert(err, IsNil) + c.Assert(cfg.Core.IsBare, Equals, true) +} + func (s *RepositorySuite) TestPlainInitAlreadyExists(c *C) { dir, clean := s.TemporalDir() defer clean() From 644929ade3ac7c07c370be8065fa2ac6faf081be Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Wed, 31 May 2023 11:54:26 -0400 Subject: [PATCH 04/56] git: allow using `InitOptions` with `PlainInitWithOptions` --- options.go | 1 + repository.go | 2 +- repository_test.go | 13 +++++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/options.go b/options.go index 00d9b9776..c68bf2bed 100644 --- a/options.go +++ b/options.go @@ -737,6 +737,7 @@ type PlainOpenOptions struct { func (o *PlainOpenOptions) Validate() error { return nil } type PlainInitOptions struct { + InitOptions // Determines if the repository will have a worktree (non-bare) or not (bare). Bare bool ObjectFormat formatcfg.ObjectFormat diff --git a/repository.go b/repository.go index e94fc4893..013b53f80 100644 --- a/repository.go +++ b/repository.go @@ -256,7 +256,7 @@ func PlainInitWithOptions(path string, opts *PlainInitOptions) (*Repository, err s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - r, err := Init(s, wt) + r, err := InitWithOptions(s, wt, opts.InitOptions) if err != nil { return nil, err } diff --git a/repository_test.go b/repository_test.go index bc6b18853..3154f1d1c 100644 --- a/repository_test.go +++ b/repository_test.go @@ -523,14 +523,23 @@ func (s *RepositorySuite) TestPlainInitWithOptions(c *C) { defer clean() r, err := PlainInitWithOptions(dir, &PlainInitOptions{ - Bare: true, + InitOptions: InitOptions{ + DefaultBranch: "refs/heads/foo", + }, + Bare: false, }) c.Assert(err, IsNil) c.Assert(r, NotNil) cfg, err := r.Config() c.Assert(err, IsNil) - c.Assert(cfg.Core.IsBare, Equals, true) + c.Assert(cfg.Core.IsBare, Equals, false) + + createCommit(c, r) + + ref, err := r.Head() + c.Assert(err, IsNil) + c.Assert(ref.Name().String(), Equals, "refs/heads/foo") } func (s *RepositorySuite) TestPlainInitAlreadyExists(c *C) { From cbfd0c5005523b131449c6fdd7fa0c0bc84012a9 Mon Sep 17 00:00:00 2001 From: Aditya Sirish Date: Wed, 27 Sep 2023 11:54:31 -0400 Subject: [PATCH 05/56] plumbing/object: Support mergetag in merge commits When a merge commit is created from merging a signed tag, the tag object is embedded in the commit object. This commit adds support for this tag data when encoding and decoding a commit object. Signed-off-by: Aditya Sirish --- plumbing/object/commit.go | 38 ++++++++++++++++++++++++++++ plumbing/object/commit_test.go | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index 8a0f35c75..508e93c8c 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -20,6 +20,11 @@ const ( beginpgp string = "-----BEGIN PGP SIGNATURE-----" endpgp string = "-----END PGP SIGNATURE-----" headerpgp string = "gpgsig" + + // https://github.com/git/git/blob/bcb6cae2966cc407ca1afc77413b3ef11103c175/Documentation/gitformat-signature.txt#L153 + // When a merge commit is created from a signed tag, the tag is embedded in + // the commit with the "mergetag" header. + headermergetag string = "mergetag" ) // Hash represents the hash of an object @@ -38,6 +43,9 @@ type Commit struct { // Committer is the one performing the commit, might be different from // Author. Committer Signature + // MergeTag is the embedded tag object when a merge commit is created by + // merging a signed tag. + MergeTag string // PGPSignature is the PGP signature of the commit. PGPSignature string // Message is the commit message, contains arbitrary text. @@ -184,6 +192,7 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { defer sync.PutBufioReader(r) var message bool + var mergetag bool var pgpsig bool var msgbuf bytes.Buffer for { @@ -192,6 +201,16 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { return err } + if mergetag { + if len(line) > 0 && line[0] == ' ' { + line = bytes.TrimLeft(line, " ") + c.MergeTag += string(line) + continue + } else { + mergetag = false + } + } + if pgpsig { if len(line) > 0 && line[0] == ' ' { line = bytes.TrimLeft(line, " ") @@ -225,6 +244,9 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { c.Author.Decode(data) case "committer": c.Committer.Decode(data) + case headermergetag: + c.MergeTag += string(data) + "\n" + mergetag = true case headerpgp: c.PGPSignature += string(data) + "\n" pgpsig = true @@ -286,6 +308,22 @@ func (c *Commit) encode(o plumbing.EncodedObject, includeSig bool) (err error) { return err } + if c.MergeTag != "" { + if _, err = fmt.Fprint(w, "\n"+headermergetag+" "); err != nil { + return err + } + + // Split tag information lines and re-write with a left padding and + // newline. Use join for this so it's clear that a newline should not be + // added after this section. The newline will be added either as part of + // the PGP signature or the commit message. + mergetag := strings.TrimSuffix(c.MergeTag, "\n") + lines := strings.Split(mergetag, "\n") + if _, err = fmt.Fprint(w, strings.Join(lines, "\n ")); err != nil { + return err + } + } + if c.PGPSignature != "" && includeSig { if _, err = fmt.Fprint(w, "\n"+headerpgp+" "); err != nil { return err diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index 4b0f6b424..f1a344775 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -3,6 +3,7 @@ package object import ( "bytes" "context" + "fmt" "io" "strings" "time" @@ -197,6 +198,27 @@ func (s *SuiteCommit) TestPatchContext_ToNil(c *C) { } func (s *SuiteCommit) TestCommitEncodeDecodeIdempotent(c *C) { + pgpsignature := `-----BEGIN PGP SIGNATURE----- + +iQEcBAABAgAGBQJTZbQlAAoJEF0+sviABDDrZbQH/09PfE51KPVPlanr6q1v4/Ut +LQxfojUWiLQdg2ESJItkcuweYg+kc3HCyFejeDIBw9dpXt00rY26p05qrpnG+85b +hM1/PswpPLuBSr+oCIDj5GMC2r2iEKsfv2fJbNW8iWAXVLoWZRF8B0MfqX/YTMbm +ecorc4iXzQu7tupRihslbNkfvfciMnSDeSvzCpWAHl7h8Wj6hhqePmLm9lAYqnKp +8S5B/1SSQuEAjRZgI4IexpZoeKGVDptPHxLLS38fozsyi0QyDyzEgJxcJQVMXxVi +RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= +=EFTF +-----END PGP SIGNATURE----- +` + + tag := fmt.Sprintf(`object f000000000000000000000000000000000000000 +type commit +tag change +tagger Foo 1695827841 -0400 + +change +%s +`, pgpsignature) + ts, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05-07:00") c.Assert(err, IsNil) commits := []*Commit{ @@ -219,6 +241,29 @@ func (s *SuiteCommit) TestCommitEncodeDecodeIdempotent(c *C) { plumbing.NewHash("f000000000000000000000000000000000000007"), }, }, + { + Author: Signature{Name: "Foo", Email: "foo@example.local", When: ts}, + Committer: Signature{Name: "Bar", Email: "bar@example.local", When: ts}, + Message: "Testing mergetag\n\nHere, commit is not signed", + TreeHash: plumbing.NewHash("f000000000000000000000000000000000000001"), + ParentHashes: []plumbing.Hash{ + plumbing.NewHash("f000000000000000000000000000000000000002"), + plumbing.NewHash("f000000000000000000000000000000000000003"), + }, + MergeTag: tag, + }, + { + Author: Signature{Name: "Foo", Email: "foo@example.local", When: ts}, + Committer: Signature{Name: "Bar", Email: "bar@example.local", When: ts}, + Message: "Testing mergetag\n\nHere, commit is also signed", + TreeHash: plumbing.NewHash("f000000000000000000000000000000000000001"), + ParentHashes: []plumbing.Hash{ + plumbing.NewHash("f000000000000000000000000000000000000002"), + plumbing.NewHash("f000000000000000000000000000000000000003"), + }, + MergeTag: tag, + PGPSignature: pgpsignature, + }, } for _, commit := range commits { obj := &plumbing.MemoryObject{} From db02bbcb2422bf5cc9f7c6af409821b79362f28b Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Wed, 27 Sep 2023 23:03:52 +0100 Subject: [PATCH 06/56] *: Add bot to close stale issues Due to the limited amount of active maintainers, adding a stale bot to focus the available maintainership on active PRs and issues. Signed-off-by: Paulo Gomes --- .github/workflows/stale-issues-bot.yaml | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/stale-issues-bot.yaml diff --git a/.github/workflows/stale-issues-bot.yaml b/.github/workflows/stale-issues-bot.yaml new file mode 100644 index 000000000..a62b9e00d --- /dev/null +++ b/.github/workflows/stale-issues-bot.yaml @@ -0,0 +1,33 @@ +name: "stale issues bot" +on: + schedule: + - cron: "0 7 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale-bot: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + operations-per-run: 30 + days-before-stale: 90 + days-before-close: 30 + stale-issue-label: stale + stale-pr-label: stale + exempt-issue-labels: no-autoclose + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: | + To help us keep things tidy and focus on the active tasks, we've introduced a stale bot to spot + issues/PRs that haven't had any activity in a while. + + This particular issue hasn't had any updates or activity in the past 90 days, so it's been labeled + as 'stale'. If it remains inactive for the next 30 days, it'll be automatically closed. + + We understand everyone's busy, but if this issue is still important to you, please feel free to add + a comment or make an update to keep it active. + + Thanks for your understanding and cooperation! From 85f63b17e011872817117a38f92f08a1e01297bb Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Thu, 28 Sep 2023 23:55:05 +0100 Subject: [PATCH 07/56] *: Change stale bot order to ascending and improve msg format Signed-off-by: Paulo Gomes --- .github/workflows/stale-issues-bot.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/stale-issues-bot.yaml b/.github/workflows/stale-issues-bot.yaml index a62b9e00d..69a0d3558 100644 --- a/.github/workflows/stale-issues-bot.yaml +++ b/.github/workflows/stale-issues-bot.yaml @@ -13,6 +13,7 @@ jobs: steps: - uses: actions/stale@v8 with: + ascending: true operations-per-run: 30 days-before-stale: 90 days-before-close: 30 @@ -21,13 +22,10 @@ jobs: exempt-issue-labels: no-autoclose repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: | - To help us keep things tidy and focus on the active tasks, we've introduced a stale bot to spot - issues/PRs that haven't had any activity in a while. + To help us keep things tidy and focus on the active tasks, we've introduced a stale bot to spot issues/PRs that haven't had any activity in a while. - This particular issue hasn't had any updates or activity in the past 90 days, so it's been labeled - as 'stale'. If it remains inactive for the next 30 days, it'll be automatically closed. + This particular issue hasn't had any updates or activity in the past 90 days, so it's been labeled as 'stale'. If it remains inactive for the next 30 days, it'll be automatically closed. - We understand everyone's busy, but if this issue is still important to you, please feel free to add - a comment or make an update to keep it active. + We understand everyone's busy, but if this issue is still important to you, please feel free to add a comment or make an update to keep it active. Thanks for your understanding and cooperation! From 9c9c8c331b6bdb8fd5710c433e0dfd42f756d119 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 30 Sep 2023 09:09:10 +0100 Subject: [PATCH 08/56] plumbing: commitgraph, allow SHA256 commit-graphs Since the build-tag sha256 was introduced the commit graph code should be switched to use hash.Size and only use a graph if it has the correct hash version for the version of go-git that is built. Signed-off-by: Andrew Thornton --- plumbing/format/commitgraph/encoder.go | 9 +++++-- plumbing/format/commitgraph/file.go | 35 ++++++++++++++++++-------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/plumbing/format/commitgraph/encoder.go b/plumbing/format/commitgraph/encoder.go index f61025bb4..674f52e7c 100644 --- a/plumbing/format/commitgraph/encoder.go +++ b/plumbing/format/commitgraph/encoder.go @@ -1,6 +1,7 @@ package commitgraph import ( + "crypto" "io" "github.com/go-git/go-git/v5/plumbing" @@ -30,7 +31,7 @@ func (e *Encoder) Encode(idx Index) error { hashToIndex, fanout, extraEdgesCount := e.prepare(idx, hashes) chunkSignatures := [][]byte{oidFanoutSignature, oidLookupSignature, commitDataSignature} - chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * 36} + chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * (hash.Size + commitDataSize)} if extraEdgesCount > 0 { chunkSignatures = append(chunkSignatures, extraEdgeListSignature) chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*4) @@ -88,7 +89,11 @@ func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[pl func (e *Encoder) encodeFileHeader(chunkCount int) (err error) { if _, err = e.Write(commitFileSignature); err == nil { - _, err = e.Write([]byte{1, 1, byte(chunkCount), 0}) + version := byte(1) + if hash.CryptoType == crypto.SHA256 { + version = byte(2) + } + _, err = e.Write([]byte{1, version, byte(chunkCount), 0}) } return } diff --git a/plumbing/format/commitgraph/file.go b/plumbing/format/commitgraph/file.go index 1d25238f5..17c1c5d11 100644 --- a/plumbing/format/commitgraph/file.go +++ b/plumbing/format/commitgraph/file.go @@ -2,12 +2,14 @@ package commitgraph import ( "bytes" + "crypto" encbin "encoding/binary" "errors" "io" "time" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -36,6 +38,8 @@ var ( parentLast = uint32(0x80000000) ) +const commitDataSize = 16 + type fileIndex struct { reader io.ReaderAt fanout [256]int @@ -65,7 +69,7 @@ func OpenFileIndex(reader io.ReaderAt) (Index, error) { func (fi *fileIndex) verifyFileHeader() error { // Verify file signature - var signature = make([]byte, 4) + signature := make([]byte, 4) if _, err := fi.reader.ReadAt(signature, 0); err != nil { return err } @@ -74,22 +78,31 @@ func (fi *fileIndex) verifyFileHeader() error { } // Read and verify the file header - var header = make([]byte, 4) + header := make([]byte, 4) if _, err := fi.reader.ReadAt(header, 4); err != nil { return err } if header[0] != 1 { return ErrUnsupportedVersion } - if header[1] != 1 { - return ErrUnsupportedHash + if hash.CryptoType == crypto.SHA1 { + if header[1] != 1 { + return ErrUnsupportedVersion + } + } else if hash.CryptoType == crypto.SHA256 { + if header[1] != 2 { + return ErrUnsupportedVersion + } + } else { + // Unknown hash type + return ErrUnsupportedVersion } return nil } func (fi *fileIndex) readChunkHeaders() error { - var chunkID = make([]byte, 4) + chunkID := make([]byte, 4) for i := 0; ; i++ { chunkHeader := io.NewSectionReader(fi.reader, 8+(int64(i)*12), 12) if _, err := io.ReadAtLeast(chunkHeader, chunkID, 4); err != nil { @@ -148,7 +161,7 @@ func (fi *fileIndex) GetIndexByHash(h plumbing.Hash) (int, error) { high := fi.fanout[h[0]] for low < high { mid := (low + high) >> 1 - offset := fi.oidLookupOffset + int64(mid)*20 + offset := fi.oidLookupOffset + int64(mid)*hash.Size if _, err := fi.reader.ReadAt(oid[:], offset); err != nil { return 0, err } @@ -170,8 +183,8 @@ func (fi *fileIndex) GetCommitDataByIndex(idx int) (*CommitData, error) { return nil, plumbing.ErrObjectNotFound } - offset := fi.commitDataOffset + int64(idx)*36 - commitDataReader := io.NewSectionReader(fi.reader, offset, 36) + offset := fi.commitDataOffset + int64(idx)*(hash.Size+commitDataSize) + commitDataReader := io.NewSectionReader(fi.reader, offset, hash.Size+commitDataSize) treeHash, err := binary.ReadHash(commitDataReader) if err != nil { @@ -237,7 +250,7 @@ func (fi *fileIndex) getHashesFromIndexes(indexes []int) ([]plumbing.Hash, error return nil, ErrMalformedCommitGraphFile } - offset := fi.oidLookupOffset + int64(idx)*20 + offset := fi.oidLookupOffset + int64(idx)*hash.Size if _, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil { return nil, err } @@ -250,8 +263,8 @@ func (fi *fileIndex) getHashesFromIndexes(indexes []int) ([]plumbing.Hash, error func (fi *fileIndex) Hashes() []plumbing.Hash { hashes := make([]plumbing.Hash, fi.fanout[0xff]) for i := 0; i < fi.fanout[0xff]; i++ { - offset := fi.oidLookupOffset + int64(i)*20 - if n, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil || n < 20 { + offset := fi.oidLookupOffset + int64(i)*hash.Size + if n, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil || n < hash.Size { return nil } } From c135ec2f6a34116f63ebbdfde25001b21d560f24 Mon Sep 17 00:00:00 2001 From: Arjun Singh Date: Sun, 1 Oct 2023 14:53:58 +0530 Subject: [PATCH 09/56] fuzzing : fuzz testing support for oss-fuzz integration Signed-off-by: Arjun Singh --- Makefile | 10 ++++++++++ internal/revision/parser_test.go | 9 +++++++++ plumbing/format/config/decoder_test.go | 11 +++++++++++ plumbing/format/packfile/delta_test.go | 12 ++++++++++++ plumbing/object/signature_test.go | 7 +++++++ plumbing/object/tree_test.go | 17 +++++++++++++++++ plumbing/protocol/packp/uppackresp_test.go | 12 ++++++++++++ plumbing/transport/common_test.go | 7 +++++++ utils/merkletrie/internal/fsnoder/new_test.go | 9 +++++++++ 9 files changed, 94 insertions(+) diff --git a/Makefile b/Makefile index 66adc8ced..6c6289231 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,13 @@ test-coverage: clean: rm -rf $(GIT_DIST_PATH) + +fuzz: + @go test -fuzz=FuzzParser $(PWD)/internal/revision + @go test -fuzz=FuzzParseSignedByte $(PWD)/plumbing/object + @go test -fuzz=FuzzDecode $(PWD)/plumbing/object + @go test -fuzz=FuzzNewEndpoint $(PWD)/plumbing/transport + @go test -fuzz=FuzzDecoder $(PWD)/plumbing/protocol/packp + @go test -fuzz=FuzzDecoder $(PWD)/plumbing/format/config + @go test -fuzz=FuzzPatchDelta $(PWD)/plumbing/format/packfile + @go test -fuzz=FuzzDecodeFile $(PWD)/utils/merkletrie/internal/fsnoder diff --git a/internal/revision/parser_test.go b/internal/revision/parser_test.go index 3a77b2f11..1eb386100 100644 --- a/internal/revision/parser_test.go +++ b/internal/revision/parser_test.go @@ -3,6 +3,7 @@ package revision import ( "bytes" "regexp" + "testing" "time" . "gopkg.in/check.v1" @@ -397,3 +398,11 @@ func (s *ParserSuite) TestParseRefWithInvalidName(c *C) { c.Assert(err, DeepEquals, e) } } + +func FuzzParser(f *testing.F) { + + f.Fuzz(func(t *testing.T, input string) { + parser := NewParser(bytes.NewBufferString(input)) + parser.Parse() + }) +} diff --git a/plumbing/format/config/decoder_test.go b/plumbing/format/config/decoder_test.go index 0a8e92c83..6283f5e14 100644 --- a/plumbing/format/config/decoder_test.go +++ b/plumbing/format/config/decoder_test.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "testing" . "gopkg.in/check.v1" ) @@ -91,3 +92,13 @@ func decodeFails(c *C, text string) { err := d.Decode(cfg) c.Assert(err, NotNil) } + +func FuzzDecoder(f *testing.F) { + + f.Fuzz(func(t *testing.T, input []byte) { + + d := NewDecoder(bytes.NewReader(input)) + cfg := &Config{} + d.Decode(cfg) + }) +} diff --git a/plumbing/format/packfile/delta_test.go b/plumbing/format/packfile/delta_test.go index e8f5ea68f..9417e558a 100644 --- a/plumbing/format/packfile/delta_test.go +++ b/plumbing/format/packfile/delta_test.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "math/rand" + "testing" "github.com/go-git/go-git/v5/plumbing" . "gopkg.in/check.v1" @@ -176,3 +177,14 @@ func (s *DeltaSuite) TestMaxCopySizeDeltaReader(c *C) { c.Assert(err, IsNil) c.Assert(result, DeepEquals, targetBuf) } + +func FuzzPatchDelta(f *testing.F) { + + f.Fuzz(func(t *testing.T, input []byte) { + + input_0 := input[:len(input)/2] + input_1 := input[len(input)/2:] + + PatchDelta(input_0, input_1) + }) +} diff --git a/plumbing/object/signature_test.go b/plumbing/object/signature_test.go index 1bdb1d1ca..3b20cded4 100644 --- a/plumbing/object/signature_test.go +++ b/plumbing/object/signature_test.go @@ -178,3 +178,10 @@ signed tag`), }) } } + +func FuzzParseSignedBytes(f *testing.F) { + + f.Fuzz(func(t *testing.T, input []byte) { + parseSignedBytes(input) + }) +} diff --git a/plumbing/object/tree_test.go b/plumbing/object/tree_test.go index d9dad4775..bb5fc7a09 100644 --- a/plumbing/object/tree_test.go +++ b/plumbing/object/tree_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "testing" fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5/plumbing" @@ -1623,3 +1624,19 @@ func (s *TreeSuite) TestTreeDecodeReadBug(c *C) { c.Assert(err, IsNil) c.Assert(entriesEquals(obtained.Entries, expected.Entries), Equals, true) } + +func FuzzDecode(f *testing.F) { + + f.Fuzz(func(t *testing.T, input []byte) { + + obj := &SortReadObject{ + t: plumbing.TreeObject, + h: plumbing.ZeroHash, + cont: input, + sz: int64(len(input)), + } + + newTree := &Tree{} + newTree.Decode(obj) + }) +} diff --git a/plumbing/protocol/packp/uppackresp_test.go b/plumbing/protocol/packp/uppackresp_test.go index 8fbf92467..ec56507e2 100644 --- a/plumbing/protocol/packp/uppackresp_test.go +++ b/plumbing/protocol/packp/uppackresp_test.go @@ -3,6 +3,7 @@ package packp import ( "bytes" "io" + "testing" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" @@ -128,3 +129,14 @@ func (s *UploadPackResponseSuite) TestEncodeMultiACK(c *C) { b := bytes.NewBuffer(nil) c.Assert(res.Encode(b), NotNil) } + +func FuzzDecoder(f *testing.F) { + + f.Fuzz(func(t *testing.T, input []byte) { + req := NewUploadPackRequest() + res := NewUploadPackResponse(req) + defer res.Close() + + res.Decode(io.NopCloser(bytes.NewReader(input))) + }) +} diff --git a/plumbing/transport/common_test.go b/plumbing/transport/common_test.go index d9f12ab18..3efc555e7 100644 --- a/plumbing/transport/common_test.go +++ b/plumbing/transport/common_test.go @@ -210,3 +210,10 @@ func (s *SuiteCommon) TestNewEndpointIPv6(c *C) { c.Assert(e.Host, Equals, "[::1]") c.Assert(e.String(), Equals, "http://[::1]:8080/foo.git") } + +func FuzzNewEndpoint(f *testing.F) { + + f.Fuzz(func(t *testing.T, input string) { + NewEndpoint(input) + }) +} diff --git a/utils/merkletrie/internal/fsnoder/new_test.go b/utils/merkletrie/internal/fsnoder/new_test.go index ad069c7fc..52b3dc4b4 100644 --- a/utils/merkletrie/internal/fsnoder/new_test.go +++ b/utils/merkletrie/internal/fsnoder/new_test.go @@ -1,6 +1,8 @@ package fsnoder import ( + "testing" + "github.com/go-git/go-git/v5/utils/merkletrie/noder" . "gopkg.in/check.v1" @@ -352,3 +354,10 @@ func (s *FSNoderSuite) TestHashEqual(c *C) { c.Assert(HashEqual(t3, t1), Equals, false) c.Assert(HashEqual(t1, t3), Equals, false) } + +func FuzzDecodeFile(f *testing.F) { + + f.Fuzz(func(t *testing.T, input []byte) { + decodeFile(input) + }) +} From 3dbb11f2d99839e618cf8e100f0e8f77f543637a Mon Sep 17 00:00:00 2001 From: liwenqiu Date: Sun, 7 May 2023 22:37:08 +0800 Subject: [PATCH 10/56] plumbing: parse the encoding header of the commit object other part can re-code the commit message according to the encoding to this encoding info --- plumbing/object/commit.go | 23 ++++++++++++++++++++--- plumbing/object/commit_test.go | 6 +++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index 508e93c8c..ceed5d01e 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -17,19 +17,25 @@ import ( ) const ( - beginpgp string = "-----BEGIN PGP SIGNATURE-----" - endpgp string = "-----END PGP SIGNATURE-----" - headerpgp string = "gpgsig" + beginpgp string = "-----BEGIN PGP SIGNATURE-----" + endpgp string = "-----END PGP SIGNATURE-----" + headerpgp string = "gpgsig" + headerencoding string = "encoding" // https://github.com/git/git/blob/bcb6cae2966cc407ca1afc77413b3ef11103c175/Documentation/gitformat-signature.txt#L153 // When a merge commit is created from a signed tag, the tag is embedded in // the commit with the "mergetag" header. headermergetag string = "mergetag" + + defaultUtf8CommitMesageEncoding MessageEncoding = "UTF-8" ) // Hash represents the hash of an object type Hash plumbing.Hash +// MessageEncoding represents the encoding of a commit +type MessageEncoding string + // Commit points to a single tree, marking it as what the project looked like // at a certain point in time. It contains meta-information about that point // in time, such as a timestamp, the author of the changes since the last @@ -54,6 +60,8 @@ type Commit struct { TreeHash plumbing.Hash // ParentHashes are the hashes of the parent commits of the commit. ParentHashes []plumbing.Hash + // Encoding is the encoding of the commit. + Encoding MessageEncoding s storer.EncodedObjectStorer } @@ -181,6 +189,7 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } c.Hash = o.Hash() + c.Encoding = defaultUtf8CommitMesageEncoding reader, err := o.Reader() if err != nil { @@ -247,6 +256,8 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { case headermergetag: c.MergeTag += string(data) + "\n" mergetag = true + case headerencoding: + c.Encoding = MessageEncoding(data) case headerpgp: c.PGPSignature += string(data) + "\n" pgpsig = true @@ -324,6 +335,12 @@ func (c *Commit) encode(o plumbing.EncodedObject, includeSig bool) (err error) { } } + if string(c.Encoding) != "" && c.Encoding != defaultUtf8CommitMesageEncoding { + if _, err = fmt.Fprintf(w, "\n%s %s", headerencoding, c.Encoding); err != nil { + return err + } + } + if c.PGPSignature != "" && includeSig { if _, err = fmt.Fprint(w, "\n"+headerpgp+" "); err != nil { return err diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index f1a344775..3e1fe1b90 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -228,6 +228,7 @@ change Message: "Message\n\nFoo\nBar\nWith trailing blank lines\n\n", TreeHash: plumbing.NewHash("f000000000000000000000000000000000000001"), ParentHashes: []plumbing.Hash{plumbing.NewHash("f000000000000000000000000000000000000002")}, + Encoding: defaultUtf8CommitMesageEncoding, }, { Author: Signature{Name: "Foo", Email: "foo@example.local", When: ts}, @@ -240,6 +241,7 @@ change plumbing.NewHash("f000000000000000000000000000000000000006"), plumbing.NewHash("f000000000000000000000000000000000000007"), }, + Encoding: MessageEncoding("ISO-8859-1"), }, { Author: Signature{Name: "Foo", Email: "foo@example.local", When: ts}, @@ -251,6 +253,7 @@ change plumbing.NewHash("f000000000000000000000000000000000000003"), }, MergeTag: tag, + Encoding: defaultUtf8CommitMesageEncoding, }, { Author: Signature{Name: "Foo", Email: "foo@example.local", When: ts}, @@ -263,6 +266,7 @@ change }, MergeTag: tag, PGPSignature: pgpsignature, + Encoding: defaultUtf8CommitMesageEncoding, }, } for _, commit := range commits { @@ -530,7 +534,7 @@ func (s *SuiteCommit) TestMalformedHeader(c *C) { } func (s *SuiteCommit) TestEncodeWithoutSignature(c *C) { - //Similar to TestString since no signature + // Similar to TestString since no signature encoded := &plumbing.MemoryObject{} err := s.Commit.EncodeWithoutSignature(encoded) c.Assert(err, IsNil) From d32d6cdd1e5e144f7d3bb1f90f2076b3f3378674 Mon Sep 17 00:00:00 2001 From: Arjun Singh Date: Wed, 4 Oct 2023 11:04:06 +0530 Subject: [PATCH 11/56] [fuzzing] cifuzz, update fuzzers, bug fix Signed-off-by: Arjun Singh --- .github/workflows/cifuzz.yml | 35 +++++++++++++++++++ Makefile | 9 +++-- oss-fuzz.sh | 35 +++++++++++++++++++ utils/merkletrie/internal/fsnoder/new_test.go | 9 ----- 4 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/cifuzz.yml create mode 100644 oss-fuzz.sh diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml new file mode 100644 index 000000000..2b17ac18b --- /dev/null +++ b/.github/workflows/cifuzz.yml @@ -0,0 +1,35 @@ +name: CIFuzz +on: [pull_request] +permissions: {} +jobs: + Fuzzing: + runs-on: ubuntu-latest + permissions: + security-events: write + steps: + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'go-git' + language: go + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'go-git' + language: go + fuzz-seconds: 300 + output-sarif: true + - name: Upload Crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts + - name: Upload Sarif + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v2 + with: + # Path to SARIF file relative to the root of the repository + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/Makefile b/Makefile index 6c6289231..1e1039674 100644 --- a/Makefile +++ b/Makefile @@ -45,10 +45,9 @@ clean: fuzz: @go test -fuzz=FuzzParser $(PWD)/internal/revision - @go test -fuzz=FuzzParseSignedByte $(PWD)/plumbing/object - @go test -fuzz=FuzzDecode $(PWD)/plumbing/object - @go test -fuzz=FuzzNewEndpoint $(PWD)/plumbing/transport - @go test -fuzz=FuzzDecoder $(PWD)/plumbing/protocol/packp @go test -fuzz=FuzzDecoder $(PWD)/plumbing/format/config @go test -fuzz=FuzzPatchDelta $(PWD)/plumbing/format/packfile - @go test -fuzz=FuzzDecodeFile $(PWD)/utils/merkletrie/internal/fsnoder + @go test -fuzz=FuzzParseSignedBytes $(PWD)/plumbing/object + @go test -fuzz=FuzzDecode $(PWD)/plumbing/object + @go test -fuzz=FuzzDecoder $(PWD)/plumbing/protocol/packp + @go test -fuzz=FuzzNewEndpoint $(PWD)/plumbing/transport diff --git a/oss-fuzz.sh b/oss-fuzz.sh new file mode 100644 index 000000000..885548f40 --- /dev/null +++ b/oss-fuzz.sh @@ -0,0 +1,35 @@ +#!/bin/bash -eu +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + + +go mod download +go get github.com/AdamKorcz/go-118-fuzz-build/testing + +if [ "$SANITIZER" != "coverage" ]; then + sed -i '/func (s \*DecoderSuite) TestDecode(/,/^}/ s/^/\/\//' plumbing/format/config/decoder_test.go + sed -n '35,$p' plumbing/format/packfile/common_test.go >> plumbing/format/packfile/delta_test.go + sed -n '20,53p' plumbing/object/object_test.go >> plumbing/object/tree_test.go + sed -i 's|func Test|// func Test|' plumbing/transport/common_test.go +fi + +compile_native_go_fuzzer $(pwd)/internal/revision FuzzParser fuzz_parser +compile_native_go_fuzzer $(pwd)/plumbing/format/config FuzzDecoder fuzz_decoder_config +compile_native_go_fuzzer $(pwd)/plumbing/format/packfile FuzzPatchDelta fuzz_patch_delta +compile_native_go_fuzzer $(pwd)/plumbing/object FuzzParseSignedBytes fuzz_parse_signed_bytes +compile_native_go_fuzzer $(pwd)/plumbing/object FuzzDecode fuzz_decode +compile_native_go_fuzzer $(pwd)/plumbing/protocol/packp FuzzDecoder fuzz_decoder_packp +compile_native_go_fuzzer $(pwd)/plumbing/transport FuzzNewEndpoint fuzz_new_endpoint diff --git a/utils/merkletrie/internal/fsnoder/new_test.go b/utils/merkletrie/internal/fsnoder/new_test.go index 52b3dc4b4..ad069c7fc 100644 --- a/utils/merkletrie/internal/fsnoder/new_test.go +++ b/utils/merkletrie/internal/fsnoder/new_test.go @@ -1,8 +1,6 @@ package fsnoder import ( - "testing" - "github.com/go-git/go-git/v5/utils/merkletrie/noder" . "gopkg.in/check.v1" @@ -354,10 +352,3 @@ func (s *FSNoderSuite) TestHashEqual(c *C) { c.Assert(HashEqual(t3, t1), Equals, false) c.Assert(HashEqual(t1, t3), Equals, false) } - -func FuzzDecodeFile(f *testing.F) { - - f.Fuzz(func(t *testing.T, input []byte) { - decodeFile(input) - }) -} From 1d26511c717ea334a16499a018d2b877b557be79 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Sat, 7 Oct 2023 18:44:31 +0100 Subject: [PATCH 12/56] plumbing: protocol/packp, Add validation for decodeLine Signed-off-by: Paulo Gomes --- plumbing/protocol/packp/srvresp.go | 12 ++++++----- plumbing/protocol/packp/srvresp_test.go | 27 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/plumbing/protocol/packp/srvresp.go b/plumbing/protocol/packp/srvresp.go index 8cd0a7247..a9ddb538b 100644 --- a/plumbing/protocol/packp/srvresp.go +++ b/plumbing/protocol/packp/srvresp.go @@ -101,12 +101,14 @@ func (r *ServerResponse) decodeLine(line []byte) error { return fmt.Errorf("unexpected flush") } - if bytes.Equal(line[0:3], ack) { - return r.decodeACKLine(line) - } + if len(line) >= 3 { + if bytes.Equal(line[0:3], ack) { + return r.decodeACKLine(line) + } - if bytes.Equal(line[0:3], nak) { - return nil + if bytes.Equal(line[0:3], nak) { + return nil + } } return fmt.Errorf("unexpected content %q", string(line)) diff --git a/plumbing/protocol/packp/srvresp_test.go b/plumbing/protocol/packp/srvresp_test.go index aa0af528a..b7270e79e 100644 --- a/plumbing/protocol/packp/srvresp_test.go +++ b/plumbing/protocol/packp/srvresp_test.go @@ -3,6 +3,7 @@ package packp import ( "bufio" "bytes" + "fmt" "github.com/go-git/go-git/v5/plumbing" @@ -23,6 +24,32 @@ func (s *ServerResponseSuite) TestDecodeNAK(c *C) { c.Assert(sr.ACKs, HasLen, 0) } +func (s *ServerResponseSuite) TestDecodeNewLine(c *C) { + raw := "\n" + + sr := &ServerResponse{} + err := sr.Decode(bufio.NewReader(bytes.NewBufferString(raw)), false) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "invalid pkt-len found") +} + +func (s *ServerResponseSuite) TestDecodeEmpty(c *C) { + raw := "" + + sr := &ServerResponse{} + err := sr.Decode(bufio.NewReader(bytes.NewBufferString(raw)), false) + c.Assert(err, IsNil) +} + +func (s *ServerResponseSuite) TestDecodePartial(c *C) { + raw := "000600\n" + + sr := &ServerResponse{} + err := sr.Decode(bufio.NewReader(bytes.NewBufferString(raw)), false) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, fmt.Sprintf("unexpected content %q", "00")) +} + func (s *ServerResponseSuite) TestDecodeACK(c *C) { raw := "0031ACK 6ecf0ef2c2dffb796033e5a02219af86ec6584e5\n" From 479d3e952e75d6e41b71a81cc9a92dec792825ba Mon Sep 17 00:00:00 2001 From: enverbisevac Date: Fri, 6 Oct 2023 02:54:20 +0200 Subject: [PATCH 13/56] git: clone --shared implemented --- .gitignore | 1 + COMPATIBILITY.md | 344 ++++++++++++++-------------- options.go | 9 + plumbing/storer/object.go | 1 + plumbing/storer/object_test.go | 4 + repository.go | 26 +++ repository_test.go | 96 ++++++++ storage/filesystem/dotgit/dotgit.go | 33 ++- storage/filesystem/storage.go | 4 + storage/memory/storage.go | 4 + storage/transactional/object.go | 4 + 11 files changed, 353 insertions(+), 173 deletions(-) diff --git a/.gitignore b/.gitignore index 361133d03..b7f2c5807 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ coverage.txt profile.out .tmp/ .git-dist/ +.vscode diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index afd4f03bc..bbffea522 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -5,229 +5,229 @@ compatibility status with go-git. ## Getting and creating repositories -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `init` | | ✅ | | | -| `init` | `--bare` | ✅ | | | -| `init` | `--template`
`--separate-git-dir`
`--shared` | ❌ | | | -| `clone` | | ✅ | | - [PlainClone](_examples/clone/main.go) | -| `clone` | Authentication:
- none
- access token
- username + password
- ssh | ✅ | | - [clone ssh](_examples/clone/auth/ssh/main.go)
- [clone access token](_examples/clone/auth/basic/access_token/main.go)
- [clone user + password](_examples/clone/auth/basic/username_password/main.go) | -| `clone` | `--progress`
`--single-branch`
`--depth`
`--origin`
`--recurse-submodules` | ✅ | | - [recurse submodules](_examples/clone/main.go)
- [progress](_examples/progress/main.go) | +| Feature | Sub-feature | Status | Notes | Examples | +| ------- | ------------------------------------------------------------------------------------------------------------------ | ------ | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init` | | ✅ | | | +| `init` | `--bare` | ✅ | | | +| `init` | `--template`
`--separate-git-dir`
`--shared` | ❌ | | | +| `clone` | | ✅ | | - [PlainClone](_examples/clone/main.go) | +| `clone` | Authentication:
- none
- access token
- username + password
- ssh | ✅ | | - [clone ssh](_examples/clone/auth/ssh/main.go)
- [clone access token](_examples/clone/auth/basic/access_token/main.go)
- [clone user + password](_examples/clone/auth/basic/username_password/main.go) | +| `clone` | `--progress`
`--single-branch`
`--depth`
`--origin`
`--recurse-submodules`
`--shared` | ✅ | | - [recurse submodules](_examples/clone/main.go)
- [progress](_examples/progress/main.go) | ## Basic snapshotting -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `add` | | ✅ | Plain add is supported. Any other flags aren't supported | | -| `status` | | ✅ | | | -| `commit` | | ✅ | | - [commit](_examples/commit/main.go) | -| `reset` | | ✅ | | | -| `rm` | | ✅ | | | -| `mv` | | ✅ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| -------- | ----------- | ------ | -------------------------------------------------------- | ------------------------------------ | +| `add` | | ✅ | Plain add is supported. Any other flags aren't supported | | +| `status` | | ✅ | | | +| `commit` | | ✅ | | - [commit](_examples/commit/main.go) | +| `reset` | | ✅ | | | +| `rm` | | ✅ | | | +| `mv` | | ✅ | | | ## Branching and merging -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `branch` | | ✅ | | - [branch](_examples/branch/main.go) | -| `checkout` | | ✅ | Basic usages of checkout are supported. | - [checkout](_examples/checkout/main.go) | -| `merge` | | ❌ | | | -| `mergetool` | | ❌ | | | -| `stash` | | ❌ | | | -| `tag` | | ✅ | | - [tag](_examples/tag/main.go)
- [tag create and push](_examples/tag-create-push/main.go) | +| Feature | Sub-feature | Status | Notes | Examples | +| ----------- | ----------- | ------ | --------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `branch` | | ✅ | | - [branch](_examples/branch/main.go) | +| `checkout` | | ✅ | Basic usages of checkout are supported. | - [checkout](_examples/checkout/main.go) | +| `merge` | | ❌ | | | +| `mergetool` | | ❌ | | | +| `stash` | | ❌ | | | +| `tag` | | ✅ | | - [tag](_examples/tag/main.go)
- [tag create and push](_examples/tag-create-push/main.go) | ## Sharing and updating projects -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `fetch` | | ✅ | | | -| `pull` | | ✅ | Only supports merges where the merge can be resolved as a fast-forward. | - [pull](_examples/pull/main.go) | -| `push` | | ✅ | | - [push](_examples/push/main.go) | -| `remote` | | ✅ | | - [remotes](_examples/remotes/main.go) | -| `submodule` | | ✅ | | - [submodule](_examples/submodule/main.go) | -| `submodule` | deinit | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| ----------- | ----------- | ------ | ----------------------------------------------------------------------- | ------------------------------------------ | +| `fetch` | | ✅ | | | +| `pull` | | ✅ | Only supports merges where the merge can be resolved as a fast-forward. | - [pull](_examples/pull/main.go) | +| `push` | | ✅ | | - [push](_examples/push/main.go) | +| `remote` | | ✅ | | - [remotes](_examples/remotes/main.go) | +| `submodule` | | ✅ | | - [submodule](_examples/submodule/main.go) | +| `submodule` | deinit | ❌ | | | ## Inspection and comparison -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `show` | | ✅ | | | -| `log` | | ✅ | | - [log](_examples/log/main.go) | -| `shortlog` | | (see log) | | | -| `describe` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| ---------- | ----------- | --------- | ----- | ------------------------------ | +| `show` | | ✅ | | | +| `log` | | ✅ | | - [log](_examples/log/main.go) | +| `shortlog` | | (see log) | | | +| `describe` | | ❌ | | | ## Patching -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `apply` | | ❌ | | | -| `cherry-pick` | | ❌ | | | -| `diff` | | ✅ | Patch object with UnifiedDiff output representation. | | -| `rebase` | | ❌ | | | -| `revert` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| ------------- | ----------- | ------ | ---------------------------------------------------- | -------- | +| `apply` | | ❌ | | | +| `cherry-pick` | | ❌ | | | +| `diff` | | ✅ | Patch object with UnifiedDiff output representation. | | +| `rebase` | | ❌ | | | +| `revert` | | ❌ | | | ## Debugging -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `bisect` | | ❌ | | | -| `blame` | | ✅ | | - [blame](_examples/blame/main.go) | -| `grep` | | ✅ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| -------- | ----------- | ------ | ----- | ---------------------------------- | +| `bisect` | | ❌ | | | +| `blame` | | ✅ | | - [blame](_examples/blame/main.go) | +| `grep` | | ✅ | | | ## Email -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `am` | | ❌ | | | -| `apply` | | ❌ | | | -| `format-patch` | | ❌ | | | -| `send-email` | | ❌ | | | -| `request-pull` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| -------------- | ----------- | ------ | ----- | -------- | +| `am` | | ❌ | | | +| `apply` | | ❌ | | | +| `format-patch` | | ❌ | | | +| `send-email` | | ❌ | | | +| `request-pull` | | ❌ | | | ## External systems -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `svn` | | ❌ | | | -| `fast-import` | | ❌ | | | -| `lfs` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| ------------- | ----------- | ------ | ----- | -------- | +| `svn` | | ❌ | | | +| `fast-import` | | ❌ | | | +| `lfs` | | ❌ | | | ## Administration -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `clean` | | ✅ | | | -| `gc` | | ❌ | | | -| `fsck` | | ❌ | | | -| `reflog` | | ❌ | | | -| `filter-branch` | | ❌ | | | -| `instaweb` | | ❌ | | | -| `archive` | | ❌ | | | -| `bundle` | | ❌ | | | -| `prune` | | ❌ | | | -| `repack` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| --------------- | ----------- | ------ | ----- | -------- | +| `clean` | | ✅ | | | +| `gc` | | ❌ | | | +| `fsck` | | ❌ | | | +| `reflog` | | ❌ | | | +| `filter-branch` | | ❌ | | | +| `instaweb` | | ❌ | | | +| `archive` | | ❌ | | | +| `bundle` | | ❌ | | | +| `prune` | | ❌ | | | +| `repack` | | ❌ | | | ## Server admin -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `daemon` | | ❌ | | | -| `update-server-info` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| -------------------- | ----------- | ------ | ----- | -------- | +| `daemon` | | ❌ | | | +| `update-server-info` | | ❌ | | | ## Advanced -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `notes` | | ❌ | | | -| `replace` | | ❌ | | | -| `worktree` | | ❌ | | | -| `annotate` | | (see blame) | | | +| Feature | Sub-feature | Status | Notes | Examples | +| ---------- | ----------- | ----------- | ----- | -------- | +| `notes` | | ❌ | | | +| `replace` | | ❌ | | | +| `worktree` | | ❌ | | | +| `annotate` | | (see blame) | | | ## GPG -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `git-verify-commit` | | ✅ | | | -| `git-verify-tag` | | ✅ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| ------------------- | ----------- | ------ | ----- | -------- | +| `git-verify-commit` | | ✅ | | | +| `git-verify-tag` | | ✅ | | | ## Plumbing commands -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `cat-file` | | ✅ | | | -| `check-ignore` | | ❌ | | | -| `commit-tree` | | ❌ | | | -| `count-objects` | | ❌ | | | -| `diff-index` | | ❌ | | | -| `for-each-ref` | | ✅ | | | -| `hash-object` | | ✅ | | | -| `ls-files` | | ✅ | | | -| `ls-remote` | | ✅ | | - [ls-remote](_examples/ls-remote/main.go) | -| `merge-base` | `--independent`
`--is-ancestor` | ⚠️ (partial) | Calculates the merge-base only between two commits. | - [merge-base](_examples/merge_base/main.go) | -| `merge-base` | `--fork-point`
`--octopus` | ❌ | | | -| `read-tree` | | ❌ | | | -| `rev-list` | | ✅ | | | -| `rev-parse` | | ❌ | | | -| `show-ref` | | ✅ | | | -| `symbolic-ref` | | ✅ | | | -| `update-index` | | ❌ | | | -| `update-ref` | | ❌ | | | -| `verify-pack` | | ❌ | | | -| `write-tree` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| --------------- | ------------------------------------- | ------------ | --------------------------------------------------- | -------------------------------------------- | +| `cat-file` | | ✅ | | | +| `check-ignore` | | ❌ | | | +| `commit-tree` | | ❌ | | | +| `count-objects` | | ❌ | | | +| `diff-index` | | ❌ | | | +| `for-each-ref` | | ✅ | | | +| `hash-object` | | ✅ | | | +| `ls-files` | | ✅ | | | +| `ls-remote` | | ✅ | | - [ls-remote](_examples/ls-remote/main.go) | +| `merge-base` | `--independent`
`--is-ancestor` | ⚠️ (partial) | Calculates the merge-base only between two commits. | - [merge-base](_examples/merge_base/main.go) | +| `merge-base` | `--fork-point`
`--octopus` | ❌ | | | +| `read-tree` | | ❌ | | | +| `rev-list` | | ✅ | | | +| `rev-parse` | | ❌ | | | +| `show-ref` | | ✅ | | | +| `symbolic-ref` | | ✅ | | | +| `update-index` | | ❌ | | | +| `update-ref` | | ❌ | | | +| `verify-pack` | | ❌ | | | +| `write-tree` | | ❌ | | | ## Indexes and Git Protocols -| Feature | Version | Status | Notes | -|---|---|---|---| -| index | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-index.txt) | ❌ | | -| index | [v2](https://github.com/git/git/blob/master/Documentation/gitformat-index.txt) | ✅ | | -| index | [v3](https://github.com/git/git/blob/master/Documentation/gitformat-index.txt) | ❌ | | -| pack-protocol | [v1](https://github.com/git/git/blob/master/Documentation/gitprotocol-pack.txt) | ✅ | | -| pack-protocol | [v2](https://github.com/git/git/blob/master/Documentation/gitprotocol-v2.txt) | ❌ | | -| multi-pack-index | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-pack.txt) | ❌ | | -| pack-*.rev files | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-pack.txt) | ❌ | | -| pack-*.mtimes files | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-pack.txt) | ❌ | | -| cruft packs | | ❌ | | +| Feature | Version | Status | Notes | +| -------------------- | ------------------------------------------------------------------------------- | ------ | ----- | +| index | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-index.txt) | ❌ | | +| index | [v2](https://github.com/git/git/blob/master/Documentation/gitformat-index.txt) | ✅ | | +| index | [v3](https://github.com/git/git/blob/master/Documentation/gitformat-index.txt) | ❌ | | +| pack-protocol | [v1](https://github.com/git/git/blob/master/Documentation/gitprotocol-pack.txt) | ✅ | | +| pack-protocol | [v2](https://github.com/git/git/blob/master/Documentation/gitprotocol-v2.txt) | ❌ | | +| multi-pack-index | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-pack.txt) | ❌ | | +| pack-\*.rev files | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-pack.txt) | ❌ | | +| pack-\*.mtimes files | [v1](https://github.com/git/git/blob/master/Documentation/gitformat-pack.txt) | ❌ | | +| cruft packs | | ❌ | | ## Capabilities -| Feature | Status | Notes | -|---|---|---| -| `multi_ack` | ❌ | | -| `multi_ack_detailed` | ❌ | | -| `no-done` | ❌ | | -| `thin-pack` | ❌ | | -| `side-band` | ⚠️ (partial) | | -| `side-band-64k` | ⚠️ (partial) | | -| `ofs-delta` | ✅ | | -| `agent` | ✅ | | -| `object-format` | ❌ | | -| `symref` | ✅ | | -| `shallow` | ✅ | | -| `deepen-since` | ✅ | | -| `deepen-not` | ❌ | | -| `deepen-relative` | ❌ | | -| `no-progress` | ✅ | | -| `include-tag` | ✅ | | -| `report-status` | ✅ | | -| `report-status-v2` | ❌ | | -| `delete-refs` | ✅ | | -| `quiet` | ❌ | | -| `atomic` | ✅ | | -| `push-options` | ✅ | | -| `allow-tip-sha1-in-want` | ✅ | | -| `allow-reachable-sha1-in-want` | ❌ | | -| `push-cert=` | ❌ | | -| `filter` | ❌ | | -| `session-id=` | ❌ | | +| Feature | Status | Notes | +| ------------------------------ | ------------ | ----- | +| `multi_ack` | ❌ | | +| `multi_ack_detailed` | ❌ | | +| `no-done` | ❌ | | +| `thin-pack` | ❌ | | +| `side-band` | ⚠️ (partial) | | +| `side-band-64k` | ⚠️ (partial) | | +| `ofs-delta` | ✅ | | +| `agent` | ✅ | | +| `object-format` | ❌ | | +| `symref` | ✅ | | +| `shallow` | ✅ | | +| `deepen-since` | ✅ | | +| `deepen-not` | ❌ | | +| `deepen-relative` | ❌ | | +| `no-progress` | ✅ | | +| `include-tag` | ✅ | | +| `report-status` | ✅ | | +| `report-status-v2` | ❌ | | +| `delete-refs` | ✅ | | +| `quiet` | ❌ | | +| `atomic` | ✅ | | +| `push-options` | ✅ | | +| `allow-tip-sha1-in-want` | ✅ | | +| `allow-reachable-sha1-in-want` | ❌ | | +| `push-cert=` | ❌ | | +| `filter` | ❌ | | +| `session-id=` | ❌ | | ## Transport Schemes -| Scheme | Status | Notes | Examples | -|---|---|---|---| -| `http(s)://` (dumb) | ❌ | | | -| `http(s)://` (smart) | ✅ | | | -| `git://` | ✅ | | | -| `ssh://` | ✅ | | | -| `file://` | ⚠️ (partial) | Warning: this is not pure Golang. This shells out to the `git` binary. | | -| Custom | ✅ | All existing schemes can be replaced by custom implementations. | - [custom_http](_examples/custom_http/main.go) | +| Scheme | Status | Notes | Examples | +| -------------------- | ------------ | ---------------------------------------------------------------------- | ---------------------------------------------- | +| `http(s)://` (dumb) | ❌ | | | +| `http(s)://` (smart) | ✅ | | | +| `git://` | ✅ | | | +| `ssh://` | ✅ | | | +| `file://` | ⚠️ (partial) | Warning: this is not pure Golang. This shells out to the `git` binary. | | +| Custom | ✅ | All existing schemes can be replaced by custom implementations. | - [custom_http](_examples/custom_http/main.go) | ## SHA256 -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `init` | | ✅ | Requires building with tag sha256. | - [init](_examples/sha256/main.go) | -| `commit` | | ✅ | Requires building with tag sha256. | - [commit](_examples/sha256/main.go) | -| `pull` | | ❌ | | | -| `fetch` | | ❌ | | | -| `push` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| -------- | ----------- | ------ | ---------------------------------- | ------------------------------------ | +| `init` | | ✅ | Requires building with tag sha256. | - [init](_examples/sha256/main.go) | +| `commit` | | ✅ | Requires building with tag sha256. | - [commit](_examples/sha256/main.go) | +| `pull` | | ❌ | | | +| `fetch` | | ❌ | | | +| `push` | | ❌ | | | ## Other features -| Feature | Sub-feature | Status | Notes | Examples | -|---|---|---|---|---| -| `config` | `--local` | ✅ | Read and write per-repository (`.git/config`). | | -| `config` | `--global`
`--system` | ✅ | Read-only. | | -| `gitignore` | | ✅ | | | -| `gitattributes` | | ✅ | | | -| `git-worktree` | | ❌ | Multiple worktrees are not supported. | | +| Feature | Sub-feature | Status | Notes | Examples | +| --------------- | --------------------------- | ------ | ---------------------------------------------- | -------- | +| `config` | `--local` | ✅ | Read and write per-repository (`.git/config`). | | +| `config` | `--global`
`--system` | ✅ | Read-only. | | +| `gitignore` | | ✅ | | | +| `gitattributes` | | ✅ | | | +| `git-worktree` | | ❌ | Multiple worktrees are not supported. | | diff --git a/options.go b/options.go index c68bf2bed..8902b7e3e 100644 --- a/options.go +++ b/options.go @@ -78,6 +78,15 @@ type CloneOptions struct { CABundle []byte // ProxyOptions provides info required for connecting to a proxy. ProxyOptions transport.ProxyOptions + // When the repository to clone is on the local machine, instead of + // using hard links, automatically setup .git/objects/info/alternates + // to share the objects with the source repository. + // The resulting repository starts out without any object of its own. + // NOTE: this is a possibly dangerous operation; do not use it unless + // you understand what it does. + // + // [Reference]: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---shared + Shared bool } // Validate validates the fields and sets the default values. diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index d8a9c27a6..126b3742d 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -42,6 +42,7 @@ type EncodedObjectStorer interface { HasEncodedObject(plumbing.Hash) error // EncodedObjectSize returns the plaintext size of the encoded object. EncodedObjectSize(plumbing.Hash) (int64, error) + AddAlternate(remote string) error } // DeltaObjectStorer is an EncodedObjectStorer that can return delta diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index 30424ffd3..f2e6a5e05 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -168,3 +168,7 @@ func (o *MockObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (EncodedOb func (o *MockObjectStorage) Begin() Transaction { return nil } + +func (o *MockObjectStorage) AddAlternate(remote string) error { + return nil +} diff --git a/repository.go b/repository.go index 013b53f80..48988383d 100644 --- a/repository.go +++ b/repository.go @@ -22,6 +22,7 @@ import ( "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/internal/path_util" "github.com/go-git/go-git/v5/internal/revision" + "github.com/go-git/go-git/v5/internal/url" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" formatcfg "github.com/go-git/go-git/v5/plumbing/format/config" @@ -62,6 +63,7 @@ var ( ErrUnableToResolveCommit = errors.New("unable to resolve commit") ErrPackedObjectsNotSupported = errors.New("packed objects not supported") ErrSHA256NotSupported = errors.New("go-git was not compiled with SHA256 support") + ErrAlternatePathNotSupported = errors.New("alternate path must use the file scheme") ) // Repository represents a git repository @@ -887,6 +889,30 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { return err } + // When the repository to clone is on the local machine, + // instead of using hard links, automatically setup .git/objects/info/alternates + // to share the objects with the source repository + if o.Shared { + if !url.IsLocalEndpoint(o.URL) { + return ErrAlternatePathNotSupported + } + altpath := o.URL + remoteRepo, err := PlainOpen(o.URL) + if err != nil { + return fmt.Errorf("failed to open remote repository: %w", err) + } + conf, err := remoteRepo.Config() + if err != nil { + return fmt.Errorf("failed to read remote repository configuration: %w", err) + } + if !conf.Core.IsBare { + altpath = path.Join(altpath, GitDirName) + } + if err := r.Storer.AddAlternate(altpath); err != nil { + return fmt.Errorf("failed to add alternate file to git objects dir: %w", err) + } + } + ref, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ RefSpecs: c.Fetch, Depth: o.Depth, diff --git a/repository_test.go b/repository_test.go index 3154f1d1c..f6839b6da 100644 --- a/repository_test.go +++ b/repository_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "os/user" + "path" "path/filepath" "regexp" "strings" @@ -791,6 +792,101 @@ func (s *RepositorySuite) TestPlainClone(c *C) { c.Assert(cfg.Branches["master"].Name, Equals, "master") } +func (s *RepositorySuite) TestPlainCloneBareAndShared(c *C) { + dir, clean := s.TemporalDir() + defer clean() + + remote := s.GetBasicLocalRepositoryURL() + + r, err := PlainClone(dir, true, &CloneOptions{ + URL: remote, + Shared: true, + }) + c.Assert(err, IsNil) + + altpath := path.Join(dir, "objects", "info", "alternates") + _, err = os.Stat(altpath) + c.Assert(err, IsNil) + + data, err := os.ReadFile(altpath) + c.Assert(err, IsNil) + + line := path.Join(remote, GitDirName, "objects") + "\n" + c.Assert(string(data), Equals, line) + + cfg, err := r.Config() + c.Assert(err, IsNil) + c.Assert(cfg.Branches, HasLen, 1) + c.Assert(cfg.Branches["master"].Name, Equals, "master") +} + +func (s *RepositorySuite) TestPlainCloneShared(c *C) { + dir, clean := s.TemporalDir() + defer clean() + + remote := s.GetBasicLocalRepositoryURL() + + r, err := PlainClone(dir, false, &CloneOptions{ + URL: remote, + Shared: true, + }) + c.Assert(err, IsNil) + + altpath := path.Join(dir, GitDirName, "objects", "info", "alternates") + _, err = os.Stat(altpath) + c.Assert(err, IsNil) + + data, err := os.ReadFile(altpath) + c.Assert(err, IsNil) + + line := path.Join(remote, GitDirName, "objects") + "\n" + c.Assert(string(data), Equals, line) + + cfg, err := r.Config() + c.Assert(err, IsNil) + c.Assert(cfg.Branches, HasLen, 1) + c.Assert(cfg.Branches["master"].Name, Equals, "master") +} + +func (s *RepositorySuite) TestPlainCloneSharedHttpShouldReturnError(c *C) { + dir, clean := s.TemporalDir() + defer clean() + + remote := "http://somerepo" + + _, err := PlainClone(dir, false, &CloneOptions{ + URL: remote, + Shared: true, + }) + c.Assert(err, Equals, ErrAlternatePathNotSupported) +} + +func (s *RepositorySuite) TestPlainCloneSharedHttpsShouldReturnError(c *C) { + dir, clean := s.TemporalDir() + defer clean() + + remote := "https://somerepo" + + _, err := PlainClone(dir, false, &CloneOptions{ + URL: remote, + Shared: true, + }) + c.Assert(err, Equals, ErrAlternatePathNotSupported) +} + +func (s *RepositorySuite) TestPlainCloneSharedSSHShouldReturnError(c *C) { + dir, clean := s.TemporalDir() + defer clean() + + remote := "ssh://somerepo" + + _, err := PlainClone(dir, false, &CloneOptions{ + URL: remote, + Shared: true, + }) + c.Assert(err, Equals, ErrAlternatePathNotSupported) +} + func (s *RepositorySuite) TestPlainCloneWithRemoteName(c *C) { dir, clean := s.TemporalDir() defer clean() diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index e02e6ddfd..3080e4acc 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -8,7 +8,9 @@ import ( "fmt" "io" "os" + "path" "path/filepath" + "runtime" "sort" "strings" "time" @@ -38,6 +40,7 @@ const ( remotesPath = "remotes" logsPath = "logs" worktreesPath = "worktrees" + alternatesPath = "alternates" tmpPackedRefsPrefix = "._packed-refs" @@ -1105,10 +1108,38 @@ func (d *DotGit) Module(name string) (billy.Filesystem, error) { return d.fs.Chroot(d.fs.Join(modulePath, name)) } +func (d *DotGit) AddAlternate(remote string) error { + altpath := d.fs.Join(objectsPath, infoPath, alternatesPath) + + f, err := d.fs.OpenFile(altpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return fmt.Errorf("cannot open file: %w", err) + } + defer f.Close() + + // locking in windows throws an error, based on comments + // https://github.com/go-git/go-git/pull/860#issuecomment-1751823044 + // do not lock on windows platform. + if runtime.GOOS != "windows" { + if err = f.Lock(); err != nil { + return fmt.Errorf("cannot lock file: %w", err) + } + defer f.Unlock() + } + + line := path.Join(remote, objectsPath) + "\n" + _, err = io.WriteString(f, line) + if err != nil { + return fmt.Errorf("error writing 'alternates' file: %w", err) + } + + return nil +} + // Alternates returns DotGit(s) based off paths in objects/info/alternates if // available. This can be used to checks if it's a shared repository. func (d *DotGit) Alternates() ([]*DotGit, error) { - altpath := d.fs.Join("objects", "info", "alternates") + altpath := d.fs.Join(objectsPath, infoPath, alternatesPath) f, err := d.fs.Open(altpath) if err != nil { return nil, err diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 7e7a2c50f..2069d3a6f 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -74,3 +74,7 @@ func (s *Storage) Filesystem() billy.Filesystem { func (s *Storage) Init() error { return s.dir.Initialize() } + +func (s *Storage) AddAlternate(remote string) error { + return s.dir.AddAlternate(remote) +} diff --git a/storage/memory/storage.go b/storage/memory/storage.go index ef6a44551..79211c7c0 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -202,6 +202,10 @@ func (o *ObjectStorage) DeleteLooseObject(plumbing.Hash) error { return errNotSupported } +func (o *ObjectStorage) AddAlternate(remote string) error { + return errNotSupported +} + type TxObjectStorage struct { Storage *ObjectStorage Objects map[plumbing.Hash]plumbing.EncodedObject diff --git a/storage/transactional/object.go b/storage/transactional/object.go index 5d102b0e1..b43c96d3b 100644 --- a/storage/transactional/object.go +++ b/storage/transactional/object.go @@ -82,3 +82,7 @@ func (o *ObjectStorage) Commit() error { return err }) } + +func (o *ObjectStorage) AddAlternate(remote string) error { + return o.temporal.AddAlternate(remote) +} From 946bb8183643bdda90810fc48e450a008894b244 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 30 Sep 2023 11:41:23 +0100 Subject: [PATCH 14/56] plumbing: commitgraph, fix types and handle commit-graph-chains Unfortunately the original variant makes some incorrect typing assumptions about commit-graphs which make handling graph chains difficult to do correctly. This creates a new subpackage and deprecates the old one. It then adds support commit graph chains. Signed-off-by: Andrew Thornton --- go.mod | 4 +- go.sum | 16 +- plumbing/format/commitgraph/commitgraph.go | 6 + plumbing/format/commitgraph/doc.go | 113 +++--- plumbing/format/commitgraph/encoder.go | 9 + plumbing/format/commitgraph/file.go | 6 + plumbing/format/commitgraph/memory.go | 6 + plumbing/format/commitgraph/v2/chain.go | 100 ++++++ plumbing/format/commitgraph/v2/chain_test.go | 100 ++++++ plumbing/format/commitgraph/v2/chunk.go | 48 +++ plumbing/format/commitgraph/v2/commitgraph.go | 40 +++ .../format/commitgraph/v2/commitgraph_test.go | 165 +++++++++ plumbing/format/commitgraph/v2/doc.go | 106 ++++++ plumbing/format/commitgraph/v2/encoder.go | 192 ++++++++++ plumbing/format/commitgraph/v2/file.go | 338 ++++++++++++++++++ plumbing/format/commitgraph/v2/memory.go | 91 +++++ .../object/commitgraph/commitnode_graph.go | 264 +++++++------- .../object/commitgraph/commitnode_object.go | 180 +++++----- .../object/commitgraph/commitnode_test.go | 9 +- .../commitgraph/commitnode_walker_ctime.go | 210 +++++------ 20 files changed, 1607 insertions(+), 396 deletions(-) create mode 100644 plumbing/format/commitgraph/v2/chain.go create mode 100644 plumbing/format/commitgraph/v2/chain_test.go create mode 100644 plumbing/format/commitgraph/v2/chunk.go create mode 100644 plumbing/format/commitgraph/v2/commitgraph.go create mode 100644 plumbing/format/commitgraph/v2/commitgraph_test.go create mode 100644 plumbing/format/commitgraph/v2/doc.go create mode 100644 plumbing/format/commitgraph/v2/encoder.go create mode 100644 plumbing/format/commitgraph/v2/file.go create mode 100644 plumbing/format/commitgraph/v2/memory.go diff --git a/go.mod b/go.mod index 8de45b2be..9df2020f1 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/gliderlabs/ssh v0.3.5 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 github.com/go-git/go-billy/v5 v5.5.0 - github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f + github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/go-cmp v0.5.9 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 @@ -25,7 +25,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.13.0 golang.org/x/net v0.15.0 - golang.org/x/sys v0.12.0 + golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) diff --git a/go.sum b/go.sum index 8deb90eaa..7c62b1094 100644 --- a/go.sum +++ b/go.sum @@ -30,11 +30,10 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389 h1:AlfdJ8f+G+4a4fXeHmAlKfyR3Yup4sVGCXlh+e+TrE8= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= @@ -55,7 +54,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= @@ -76,8 +74,7 @@ github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2Iqp github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -109,7 +106,6 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -124,8 +120,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -151,13 +147,11 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/plumbing/format/commitgraph/commitgraph.go b/plumbing/format/commitgraph/commitgraph.go index 3d59323f3..e772d2636 100644 --- a/plumbing/format/commitgraph/commitgraph.go +++ b/plumbing/format/commitgraph/commitgraph.go @@ -8,6 +8,9 @@ import ( // CommitData is a reduced representation of Commit as presented in the commit graph // file. It is merely useful as an optimization for walking the commit graphs. +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. type CommitData struct { // TreeHash is the hash of the root tree of the commit. TreeHash plumbing.Hash @@ -24,6 +27,9 @@ type CommitData struct { // Index represents a representation of commit graph that allows indexed // access to the nodes using commit object hash +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. type Index interface { // GetIndexByHash gets the index in the commit graph from commit hash, if available GetIndexByHash(h plumbing.Hash) (int, error) diff --git a/plumbing/format/commitgraph/doc.go b/plumbing/format/commitgraph/doc.go index 41cd8b1e3..c320e1811 100644 --- a/plumbing/format/commitgraph/doc.go +++ b/plumbing/format/commitgraph/doc.go @@ -1,23 +1,26 @@ // Package commitgraph implements encoding and decoding of commit-graph files. // +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. +// // Git commit graph format // ======================= // // The Git commit graph stores a list of commit OIDs and some associated // metadata, including: // -// - The generation number of the commit. Commits with no parents have -// generation number 1; commits with parents have generation number -// one more than the maximum generation number of its parents. We -// reserve zero as special, and can be used to mark a generation -// number invalid or as "not computed". +// - The generation number of the commit. Commits with no parents have +// generation number 1; commits with parents have generation number +// one more than the maximum generation number of its parents. We +// reserve zero as special, and can be used to mark a generation +// number invalid or as "not computed". // // - The root tree OID. // // - The commit date. // -// - The parents of the commit, stored using positional references within -// the graph file. +// - The parents of the commit, stored using positional references within +// the graph file. // // These positional references are stored as unsigned 32-bit integers // corresponding to the array position within the list of commit OIDs. Due @@ -35,68 +38,68 @@ // // HEADER: // -// 4-byte signature: -// The signature is: {'C', 'G', 'P', 'H'} +// 4-byte signature: +// The signature is: {'C', 'G', 'P', 'H'} // -// 1-byte version number: -// Currently, the only valid version is 1. +// 1-byte version number: +// Currently, the only valid version is 1. // -// 1-byte Hash Version (1 = SHA-1) -// We infer the hash length (H) from this value. +// 1-byte Hash Version (1 = SHA-1) +// We infer the hash length (H) from this value. // -// 1-byte number (C) of "chunks" +// 1-byte number (C) of "chunks" // -// 1-byte (reserved for later use) -// Current clients should ignore this value. +// 1-byte (reserved for later use) +// Current clients should ignore this value. // // CHUNK LOOKUP: // -// (C + 1) * 12 bytes listing the table of contents for the chunks: -// First 4 bytes describe the chunk id. Value 0 is a terminating label. -// Other 8 bytes provide the byte-offset in current file for chunk to -// start. (Chunks are ordered contiguously in the file, so you can infer -// the length using the next chunk position if necessary.) Each chunk -// ID appears at most once. +// (C + 1) * 12 bytes listing the table of contents for the chunks: +// First 4 bytes describe the chunk id. Value 0 is a terminating label. +// Other 8 bytes provide the byte-offset in current file for chunk to +// start. (Chunks are ordered contiguously in the file, so you can infer +// the length using the next chunk position if necessary.) Each chunk +// ID appears at most once. // -// The remaining data in the body is described one chunk at a time, and -// these chunks may be given in any order. Chunks are required unless -// otherwise specified. +// The remaining data in the body is described one chunk at a time, and +// these chunks may be given in any order. Chunks are required unless +// otherwise specified. // // CHUNK DATA: // -// OID Fanout (ID: {'O', 'I', 'D', 'F'}) (256 * 4 bytes) -// The ith entry, F[i], stores the number of OIDs with first -// byte at most i. Thus F[255] stores the total -// number of commits (N). -// -// OID Lookup (ID: {'O', 'I', 'D', 'L'}) (N * H bytes) -// The OIDs for all commits in the graph, sorted in ascending order. -// -// Commit Data (ID: {'C', 'D', 'A', 'T' }) (N * (H + 16) bytes) -// * The first H bytes are for the OID of the root tree. -// * The next 8 bytes are for the positions of the first two parents -// of the ith commit. Stores value 0x7000000 if no parent in that -// position. If there are more than two parents, the second value -// has its most-significant bit on and the other bits store an array -// position into the Extra Edge List chunk. -// * The next 8 bytes store the generation number of the commit and -// the commit time in seconds since EPOCH. The generation number -// uses the higher 30 bits of the first 4 bytes, while the commit -// time uses the 32 bits of the second 4 bytes, along with the lowest -// 2 bits of the lowest byte, storing the 33rd and 34th bit of the -// commit time. -// -// Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional] -// This list of 4-byte values store the second through nth parents for -// all octopus merges. The second parent value in the commit data stores -// an array position within this list along with the most-significant bit -// on. Starting at that array position, iterate through this list of commit -// positions for the parents until reaching a value with the most-significant -// bit on. The other bits correspond to the position of the last parent. +// OID Fanout (ID: {'O', 'I', 'D', 'F'}) (256 * 4 bytes) +// The ith entry, F[i], stores the number of OIDs with first +// byte at most i. Thus F[255] stores the total +// number of commits (N). +// +// OID Lookup (ID: {'O', 'I', 'D', 'L'}) (N * H bytes) +// The OIDs for all commits in the graph, sorted in ascending order. +// +// Commit Data (ID: {'C', 'D', 'A', 'T' }) (N * (H + 16) bytes) +// * The first H bytes are for the OID of the root tree. +// * The next 8 bytes are for the positions of the first two parents +// of the ith commit. Stores value 0x7000000 if no parent in that +// position. If there are more than two parents, the second value +// has its most-significant bit on and the other bits store an array +// position into the Extra Edge List chunk. +// * The next 8 bytes store the generation number of the commit and +// the commit time in seconds since EPOCH. The generation number +// uses the higher 30 bits of the first 4 bytes, while the commit +// time uses the 32 bits of the second 4 bytes, along with the lowest +// 2 bits of the lowest byte, storing the 33rd and 34th bit of the +// commit time. +// +// Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional] +// This list of 4-byte values store the second through nth parents for +// all octopus merges. The second parent value in the commit data stores +// an array position within this list along with the most-significant bit +// on. Starting at that array position, iterate through this list of commit +// positions for the parents until reaching a value with the most-significant +// bit on. The other bits correspond to the position of the last parent. // // TRAILER: // -// H-byte HASH-checksum of all of the above. +// H-byte HASH-checksum of all of the above. // // Source: // https://raw.githubusercontent.com/git/git/master/Documentation/technical/commit-graph-format.txt diff --git a/plumbing/format/commitgraph/encoder.go b/plumbing/format/commitgraph/encoder.go index 674f52e7c..317635384 100644 --- a/plumbing/format/commitgraph/encoder.go +++ b/plumbing/format/commitgraph/encoder.go @@ -10,12 +10,18 @@ import ( ) // Encoder writes MemoryIndex structs to an output stream. +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. type Encoder struct { io.Writer hash hash.Hash } // NewEncoder returns a new stream encoder that writes to w. +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. func NewEncoder(w io.Writer) *Encoder { h := hash.New(hash.CryptoType) mw := io.MultiWriter(w, h) @@ -23,6 +29,9 @@ func NewEncoder(w io.Writer) *Encoder { } // Encode writes an index into the commit-graph file +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. func (e *Encoder) Encode(idx Index) error { // Get all the hashes in the input index hashes := idx.Hashes() diff --git a/plumbing/format/commitgraph/file.go b/plumbing/format/commitgraph/file.go index 17c1c5d11..ef8fb3496 100644 --- a/plumbing/format/commitgraph/file.go +++ b/plumbing/format/commitgraph/file.go @@ -13,6 +13,9 @@ import ( "github.com/go-git/go-git/v5/utils/binary" ) +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. + var ( // ErrUnsupportedVersion is returned by OpenFileIndex when the commit graph // file version is not supported. @@ -51,6 +54,9 @@ type fileIndex struct { // OpenFileIndex opens a serialized commit graph file in the format described at // https://github.com/git/git/blob/master/Documentation/technical/commit-graph-format.txt +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. func OpenFileIndex(reader io.ReaderAt) (Index, error) { fi := &fileIndex{reader: reader} diff --git a/plumbing/format/commitgraph/memory.go b/plumbing/format/commitgraph/memory.go index b24ce36d9..06415e515 100644 --- a/plumbing/format/commitgraph/memory.go +++ b/plumbing/format/commitgraph/memory.go @@ -6,12 +6,18 @@ import ( // MemoryIndex provides a way to build the commit-graph in memory // for later encoding to file. +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. type MemoryIndex struct { commitData []*CommitData indexMap map[plumbing.Hash]int } // NewMemoryIndex creates in-memory commit graph representation +// +// Deprecated: This package uses the wrong types for Generation and Index in CommitData. +// Use the v2 package instead. func NewMemoryIndex() *MemoryIndex { return &MemoryIndex{ indexMap: make(map[plumbing.Hash]int), diff --git a/plumbing/format/commitgraph/v2/chain.go b/plumbing/format/commitgraph/v2/chain.go new file mode 100644 index 000000000..8da60d01b --- /dev/null +++ b/plumbing/format/commitgraph/v2/chain.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "bufio" + "io" + "path" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +// OpenChainFile reads a commit chain file and returns a slice of the hashes within it +// +// Commit-Graph chains are described at https://git-scm.com/docs/commit-graph +// and are new line separated list of graph file hashes, oldest to newest. +// +// This function simply reads the file and returns the hashes as a slice. +func OpenChainFile(r io.Reader) ([]string, error) { + if r == nil { + return nil, io.ErrUnexpectedEOF + } + bufRd := bufio.NewReader(r) + chain := make([]string, 0, 8) + for { + line, err := bufRd.ReadSlice('\n') + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + hashStr := string(line[:len(line)-1]) + if !plumbing.IsHash(hashStr) { + return nil, ErrMalformedCommitGraphFile + } + chain = append(chain, hashStr) + } + return chain, nil +} + +// OpenChainOrFileIndex expects a billy.Filesystem representing a .git directory. +// It will first attempt to read a commit-graph index file, before trying to read a +// commit-graph chain file and its index files. If neither are present, an error is returned. +// Otherwise an Index will be returned. +// +// See: https://git-scm.com/docs/commit-graph +func OpenChainOrFileIndex(fs billy.Filesystem) (Index, error) { + file, err := fs.Open(path.Join("objects", "info", "commit-graph")) + if err != nil { + // try to open a chain file + return OpenChainIndex(fs) + } + + index, err := OpenFileIndex(file) + if err != nil { + // Ignore any file closing errors and return the error from OpenFileIndex instead + _ = file.Close() + return nil, err + } + return index, nil +} + +// OpenChainIndex expects a billy.Filesystem representing a .git directory. +// It will read a commit-graph chain file and return a coalesced index. +// If the chain file or a graph in that chain is not present, an error is returned. +// +// See: https://git-scm.com/docs/commit-graph +func OpenChainIndex(fs billy.Filesystem) (Index, error) { + chainFile, err := fs.Open(path.Join("objects", "info", "commit-graphs", "commit-graph-chain")) + if err != nil { + return nil, err + } + + chain, err := OpenChainFile(chainFile) + _ = chainFile.Close() + if err != nil { + return nil, err + } + + var index Index + for _, hash := range chain { + + file, err := fs.Open(path.Join("objects", "info", "commit-graphs", "graph-"+hash+".graph")) + if err != nil { + // Ignore all other file closing errors and return the error from opening the last file in the graph + _ = index.Close() + return nil, err + } + + index, err = OpenFileIndexWithParent(file, index) + if err != nil { + // Ignore file closing errors and return the error from OpenFileIndex instead + _ = index.Close() + return nil, err + } + } + + return index, nil +} diff --git a/plumbing/format/commitgraph/v2/chain_test.go b/plumbing/format/commitgraph/v2/chain_test.go new file mode 100644 index 000000000..32ffd69e1 --- /dev/null +++ b/plumbing/format/commitgraph/v2/chain_test.go @@ -0,0 +1,100 @@ +package v2_test + +import ( + "bytes" + "crypto" + "strings" + + commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" + "github.com/go-git/go-git/v5/plumbing/hash" + + . "gopkg.in/check.v1" +) + +func (s *CommitgraphSuite) TestOpenChainFile(c *C) { + sha1Data := []string{ + "c336d16298a017486c4164c40f8acb28afe64e84", + "31eae7b619d166c366bf5df4991f04ba8cebea0a", + "b977a025ca21e3b5ca123d8093bd7917694f6da7", + "d2a38b4a5965d529566566640519d03d2bd10f6c", + "35b585759cbf29f8ec428ef89da20705d59f99ec", + "c2bbf9fe8009b22d0f390f3c8c3f13937067590f", + "fc9f0643b21cfe571046e27e0c4565f3a1ee96c8", + "c088fd6a7e1a38e9d5a9815265cb575bb08d08ff", + "5fddbeb678bd2c36c5e5c891ab8f2b143ced5baf", + "5d7303c49ac984a9fec60523f2d5297682e16646", + } + + sha256Data := []string{ + "b9efda7160f2647e0974ca623f8a8f8e25fb6944f1b8f78f4db1bf07932de8eb", + "7095c59f8bf46e12c21d2d9da344cfe383fae18d26f3ae4d4ab7b71e3d0ddfae", + "25a395cb62f7656294e40a001ee19fefcdf3013d265dfcf4b744cd2549891dec", + "7fbd564813a82227507d9dd70f1fd21fc1f180223cd3f42e0c3090c9a8b6a7d0", + "aa95db1db2df91bd7200a892dd1c03bc2704c4793400d016b3ca08c148b0f7c1", + "2176988184b570565dc33823a02f474ad59f667a0e971c86063a7fea64776a87", + "d0afc0e64171140eb7902110f807a1beaa38a603d4312fd4bd14a5db2784ba62", + "2822136f60bfc58bbd9d624cc19fbef9f0fc0efe2a61729242e1e5f9b77fa3d0", + "6f207b5c43463af96bc38c43b0bf45275fa327e656a8bba8e7fc55c5ab6870d8", + "6cf33782619b6ff0af9c081e46323f423f8b49bf3d043887c0549bef47d60f55", + "60ea0753d2d4e828983528294be3f57e2a3ba37df4f59e3236133c9e2b17afc5", + "6b3c9f4ba5092e0807774097953ec6e9f58e8371d775bd8738a0fa98d728ba3d", + "c97cab8564054e30515dbe67dda4e14638aabf17b3f042d18dc8461cd098b362", + "9f7ece76fd2c9dae08e75176347efffc1446ad74af66004dd34680edb205dfb5", + "23e7a7e481b00571b63c2a7d0432f9733dd85d18a9841a3d7b96743100da5824", + "e684b1253fa8eb6572f35bab2fd3b6efecabf8472ede43497cd9c171973cc341", + "8b9f04080b0c40f7ad2a6bb5e5296cd6c06e730dffce87a0375ae7bd0f85f86e", + "384a745f3b14edc89526a98b96b3247b2b548541c755aadee7664352ed7f12ae", + "b68c8a82cd5b839917e1058570a0408819b81d16dbab81db118cc8dfc3def044", + "fbaf04f1a401335be57e172f4326102c658d857fde6cf2bc987520d11fc99770", + "57acf2aa5ac736337b120c951536c8a2b2cb23a4f0f198e86f3433370fa63105", + "dd7fcba4c13b6ced0b6190cdb5861adcd08446a92d67f7ec0f02f9533e09bbb0", + "744ef481c9b13ebd3b6e43d7e9ba25f7c7a5c8e453e6f0d50f5d71aae1591689", + "2c573142f1edd52b64dcd42a9c3b0ca5c9c615f757d80d25bfb02ff3eb2257e2", + "ea65cc58ef8520cd0335de4318a0d3b3a1ac257b7e9f82e12483fa3bce6cc0cd", + "1dfa626ff1523b82e21a4c29476edcdc9a89842f3c7181f63a28cd4f46cc9923", + "aa1153e71af836121e6f6cc716cf64880c19221d8dc367ff42359de1b8ef30e9", + "a7c6ec6f6569e22d2fa6e8281639d27c59b633ea00ad8ef27a43171cc985fbda", + "627b706d63d2cfd5a388deeaa76655ef09146fe492ee17cb0043578cef9c2800", + "d40eaf091ef8357b734d1047a552436eaf057d99a0c6f2068b097c324099d360", + "87f0ef81641da4fd3438dcaae4819f0c92a0ade54e262b21f9ded4575ff3f234", + "3a00a29e08d29454b5197662f70ccab5699b0ce8c85af7fbf511b8915d97cfd0", + } + + goodShas := sha1Data + badShas := sha256Data + if hash.CryptoType == crypto.SHA256 { + goodShas = sha256Data + badShas = sha1Data + } + chainData := strings.Join(goodShas, "\n") + "\n" + + chainReader := strings.NewReader(chainData) + + chain, err := commitgraph.OpenChainFile(chainReader) + c.Assert(err, IsNil) + c.Assert(goodShas, DeepEquals, chain) + + // Test with bad shas + chainData = strings.Join(badShas, "\n") + "\n" + + chainReader = strings.NewReader(chainData) + + chain, err = commitgraph.OpenChainFile(chainReader) + c.Assert(err, Equals, commitgraph.ErrMalformedCommitGraphFile) + c.Assert(chain, IsNil) + + // Test with empty file + emptyChainReader := bytes.NewReader(nil) + + chain, err = commitgraph.OpenChainFile(emptyChainReader) + c.Assert(err, IsNil) + c.Assert(chain, DeepEquals, []string{}) + + // Test with file containing only newlines + newlineChainData := []byte("\n\n\n") + newlineChainReader := bytes.NewReader(newlineChainData) + + chain, err = commitgraph.OpenChainFile(newlineChainReader) + c.Assert(err, Equals, commitgraph.ErrMalformedCommitGraphFile) + c.Assert(chain, IsNil) +} diff --git a/plumbing/format/commitgraph/v2/chunk.go b/plumbing/format/commitgraph/v2/chunk.go new file mode 100644 index 000000000..ab2432072 --- /dev/null +++ b/plumbing/format/commitgraph/v2/chunk.go @@ -0,0 +1,48 @@ +package v2 + +import "bytes" + +const ( + chunkSigLen = 4 // Length of a chunk signature + chunkSigOffset = 4 // Offset of each chunk signature in chunkSignatures +) + +// chunkSignatures contains the coalesced byte signatures for each chunk type. +// The order of the signatures must match the order of the ChunkType constants. +// (When adding new chunk types you must avoid introducing ambiguity, and you may need to add padding separators to this list or reorder these signatures.) +// (i.e. it would not be possible to add a new chunk type with the signature "IDFO" without some reordering or the addition of separators.) +var chunkSignatures = []byte("OIDFOIDLCDATGDA2GDO2EDGEBIDXBDATBASE\000\000\000\000") + +// ChunkType represents the type of a chunk in the commit graph file. +type ChunkType int + +const ( + OIDFanoutChunk ChunkType = iota // "OIDF" + OIDLookupChunk // "OIDL" + CommitDataChunk // "CDAT" + GenerationDataChunk // "GDA2" + GenerationDataOverflowChunk // "GDO2" + ExtraEdgeListChunk // "EDGE" + BloomFilterIndexChunk // "BIDX" + BloomFilterDataChunk // "BDAT" + BaseGraphsListChunk // "BASE" + ZeroChunk // "\000\000\000\000" +) + +// Signature returns the byte signature for the chunk type. +func (ct ChunkType) Signature() []byte { + if ct >= BaseGraphsListChunk || ct < 0 { // not a valid chunk type just return ZeroChunk + return chunkSignatures[ZeroChunk*chunkSigOffset : ZeroChunk*chunkSigOffset+chunkSigLen] + } + + return chunkSignatures[ct*chunkSigOffset : ct*chunkSigOffset+chunkSigLen] +} + +// ChunkTypeFromBytes returns the chunk type for the given byte signature. +func ChunkTypeFromBytes(b []byte) (ChunkType, bool) { + idx := bytes.Index(chunkSignatures, b) + if idx == -1 || idx%chunkSigOffset != 0 { // not found, or not aligned at chunkSigOffset + return -1, false + } + return ChunkType(idx / chunkSigOffset), true +} diff --git a/plumbing/format/commitgraph/v2/commitgraph.go b/plumbing/format/commitgraph/v2/commitgraph.go new file mode 100644 index 000000000..7c67b6395 --- /dev/null +++ b/plumbing/format/commitgraph/v2/commitgraph.go @@ -0,0 +1,40 @@ +package v2 + +import ( + "io" + "time" + + "github.com/go-git/go-git/v5/plumbing" +) + +// CommitData is a reduced representation of Commit as presented in the commit graph +// file. It is merely useful as an optimization for walking the commit graphs. +type CommitData struct { + // TreeHash is the hash of the root tree of the commit. + TreeHash plumbing.Hash + // ParentIndexes are the indexes of the parent commits of the commit. + ParentIndexes []uint32 + // ParentHashes are the hashes of the parent commits of the commit. + ParentHashes []plumbing.Hash + // Generation number is the pre-computed generation in the commit graph + // or zero if not available. + Generation uint64 + // When is the timestamp of the commit. + When time.Time +} + +// Index represents a representation of commit graph that allows indexed +// access to the nodes using commit object hash +type Index interface { + // GetIndexByHash gets the index in the commit graph from commit hash, if available + GetIndexByHash(h plumbing.Hash) (uint32, error) + // GetHashByIndex gets the hash given an index in the commit graph + GetHashByIndex(i uint32) (plumbing.Hash, error) + // GetNodeByIndex gets the commit node from the commit graph using index + // obtained from child node, if available + GetCommitDataByIndex(i uint32) (*CommitData, error) + // Hashes returns all the hashes that are available in the index + Hashes() []plumbing.Hash + + io.Closer +} diff --git a/plumbing/format/commitgraph/v2/commitgraph_test.go b/plumbing/format/commitgraph/v2/commitgraph_test.go new file mode 100644 index 000000000..69fdcd9fd --- /dev/null +++ b/plumbing/format/commitgraph/v2/commitgraph_test.go @@ -0,0 +1,165 @@ +package v2_test + +import ( + "os" + "testing" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/util" + "github.com/go-git/go-git/v5/plumbing" + commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" + + fixtures "github.com/go-git/go-git-fixtures/v4" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type CommitgraphSuite struct { + fixtures.Suite +} + +var _ = Suite(&CommitgraphSuite{}) + +func testReadIndex(c *C, fs billy.Filesystem, path string) commitgraph.Index { + reader, err := fs.Open(path) + c.Assert(err, IsNil) + index, err := commitgraph.OpenFileIndex(reader) + c.Assert(err, IsNil) + c.Assert(index, NotNil) + return index +} + +func testDecodeHelper(c *C, index commitgraph.Index) { + // Root commit + nodeIndex, err := index.GetIndexByHash(plumbing.NewHash("347c91919944a68e9413581a1bc15519550a3afe")) + c.Assert(err, IsNil) + commitData, err := index.GetCommitDataByIndex(nodeIndex) + c.Assert(err, IsNil) + c.Assert(len(commitData.ParentIndexes), Equals, 0) + c.Assert(len(commitData.ParentHashes), Equals, 0) + + // Regular commit + nodeIndex, err = index.GetIndexByHash(plumbing.NewHash("e713b52d7e13807e87a002e812041f248db3f643")) + c.Assert(err, IsNil) + commitData, err = index.GetCommitDataByIndex(nodeIndex) + c.Assert(err, IsNil) + c.Assert(len(commitData.ParentIndexes), Equals, 1) + c.Assert(len(commitData.ParentHashes), Equals, 1) + c.Assert(commitData.ParentHashes[0].String(), Equals, "347c91919944a68e9413581a1bc15519550a3afe") + + // Merge commit + nodeIndex, err = index.GetIndexByHash(plumbing.NewHash("b29328491a0682c259bcce28741eac71f3499f7d")) + c.Assert(err, IsNil) + commitData, err = index.GetCommitDataByIndex(nodeIndex) + c.Assert(err, IsNil) + c.Assert(len(commitData.ParentIndexes), Equals, 2) + c.Assert(len(commitData.ParentHashes), Equals, 2) + c.Assert(commitData.ParentHashes[0].String(), Equals, "e713b52d7e13807e87a002e812041f248db3f643") + c.Assert(commitData.ParentHashes[1].String(), Equals, "03d2c021ff68954cf3ef0a36825e194a4b98f981") + + // Octopus merge commit + nodeIndex, err = index.GetIndexByHash(plumbing.NewHash("6f6c5d2be7852c782be1dd13e36496dd7ad39560")) + c.Assert(err, IsNil) + commitData, err = index.GetCommitDataByIndex(nodeIndex) + c.Assert(err, IsNil) + c.Assert(len(commitData.ParentIndexes), Equals, 3) + c.Assert(len(commitData.ParentHashes), Equals, 3) + c.Assert(commitData.ParentHashes[0].String(), Equals, "ce275064ad67d51e99f026084e20827901a8361c") + c.Assert(commitData.ParentHashes[1].String(), Equals, "bb13916df33ed23004c3ce9ed3b8487528e655c1") + c.Assert(commitData.ParentHashes[2].String(), Equals, "a45273fe2d63300e1962a9e26a6b15c276cd7082") + + // Check all hashes + hashes := index.Hashes() + c.Assert(len(hashes), Equals, 11) + c.Assert(hashes[0].String(), Equals, "03d2c021ff68954cf3ef0a36825e194a4b98f981") + c.Assert(hashes[10].String(), Equals, "e713b52d7e13807e87a002e812041f248db3f643") +} + +func (s *CommitgraphSuite) TestDecode(c *C) { + fixtures.ByTag("commit-graph").Test(c, func(f *fixtures.Fixture) { + dotgit := f.DotGit() + index := testReadIndex(c, dotgit, dotgit.Join("objects", "info", "commit-graph")) + defer index.Close() + testDecodeHelper(c, index) + }) +} + +func (s *CommitgraphSuite) TestDecodeChain(c *C) { + fixtures.ByTag("commit-graph").Test(c, func(f *fixtures.Fixture) { + dotgit := f.DotGit() + index, err := commitgraph.OpenChainOrFileIndex(dotgit) + c.Assert(err, IsNil) + defer index.Close() + testDecodeHelper(c, index) + }) + + fixtures.ByTag("commit-graph-chain").Test(c, func(f *fixtures.Fixture) { + dotgit := f.DotGit() + index, err := commitgraph.OpenChainOrFileIndex(dotgit) + c.Assert(err, IsNil) + defer index.Close() + testDecodeHelper(c, index) + }) +} + +func (s *CommitgraphSuite) TestReencode(c *C) { + fixtures.ByTag("commit-graph").Test(c, func(f *fixtures.Fixture) { + dotgit := f.DotGit() + + reader, err := dotgit.Open(dotgit.Join("objects", "info", "commit-graph")) + c.Assert(err, IsNil) + defer reader.Close() + index, err := commitgraph.OpenFileIndex(reader) + c.Assert(err, IsNil) + defer index.Close() + + writer, err := util.TempFile(dotgit, "", "commit-graph") + c.Assert(err, IsNil) + tmpName := writer.Name() + defer os.Remove(tmpName) + + encoder := commitgraph.NewEncoder(writer) + err = encoder.Encode(index) + c.Assert(err, IsNil) + writer.Close() + + tmpIndex := testReadIndex(c, dotgit, tmpName) + defer tmpIndex.Close() + testDecodeHelper(c, tmpIndex) + }) +} + +func (s *CommitgraphSuite) TestReencodeInMemory(c *C) { + fixtures.ByTag("commit-graph").Test(c, func(f *fixtures.Fixture) { + dotgit := f.DotGit() + + reader, err := dotgit.Open(dotgit.Join("objects", "info", "commit-graph")) + c.Assert(err, IsNil) + index, err := commitgraph.OpenFileIndex(reader) + c.Assert(err, IsNil) + + memoryIndex := commitgraph.NewMemoryIndex() + defer memoryIndex.Close() + for i, hash := range index.Hashes() { + commitData, err := index.GetCommitDataByIndex(uint32(i)) + c.Assert(err, IsNil) + memoryIndex.Add(hash, commitData) + } + index.Close() + + writer, err := util.TempFile(dotgit, "", "commit-graph") + c.Assert(err, IsNil) + tmpName := writer.Name() + defer os.Remove(tmpName) + + encoder := commitgraph.NewEncoder(writer) + err = encoder.Encode(memoryIndex) + c.Assert(err, IsNil) + writer.Close() + + tmpIndex := testReadIndex(c, dotgit, tmpName) + defer tmpIndex.Close() + testDecodeHelper(c, tmpIndex) + }) +} diff --git a/plumbing/format/commitgraph/v2/doc.go b/plumbing/format/commitgraph/v2/doc.go new file mode 100644 index 000000000..157621dc2 --- /dev/null +++ b/plumbing/format/commitgraph/v2/doc.go @@ -0,0 +1,106 @@ +// Package v2 implements encoding and decoding of commit-graph files. +// +// This package was created to work around the issues of the incorrect types in +// the commitgraph package. +// +// Git commit graph format +// ======================= +// +// The Git commit graph stores a list of commit OIDs and some associated +// metadata, including: +// +// - The generation number of the commit. Commits with no parents have +// generation number 1; commits with parents have generation number +// one more than the maximum generation number of its parents. We +// reserve zero as special, and can be used to mark a generation +// number invalid or as "not computed". +// +// - The root tree OID. +// +// - The commit date. +// +// - The parents of the commit, stored using positional references within +// the graph file. +// +// These positional references are stored as unsigned 32-bit integers +// corresponding to the array position within the list of commit OIDs. Due +// to some special constants we use to track parents, we can store at most +// (1 << 30) + (1 << 29) + (1 << 28) - 1 (around 1.8 billion) commits. +// +// == Commit graph files have the following format: +// +// In order to allow extensions that add extra data to the graph, we organize +// the body into "chunks" and provide a binary lookup table at the beginning +// of the body. The header includes certain values, such as number of chunks +// and hash type. +// +// All 4-byte numbers are in network order. +// +// HEADER: +// +// 4-byte signature: +// The signature is: {'C', 'G', 'P', 'H'} +// +// 1-byte version number: +// Currently, the only valid version is 1. +// +// 1-byte Hash Version (1 = SHA-1) +// We infer the hash length (H) from this value. +// +// 1-byte number (C) of "chunks" +// +// 1-byte (reserved for later use) +// Current clients should ignore this value. +// +// CHUNK LOOKUP: +// +// (C + 1) * 12 bytes listing the table of contents for the chunks: +// First 4 bytes describe the chunk id. Value 0 is a terminating label. +// Other 8 bytes provide the byte-offset in current file for chunk to +// start. (Chunks are ordered contiguously in the file, so you can infer +// the length using the next chunk position if necessary.) Each chunk +// ID appears at most once. +// +// The remaining data in the body is described one chunk at a time, and +// these chunks may be given in any order. Chunks are required unless +// otherwise specified. +// +// CHUNK DATA: +// +// OID Fanout (ID: {'O', 'I', 'D', 'F'}) (256 * 4 bytes) +// The ith entry, F[i], stores the number of OIDs with first +// byte at most i. Thus F[255] stores the total +// number of commits (N). +// +// OID Lookup (ID: {'O', 'I', 'D', 'L'}) (N * H bytes) +// The OIDs for all commits in the graph, sorted in ascending order. +// +// Commit Data (ID: {'C', 'D', 'A', 'T' }) (N * (H + 16) bytes) +// * The first H bytes are for the OID of the root tree. +// * The next 8 bytes are for the positions of the first two parents +// of the ith commit. Stores value 0x7000000 if no parent in that +// position. If there are more than two parents, the second value +// has its most-significant bit on and the other bits store an array +// position into the Extra Edge List chunk. +// * The next 8 bytes store the generation number of the commit and +// the commit time in seconds since EPOCH. The generation number +// uses the higher 30 bits of the first 4 bytes, while the commit +// time uses the 32 bits of the second 4 bytes, along with the lowest +// 2 bits of the lowest byte, storing the 33rd and 34th bit of the +// commit time. +// +// Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional] +// This list of 4-byte values store the second through nth parents for +// all octopus merges. The second parent value in the commit data stores +// an array position within this list along with the most-significant bit +// on. Starting at that array position, iterate through this list of commit +// positions for the parents until reaching a value with the most-significant +// bit on. The other bits correspond to the position of the last parent. +// +// TRAILER: +// +// H-byte HASH-checksum of all of the above. +// +// Source: +// https://raw.githubusercontent.com/git/git/master/Documentation/technical/commit-graph-format.txt +package v2 diff --git a/plumbing/format/commitgraph/v2/encoder.go b/plumbing/format/commitgraph/v2/encoder.go new file mode 100644 index 000000000..d1e41f892 --- /dev/null +++ b/plumbing/format/commitgraph/v2/encoder.go @@ -0,0 +1,192 @@ +package v2 + +import ( + "crypto" + "io" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" + "github.com/go-git/go-git/v5/utils/binary" +) + +// Encoder writes MemoryIndex structs to an output stream. +type Encoder struct { + io.Writer + hash hash.Hash +} + +// NewEncoder returns a new stream encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + h := hash.New(hash.CryptoType) + mw := io.MultiWriter(w, h) + return &Encoder{mw, h} +} + +// Encode writes an index into the commit-graph file +func (e *Encoder) Encode(idx Index) error { + // Get all the hashes in the input index + hashes := idx.Hashes() + + // Sort the inout and prepare helper structures we'll need for encoding + hashToIndex, fanout, extraEdgesCount := e.prepare(idx, hashes) + + chunkSignatures := [][]byte{OIDFanoutChunk.Signature(), OIDLookupChunk.Signature(), CommitDataChunk.Signature()} + chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * (hash.Size + commitDataSize)} + if extraEdgesCount > 0 { + chunkSignatures = append(chunkSignatures, ExtraEdgeListChunk.Signature()) + chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*4) + } + + if err := e.encodeFileHeader(len(chunkSignatures)); err != nil { + return err + } + if err := e.encodeChunkHeaders(chunkSignatures, chunkSizes); err != nil { + return err + } + if err := e.encodeFanout(fanout); err != nil { + return err + } + if err := e.encodeOidLookup(hashes); err != nil { + return err + } + if extraEdges, err := e.encodeCommitData(hashes, hashToIndex, idx); err == nil { + if err = e.encodeExtraEdges(extraEdges); err != nil { + return err + } + } else { + return err + } + + return e.encodeChecksum() +} + +func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[plumbing.Hash]uint32, fanout []uint32, extraEdgesCount uint32) { + // Sort the hashes and build our index + plumbing.HashesSort(hashes) + hashToIndex = make(map[plumbing.Hash]uint32) + fanout = make([]uint32, 256) + for i, hash := range hashes { + hashToIndex[hash] = uint32(i) + fanout[hash[0]]++ + } + + // Convert the fanout to cumulative values + for i := 1; i <= 0xff; i++ { + fanout[i] += fanout[i-1] + } + + // Find out if we will need extra edge table + for i := 0; i < len(hashes); i++ { + v, _ := idx.GetCommitDataByIndex(uint32(i)) + if len(v.ParentHashes) > 2 { + extraEdgesCount += uint32(len(v.ParentHashes) - 1) + break + } + } + + return +} + +func (e *Encoder) encodeFileHeader(chunkCount int) (err error) { + if _, err = e.Write(commitFileSignature); err == nil { + version := byte(1) + if hash.CryptoType == crypto.SHA256 { + version = byte(2) + } + _, err = e.Write([]byte{1, version, byte(chunkCount), 0}) + } + return +} + +func (e *Encoder) encodeChunkHeaders(chunkSignatures [][]byte, chunkSizes []uint64) (err error) { + // 8 bytes of file header, 12 bytes for each chunk header and 12 byte for terminator + offset := uint64(8 + len(chunkSignatures)*12 + 12) + for i, signature := range chunkSignatures { + if _, err = e.Write(signature); err == nil { + err = binary.WriteUint64(e, offset) + } + if err != nil { + return + } + offset += chunkSizes[i] + } + if _, err = e.Write(ZeroChunk.Signature()); err == nil { + err = binary.WriteUint64(e, offset) + } + return +} + +func (e *Encoder) encodeFanout(fanout []uint32) (err error) { + for i := 0; i <= 0xff; i++ { + if err = binary.WriteUint32(e, fanout[i]); err != nil { + return + } + } + return +} + +func (e *Encoder) encodeOidLookup(hashes []plumbing.Hash) (err error) { + for _, hash := range hashes { + if _, err = e.Write(hash[:]); err != nil { + return err + } + } + return +} + +func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumbing.Hash]uint32, idx Index) (extraEdges []uint32, err error) { + for _, hash := range hashes { + origIndex, _ := idx.GetIndexByHash(hash) + commitData, _ := idx.GetCommitDataByIndex(origIndex) + if _, err = e.Write(commitData.TreeHash[:]); err != nil { + return + } + + var parent1, parent2 uint32 + if len(commitData.ParentHashes) == 0 { + parent1 = parentNone + parent2 = parentNone + } else if len(commitData.ParentHashes) == 1 { + parent1 = hashToIndex[commitData.ParentHashes[0]] + parent2 = parentNone + } else if len(commitData.ParentHashes) == 2 { + parent1 = hashToIndex[commitData.ParentHashes[0]] + parent2 = hashToIndex[commitData.ParentHashes[1]] + } else if len(commitData.ParentHashes) > 2 { + parent1 = hashToIndex[commitData.ParentHashes[0]] + parent2 = uint32(len(extraEdges)) | parentOctopusUsed + for _, parentHash := range commitData.ParentHashes[1:] { + extraEdges = append(extraEdges, hashToIndex[parentHash]) + } + extraEdges[len(extraEdges)-1] |= parentLast + } + + if err = binary.WriteUint32(e, parent1); err == nil { + err = binary.WriteUint32(e, parent2) + } + if err != nil { + return + } + + unixTime := uint64(commitData.When.Unix()) + unixTime |= uint64(commitData.Generation) << 34 + if err = binary.WriteUint64(e, unixTime); err != nil { + return + } + } + return +} + +func (e *Encoder) encodeExtraEdges(extraEdges []uint32) (err error) { + for _, parent := range extraEdges { + if err = binary.WriteUint32(e, parent); err != nil { + return + } + } + return +} + +func (e *Encoder) encodeChecksum() error { + _, err := e.Write(e.hash.Sum(nil)[:hash.Size]) + return err +} diff --git a/plumbing/format/commitgraph/v2/file.go b/plumbing/format/commitgraph/v2/file.go new file mode 100644 index 000000000..69e02508e --- /dev/null +++ b/plumbing/format/commitgraph/v2/file.go @@ -0,0 +1,338 @@ +package v2 + +import ( + "bytes" + "crypto" + encbin "encoding/binary" + "errors" + "io" + "time" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" + "github.com/go-git/go-git/v5/utils/binary" +) + +var ( + // ErrUnsupportedVersion is returned by OpenFileIndex when the commit graph + // file version is not supported. + ErrUnsupportedVersion = errors.New("unsupported version") + // ErrUnsupportedHash is returned by OpenFileIndex when the commit graph + // hash function is not supported. Currently only SHA-1 is defined and + // supported. + ErrUnsupportedHash = errors.New("unsupported hash algorithm") + // ErrMalformedCommitGraphFile is returned by OpenFileIndex when the commit + // graph file is corrupted. + ErrMalformedCommitGraphFile = errors.New("malformed commit graph file") + + commitFileSignature = []byte{'C', 'G', 'P', 'H'} + + parentNone = uint32(0x70000000) + parentOctopusUsed = uint32(0x80000000) + parentOctopusMask = uint32(0x7fffffff) + parentLast = uint32(0x80000000) +) + +const ( + commitDataSize = 16 +) + +type fileIndex struct { + reader ReaderAtCloser + fanout [256]uint32 + offsets [9]int64 + parent Index +} + +// ReaderAtCloser is an interface that combines io.ReaderAt and io.Closer. +type ReaderAtCloser interface { + io.ReaderAt + io.Closer +} + +// OpenFileIndex opens a serialized commit graph file in the format described at +// https://github.com/git/git/blob/master/Documentation/technical/commit-graph-format.txt +func OpenFileIndex(reader ReaderAtCloser) (Index, error) { + return OpenFileIndexWithParent(reader, nil) +} + +// OpenFileIndexWithParent opens a serialized commit graph file in the format described at +// https://github.com/git/git/blob/master/Documentation/technical/commit-graph-format.txt +func OpenFileIndexWithParent(reader ReaderAtCloser, parent Index) (Index, error) { + if reader == nil { + return nil, io.ErrUnexpectedEOF + } + fi := &fileIndex{reader: reader, parent: parent} + + if err := fi.verifyFileHeader(); err != nil { + return nil, err + } + if err := fi.readChunkHeaders(); err != nil { + return nil, err + } + if err := fi.readFanout(); err != nil { + return nil, err + } + + return fi, nil +} + +// Close closes the underlying reader and the parent index if it exists. +func (fi *fileIndex) Close() (err error) { + if fi.parent != nil { + defer func() { + parentErr := fi.parent.Close() + // only report the error from the parent if there is no error from the reader + if err == nil { + err = parentErr + } + }() + } + err = fi.reader.Close() + return +} + +func (fi *fileIndex) verifyFileHeader() error { + // Verify file signature + signature := make([]byte, 4) + if _, err := fi.reader.ReadAt(signature, 0); err != nil { + return err + } + if !bytes.Equal(signature, commitFileSignature) { + return ErrMalformedCommitGraphFile + } + + // Read and verify the file header + header := make([]byte, 4) + if _, err := fi.reader.ReadAt(header, 4); err != nil { + return err + } + if header[0] != 1 { + return ErrUnsupportedVersion + } + if !(hash.CryptoType == crypto.SHA1 && header[1] == 1) && + !(hash.CryptoType == crypto.SHA256 && header[1] == 2) { + // Unknown hash type / unsupported hash type + return ErrUnsupportedHash + } + + return nil +} + +func (fi *fileIndex) readChunkHeaders() error { + chunkID := make([]byte, 4) + for i := 0; ; i++ { + chunkHeader := io.NewSectionReader(fi.reader, 8+(int64(i)*12), 12) + if _, err := io.ReadAtLeast(chunkHeader, chunkID, 4); err != nil { + return err + } + chunkOffset, err := binary.ReadUint64(chunkHeader) + if err != nil { + return err + } + + chunkType, ok := ChunkTypeFromBytes(chunkID) + if !ok { + continue + } + if chunkType == ZeroChunk || int(chunkType) >= len(fi.offsets) { + break + } + fi.offsets[chunkType] = int64(chunkOffset) + } + + if fi.offsets[OIDFanoutChunk] <= 0 || fi.offsets[OIDLookupChunk] <= 0 || fi.offsets[CommitDataChunk] <= 0 { + return ErrMalformedCommitGraphFile + } + + return nil +} + +func (fi *fileIndex) readFanout() error { + fanoutReader := io.NewSectionReader(fi.reader, fi.offsets[OIDFanoutChunk], 256*4) + for i := 0; i < 256; i++ { + fanoutValue, err := binary.ReadUint32(fanoutReader) + if err != nil { + return err + } + if fanoutValue > 0x7fffffff { + return ErrMalformedCommitGraphFile + } + fi.fanout[i] = fanoutValue + } + return nil +} + +// GetIndexByHash looks up the provided hash in the commit-graph fanout and returns the index of the commit data for the given hash. +func (fi *fileIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) { + var oid plumbing.Hash + + // Find the hash in the oid lookup table + var low uint32 + if h[0] == 0 { + low = 0 + } else { + low = fi.fanout[h[0]-1] + } + high := fi.fanout[h[0]] + for low < high { + mid := (low + high) >> 1 + offset := fi.offsets[OIDLookupChunk] + int64(mid)*hash.Size + if _, err := fi.reader.ReadAt(oid[:], offset); err != nil { + return 0, err + } + cmp := bytes.Compare(h[:], oid[:]) + if cmp < 0 { + high = mid + } else if cmp == 0 { + return mid, nil + } else { + low = mid + 1 + } + } + + if fi.parent != nil { + idx, err := fi.parent.GetIndexByHash(h) + if err != nil { + return 0, err + } + return idx + fi.fanout[0xff], nil + } + + return 0, plumbing.ErrObjectNotFound +} + +// GetCommitDataByIndex returns the commit data for the given index in the commit-graph. +func (fi *fileIndex) GetCommitDataByIndex(idx uint32) (*CommitData, error) { + if idx >= fi.fanout[0xff] { + if fi.parent != nil { + data, err := fi.parent.GetCommitDataByIndex(idx - fi.fanout[0xff]) + if err != nil { + return nil, err + } + for i := range data.ParentIndexes { + data.ParentIndexes[i] += fi.fanout[0xff] + } + return data, nil + } + + return nil, plumbing.ErrObjectNotFound + } + + offset := fi.offsets[CommitDataChunk] + int64(idx)*(hash.Size+commitDataSize) + commitDataReader := io.NewSectionReader(fi.reader, offset, hash.Size+commitDataSize) + + treeHash, err := binary.ReadHash(commitDataReader) + if err != nil { + return nil, err + } + parent1, err := binary.ReadUint32(commitDataReader) + if err != nil { + return nil, err + } + parent2, err := binary.ReadUint32(commitDataReader) + if err != nil { + return nil, err + } + genAndTime, err := binary.ReadUint64(commitDataReader) + if err != nil { + return nil, err + } + + var parentIndexes []uint32 + if parent2&parentOctopusUsed == parentOctopusUsed { + // Octopus merge + parentIndexes = []uint32{parent1 & parentOctopusMask} + offset := fi.offsets[ExtraEdgeListChunk] + 4*int64(parent2&parentOctopusMask) + buf := make([]byte, 4) + for { + _, err := fi.reader.ReadAt(buf, offset) + if err != nil { + return nil, err + } + + parent := encbin.BigEndian.Uint32(buf) + offset += 4 + parentIndexes = append(parentIndexes, parent&parentOctopusMask) + if parent&parentLast == parentLast { + break + } + } + } else if parent2 != parentNone { + parentIndexes = []uint32{parent1 & parentOctopusMask, parent2 & parentOctopusMask} + } else if parent1 != parentNone { + parentIndexes = []uint32{parent1 & parentOctopusMask} + } + + parentHashes, err := fi.getHashesFromIndexes(parentIndexes) + if err != nil { + return nil, err + } + + return &CommitData{ + TreeHash: treeHash, + ParentIndexes: parentIndexes, + ParentHashes: parentHashes, + Generation: genAndTime >> 34, + When: time.Unix(int64(genAndTime&0x3FFFFFFFF), 0), + }, nil +} + +// GetHashByIndex looks up the hash for the given index in the commit-graph. +func (fi *fileIndex) GetHashByIndex(idx uint32) (found plumbing.Hash, err error) { + if idx >= fi.fanout[0xff] { + if fi.parent != nil { + return fi.parent.GetHashByIndex(idx - fi.fanout[0xff]) + } + return found, ErrMalformedCommitGraphFile + } + + offset := fi.offsets[OIDLookupChunk] + int64(idx)*hash.Size + if _, err := fi.reader.ReadAt(found[:], offset); err != nil { + return found, err + } + + return found, nil +} + +func (fi *fileIndex) getHashesFromIndexes(indexes []uint32) ([]plumbing.Hash, error) { + hashes := make([]plumbing.Hash, len(indexes)) + + for i, idx := range indexes { + if idx >= fi.fanout[0xff] { + if fi.parent != nil { + hash, err := fi.parent.GetHashByIndex(idx - fi.fanout[0xff]) + if err != nil { + return nil, err + } + hashes[i] = hash + continue + } + + return nil, ErrMalformedCommitGraphFile + } + + offset := fi.offsets[OIDLookupChunk] + int64(idx)*hash.Size + if _, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil { + return nil, err + } + } + + return hashes, nil +} + +// Hashes returns all the hashes that are available in the index. +func (fi *fileIndex) Hashes() []plumbing.Hash { + hashes := make([]plumbing.Hash, fi.fanout[0xff]) + for i := uint32(0); i < fi.fanout[0xff]; i++ { + offset := fi.offsets[OIDLookupChunk] + int64(i)*hash.Size + if n, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil || n < hash.Size { + return nil + } + } + if fi.parent != nil { + parentHashes := fi.parent.Hashes() + hashes = append(hashes, parentHashes...) + } + return hashes +} diff --git a/plumbing/format/commitgraph/v2/memory.go b/plumbing/format/commitgraph/v2/memory.go new file mode 100644 index 000000000..ab7ddfad9 --- /dev/null +++ b/plumbing/format/commitgraph/v2/memory.go @@ -0,0 +1,91 @@ +package v2 + +import ( + "github.com/go-git/go-git/v5/plumbing" +) + +// MemoryIndex provides a way to build the commit-graph in memory +// for later encoding to file. +type MemoryIndex struct { + commitData []commitData + indexMap map[plumbing.Hash]uint32 +} + +type commitData struct { + Hash plumbing.Hash + *CommitData +} + +// NewMemoryIndex creates in-memory commit graph representation +func NewMemoryIndex() *MemoryIndex { + return &MemoryIndex{ + indexMap: make(map[plumbing.Hash]uint32), + } +} + +// GetIndexByHash gets the index in the commit graph from commit hash, if available +func (mi *MemoryIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) { + i, ok := mi.indexMap[h] + if ok { + return i, nil + } + + return 0, plumbing.ErrObjectNotFound +} + +// GetHashByIndex gets the hash given an index in the commit graph +func (mi *MemoryIndex) GetHashByIndex(i uint32) (plumbing.Hash, error) { + if i >= uint32(len(mi.commitData)) { + return plumbing.ZeroHash, plumbing.ErrObjectNotFound + } + + return mi.commitData[i].Hash, nil +} + +// GetCommitDataByIndex gets the commit node from the commit graph using index +// obtained from child node, if available +func (mi *MemoryIndex) GetCommitDataByIndex(i uint32) (*CommitData, error) { + if i >= uint32(len(mi.commitData)) { + return nil, plumbing.ErrObjectNotFound + } + + commitData := mi.commitData[i] + + // Map parent hashes to parent indexes + if commitData.ParentIndexes == nil { + parentIndexes := make([]uint32, len(commitData.ParentHashes)) + for i, parentHash := range commitData.ParentHashes { + var err error + if parentIndexes[i], err = mi.GetIndexByHash(parentHash); err != nil { + return nil, err + } + } + commitData.ParentIndexes = parentIndexes + } + + return commitData.CommitData, nil +} + +// Hashes returns all the hashes that are available in the index +func (mi *MemoryIndex) Hashes() []plumbing.Hash { + hashes := make([]plumbing.Hash, 0, len(mi.indexMap)) + for k := range mi.indexMap { + hashes = append(hashes, k) + } + return hashes +} + +// Add adds new node to the memory index +func (mi *MemoryIndex) Add(hash plumbing.Hash, data *CommitData) { + // The parent indexes are calculated lazily in GetNodeByIndex + // which allows adding nodes out of order as long as all parents + // are eventually resolved + data.ParentIndexes = nil + mi.indexMap[hash] = uint32(len(mi.commitData)) + mi.commitData = append(mi.commitData, commitData{Hash: hash, CommitData: data}) +} + +// Close closes the index +func (mi *MemoryIndex) Close() error { + return nil +} diff --git a/plumbing/object/commitgraph/commitnode_graph.go b/plumbing/object/commitgraph/commitnode_graph.go index 8e5d4e34a..252b5181e 100644 --- a/plumbing/object/commitgraph/commitnode_graph.go +++ b/plumbing/object/commitgraph/commitnode_graph.go @@ -1,131 +1,133 @@ -package commitgraph - -import ( - "fmt" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/format/commitgraph" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -// graphCommitNode is a reduced representation of Commit as presented in the commit -// graph file (commitgraph.Node). It is merely useful as an optimization for walking -// the commit graphs. -// -// graphCommitNode implements the CommitNode interface. -type graphCommitNode struct { - // Hash for the Commit object - hash plumbing.Hash - // Index of the node in the commit graph file - index int - - commitData *commitgraph.CommitData - gci *graphCommitNodeIndex -} - -// graphCommitNodeIndex is an index that can load CommitNode objects from both the commit -// graph files and the object store. -// -// graphCommitNodeIndex implements the CommitNodeIndex interface -type graphCommitNodeIndex struct { - commitGraph commitgraph.Index - s storer.EncodedObjectStorer -} - -// NewGraphCommitNodeIndex returns CommitNodeIndex implementation that uses commit-graph -// files as backing storage and falls back to object storage when necessary -func NewGraphCommitNodeIndex(commitGraph commitgraph.Index, s storer.EncodedObjectStorer) CommitNodeIndex { - return &graphCommitNodeIndex{commitGraph, s} -} - -func (gci *graphCommitNodeIndex) Get(hash plumbing.Hash) (CommitNode, error) { - // Check the commit graph first - parentIndex, err := gci.commitGraph.GetIndexByHash(hash) - if err == nil { - parent, err := gci.commitGraph.GetCommitDataByIndex(parentIndex) - if err != nil { - return nil, err - } - - return &graphCommitNode{ - hash: hash, - index: parentIndex, - commitData: parent, - gci: gci, - }, nil - } - - // Fallback to loading full commit object - commit, err := object.GetCommit(gci.s, hash) - if err != nil { - return nil, err - } - - return &objectCommitNode{ - nodeIndex: gci, - commit: commit, - }, nil -} - -func (c *graphCommitNode) ID() plumbing.Hash { - return c.hash -} - -func (c *graphCommitNode) Tree() (*object.Tree, error) { - return object.GetTree(c.gci.s, c.commitData.TreeHash) -} - -func (c *graphCommitNode) CommitTime() time.Time { - return c.commitData.When -} - -func (c *graphCommitNode) NumParents() int { - return len(c.commitData.ParentIndexes) -} - -func (c *graphCommitNode) ParentNodes() CommitNodeIter { - return newParentgraphCommitNodeIter(c) -} - -func (c *graphCommitNode) ParentNode(i int) (CommitNode, error) { - if i < 0 || i >= len(c.commitData.ParentIndexes) { - return nil, object.ErrParentNotFound - } - - parent, err := c.gci.commitGraph.GetCommitDataByIndex(c.commitData.ParentIndexes[i]) - if err != nil { - return nil, err - } - - return &graphCommitNode{ - hash: c.commitData.ParentHashes[i], - index: c.commitData.ParentIndexes[i], - commitData: parent, - gci: c.gci, - }, nil -} - -func (c *graphCommitNode) ParentHashes() []plumbing.Hash { - return c.commitData.ParentHashes -} - -func (c *graphCommitNode) Generation() uint64 { - // If the commit-graph file was generated with older Git version that - // set the generation to zero for every commit the generation assumption - // is still valid. It is just less useful. - return uint64(c.commitData.Generation) -} - -func (c *graphCommitNode) Commit() (*object.Commit, error) { - return object.GetCommit(c.gci.s, c.hash) -} - -func (c *graphCommitNode) String() string { - return fmt.Sprintf( - "%s %s\nDate: %s", - plumbing.CommitObject, c.ID(), - c.CommitTime().Format(object.DateFormat), - ) -} +package commitgraph + +import ( + "fmt" + "time" + + "github.com/go-git/go-git/v5/plumbing" + commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" +) + +// graphCommitNode is a reduced representation of Commit as presented in the commit +// graph file (commitgraph.Node). It is merely useful as an optimization for walking +// the commit graphs. +// +// graphCommitNode implements the CommitNode interface. +type graphCommitNode struct { + // Hash for the Commit object + hash plumbing.Hash + // Index of the node in the commit graph file + index uint32 + + commitData *commitgraph.CommitData + gci *graphCommitNodeIndex +} + +// graphCommitNodeIndex is an index that can load CommitNode objects from both the commit +// graph files and the object store. +// +// graphCommitNodeIndex implements the CommitNodeIndex interface +type graphCommitNodeIndex struct { + commitGraph commitgraph.Index + s storer.EncodedObjectStorer +} + +// NewGraphCommitNodeIndex returns CommitNodeIndex implementation that uses commit-graph +// files as backing storage and falls back to object storage when necessary +func NewGraphCommitNodeIndex(commitGraph commitgraph.Index, s storer.EncodedObjectStorer) CommitNodeIndex { + return &graphCommitNodeIndex{commitGraph, s} +} + +func (gci *graphCommitNodeIndex) Get(hash plumbing.Hash) (CommitNode, error) { + if gci.commitGraph != nil { + // Check the commit graph first + parentIndex, err := gci.commitGraph.GetIndexByHash(hash) + if err == nil { + parent, err := gci.commitGraph.GetCommitDataByIndex(parentIndex) + if err != nil { + return nil, err + } + + return &graphCommitNode{ + hash: hash, + index: parentIndex, + commitData: parent, + gci: gci, + }, nil + } + } + + // Fallback to loading full commit object + commit, err := object.GetCommit(gci.s, hash) + if err != nil { + return nil, err + } + + return &objectCommitNode{ + nodeIndex: gci, + commit: commit, + }, nil +} + +func (c *graphCommitNode) ID() plumbing.Hash { + return c.hash +} + +func (c *graphCommitNode) Tree() (*object.Tree, error) { + return object.GetTree(c.gci.s, c.commitData.TreeHash) +} + +func (c *graphCommitNode) CommitTime() time.Time { + return c.commitData.When +} + +func (c *graphCommitNode) NumParents() int { + return len(c.commitData.ParentIndexes) +} + +func (c *graphCommitNode) ParentNodes() CommitNodeIter { + return newParentgraphCommitNodeIter(c) +} + +func (c *graphCommitNode) ParentNode(i int) (CommitNode, error) { + if i < 0 || i >= len(c.commitData.ParentIndexes) { + return nil, object.ErrParentNotFound + } + + parent, err := c.gci.commitGraph.GetCommitDataByIndex(c.commitData.ParentIndexes[i]) + if err != nil { + return nil, err + } + + return &graphCommitNode{ + hash: c.commitData.ParentHashes[i], + index: c.commitData.ParentIndexes[i], + commitData: parent, + gci: c.gci, + }, nil +} + +func (c *graphCommitNode) ParentHashes() []plumbing.Hash { + return c.commitData.ParentHashes +} + +func (c *graphCommitNode) Generation() uint64 { + // If the commit-graph file was generated with older Git version that + // set the generation to zero for every commit the generation assumption + // is still valid. It is just less useful. + return c.commitData.Generation +} + +func (c *graphCommitNode) Commit() (*object.Commit, error) { + return object.GetCommit(c.gci.s, c.hash) +} + +func (c *graphCommitNode) String() string { + return fmt.Sprintf( + "%s %s\nDate: %s", + plumbing.CommitObject, c.ID(), + c.CommitTime().Format(object.DateFormat), + ) +} diff --git a/plumbing/object/commitgraph/commitnode_object.go b/plumbing/object/commitgraph/commitnode_object.go index bdf8cb74a..1bd37e3e0 100644 --- a/plumbing/object/commitgraph/commitnode_object.go +++ b/plumbing/object/commitgraph/commitnode_object.go @@ -1,90 +1,90 @@ -package commitgraph - -import ( - "math" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -// objectCommitNode is a representation of Commit as presented in the GIT object format. -// -// objectCommitNode implements the CommitNode interface. -type objectCommitNode struct { - nodeIndex CommitNodeIndex - commit *object.Commit -} - -// NewObjectCommitNodeIndex returns CommitNodeIndex implementation that uses -// only object storage to load the nodes -func NewObjectCommitNodeIndex(s storer.EncodedObjectStorer) CommitNodeIndex { - return &objectCommitNodeIndex{s} -} - -func (oci *objectCommitNodeIndex) Get(hash plumbing.Hash) (CommitNode, error) { - commit, err := object.GetCommit(oci.s, hash) - if err != nil { - return nil, err - } - - return &objectCommitNode{ - nodeIndex: oci, - commit: commit, - }, nil -} - -// objectCommitNodeIndex is an index that can load CommitNode objects only from the -// object store. -// -// objectCommitNodeIndex implements the CommitNodeIndex interface -type objectCommitNodeIndex struct { - s storer.EncodedObjectStorer -} - -func (c *objectCommitNode) CommitTime() time.Time { - return c.commit.Committer.When -} - -func (c *objectCommitNode) ID() plumbing.Hash { - return c.commit.ID() -} - -func (c *objectCommitNode) Tree() (*object.Tree, error) { - return c.commit.Tree() -} - -func (c *objectCommitNode) NumParents() int { - return c.commit.NumParents() -} - -func (c *objectCommitNode) ParentNodes() CommitNodeIter { - return newParentgraphCommitNodeIter(c) -} - -func (c *objectCommitNode) ParentNode(i int) (CommitNode, error) { - if i < 0 || i >= len(c.commit.ParentHashes) { - return nil, object.ErrParentNotFound - } - - // Note: It's necessary to go through CommitNodeIndex here to ensure - // that if the commit-graph file covers only part of the history we - // start using it when that part is reached. - return c.nodeIndex.Get(c.commit.ParentHashes[i]) -} - -func (c *objectCommitNode) ParentHashes() []plumbing.Hash { - return c.commit.ParentHashes -} - -func (c *objectCommitNode) Generation() uint64 { - // Commit nodes representing objects outside of the commit graph can never - // be reached by objects from the commit-graph thus we return the highest - // possible value. - return math.MaxUint64 -} - -func (c *objectCommitNode) Commit() (*object.Commit, error) { - return c.commit, nil -} +package commitgraph + +import ( + "math" + "time" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" +) + +// objectCommitNode is a representation of Commit as presented in the GIT object format. +// +// objectCommitNode implements the CommitNode interface. +type objectCommitNode struct { + nodeIndex CommitNodeIndex + commit *object.Commit +} + +// NewObjectCommitNodeIndex returns CommitNodeIndex implementation that uses +// only object storage to load the nodes +func NewObjectCommitNodeIndex(s storer.EncodedObjectStorer) CommitNodeIndex { + return &objectCommitNodeIndex{s} +} + +func (oci *objectCommitNodeIndex) Get(hash plumbing.Hash) (CommitNode, error) { + commit, err := object.GetCommit(oci.s, hash) + if err != nil { + return nil, err + } + + return &objectCommitNode{ + nodeIndex: oci, + commit: commit, + }, nil +} + +// objectCommitNodeIndex is an index that can load CommitNode objects only from the +// object store. +// +// objectCommitNodeIndex implements the CommitNodeIndex interface +type objectCommitNodeIndex struct { + s storer.EncodedObjectStorer +} + +func (c *objectCommitNode) CommitTime() time.Time { + return c.commit.Committer.When +} + +func (c *objectCommitNode) ID() plumbing.Hash { + return c.commit.ID() +} + +func (c *objectCommitNode) Tree() (*object.Tree, error) { + return c.commit.Tree() +} + +func (c *objectCommitNode) NumParents() int { + return c.commit.NumParents() +} + +func (c *objectCommitNode) ParentNodes() CommitNodeIter { + return newParentgraphCommitNodeIter(c) +} + +func (c *objectCommitNode) ParentNode(i int) (CommitNode, error) { + if i < 0 || i >= len(c.commit.ParentHashes) { + return nil, object.ErrParentNotFound + } + + // Note: It's necessary to go through CommitNodeIndex here to ensure + // that if the commit-graph file covers only part of the history we + // start using it when that part is reached. + return c.nodeIndex.Get(c.commit.ParentHashes[i]) +} + +func (c *objectCommitNode) ParentHashes() []plumbing.Hash { + return c.commit.ParentHashes +} + +func (c *objectCommitNode) Generation() uint64 { + // Commit nodes representing objects outside of the commit graph can never + // be reached by objects from the commit-graph thus we return the highest + // possible value. + return math.MaxUint64 +} + +func (c *objectCommitNode) Commit() (*object.Commit, error) { + return c.commit, nil +} diff --git a/plumbing/object/commitgraph/commitnode_test.go b/plumbing/object/commitgraph/commitnode_test.go index 91fb21117..441ff6f0a 100644 --- a/plumbing/object/commitgraph/commitnode_test.go +++ b/plumbing/object/commitgraph/commitnode_test.go @@ -6,7 +6,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/format/commitgraph" + commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" "github.com/go-git/go-git/v5/plumbing/format/packfile" "github.com/go-git/go-git/v5/storage/filesystem" @@ -115,6 +115,7 @@ func (s *CommitNodeSuite) TestCommitGraph(c *C) { defer reader.Close() index, err := commitgraph.OpenFileIndex(reader) c.Assert(err, IsNil) + defer index.Close() nodeIndex := NewGraphCommitNodeIndex(index, storer) testWalker(c, nodeIndex) @@ -132,10 +133,14 @@ func (s *CommitNodeSuite) TestMixedGraph(c *C) { defer reader.Close() fileIndex, err := commitgraph.OpenFileIndex(reader) c.Assert(err, IsNil) + defer fileIndex.Close() + memoryIndex := commitgraph.NewMemoryIndex() + defer memoryIndex.Close() + for i, hash := range fileIndex.Hashes() { if hash.String() != "b9d69064b190e7aedccf84731ca1d917871f8a1c" { - node, err := fileIndex.GetCommitDataByIndex(i) + node, err := fileIndex.GetCommitDataByIndex(uint32(i)) c.Assert(err, IsNil) memoryIndex.Add(hash, node) } diff --git a/plumbing/object/commitgraph/commitnode_walker_ctime.go b/plumbing/object/commitgraph/commitnode_walker_ctime.go index 281f10bdf..c26873ce5 100644 --- a/plumbing/object/commitgraph/commitnode_walker_ctime.go +++ b/plumbing/object/commitgraph/commitnode_walker_ctime.go @@ -1,105 +1,105 @@ -package commitgraph - -import ( - "io" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/storer" - - "github.com/emirpasic/gods/trees/binaryheap" -) - -type commitNodeIteratorByCTime struct { - heap *binaryheap.Heap - seenExternal map[plumbing.Hash]bool - seen map[plumbing.Hash]bool -} - -// NewCommitNodeIterCTime returns a CommitNodeIter that walks the commit history, -// starting at the given commit and visiting its parents while preserving Committer Time order. -// this appears to be the closest order to `git log` -// The given callback will be called for each visited commit. Each commit will -// be visited only once. If the callback returns an error, walking will stop -// and will return the error. Other errors might be returned if the history -// cannot be traversed (e.g. missing objects). Ignore allows to skip some -// commits from being iterated. -func NewCommitNodeIterCTime( - c CommitNode, - seenExternal map[plumbing.Hash]bool, - ignore []plumbing.Hash, -) CommitNodeIter { - seen := make(map[plumbing.Hash]bool) - for _, h := range ignore { - seen[h] = true - } - - heap := binaryheap.NewWith(func(a, b interface{}) int { - if a.(CommitNode).CommitTime().Before(b.(CommitNode).CommitTime()) { - return 1 - } - return -1 - }) - - heap.Push(c) - - return &commitNodeIteratorByCTime{ - heap: heap, - seenExternal: seenExternal, - seen: seen, - } -} - -func (w *commitNodeIteratorByCTime) Next() (CommitNode, error) { - var c CommitNode - for { - cIn, ok := w.heap.Pop() - if !ok { - return nil, io.EOF - } - c = cIn.(CommitNode) - cID := c.ID() - - if w.seen[cID] || w.seenExternal[cID] { - continue - } - - w.seen[cID] = true - - for i, h := range c.ParentHashes() { - if w.seen[h] || w.seenExternal[h] { - continue - } - pc, err := c.ParentNode(i) - if err != nil { - return nil, err - } - w.heap.Push(pc) - } - - return c, nil - } -} - -func (w *commitNodeIteratorByCTime) ForEach(cb func(CommitNode) error) error { - for { - c, err := w.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - err = cb(c) - if err == storer.ErrStop { - break - } - if err != nil { - return err - } - } - - return nil -} - -func (w *commitNodeIteratorByCTime) Close() {} +package commitgraph + +import ( + "io" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" + + "github.com/emirpasic/gods/trees/binaryheap" +) + +type commitNodeIteratorByCTime struct { + heap *binaryheap.Heap + seenExternal map[plumbing.Hash]bool + seen map[plumbing.Hash]bool +} + +// NewCommitNodeIterCTime returns a CommitNodeIter that walks the commit history, +// starting at the given commit and visiting its parents while preserving Committer Time order. +// this appears to be the closest order to `git log` +// The given callback will be called for each visited commit. Each commit will +// be visited only once. If the callback returns an error, walking will stop +// and will return the error. Other errors might be returned if the history +// cannot be traversed (e.g. missing objects). Ignore allows to skip some +// commits from being iterated. +func NewCommitNodeIterCTime( + c CommitNode, + seenExternal map[plumbing.Hash]bool, + ignore []plumbing.Hash, +) CommitNodeIter { + seen := make(map[plumbing.Hash]bool) + for _, h := range ignore { + seen[h] = true + } + + heap := binaryheap.NewWith(func(a, b interface{}) int { + if a.(CommitNode).CommitTime().Before(b.(CommitNode).CommitTime()) { + return 1 + } + return -1 + }) + + heap.Push(c) + + return &commitNodeIteratorByCTime{ + heap: heap, + seenExternal: seenExternal, + seen: seen, + } +} + +func (w *commitNodeIteratorByCTime) Next() (CommitNode, error) { + var c CommitNode + for { + cIn, ok := w.heap.Pop() + if !ok { + return nil, io.EOF + } + c = cIn.(CommitNode) + cID := c.ID() + + if w.seen[cID] || w.seenExternal[cID] { + continue + } + + w.seen[cID] = true + + for i, h := range c.ParentHashes() { + if w.seen[h] || w.seenExternal[h] { + continue + } + pc, err := c.ParentNode(i) + if err != nil { + return nil, err + } + w.heap.Push(pc) + } + + return c, nil + } +} + +func (w *commitNodeIteratorByCTime) ForEach(cb func(CommitNode) error) error { + for { + c, err := w.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + err = cb(c) + if err == storer.ErrStop { + break + } + if err != nil { + return err + } + } + + return nil +} + +func (w *commitNodeIteratorByCTime) Close() {} From 1a73661645a39169763648825f4964d0794bef26 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Mon, 9 Oct 2023 22:49:36 +0100 Subject: [PATCH 15/56] build: Add github workflow to check commit message format Adds automation to confirm the commit messages align with the contributing guidelines. Signed-off-by: Paulo Gomes --- .github/workflows/pr-validation.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/pr-validation.yml diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 000000000..04e704654 --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,30 @@ +name: 'PR Validation' + +on: + pull_request: + types: + - opened + - edited + - reopened + - synchronize + +permissions: + contents: read + +jobs: + check-commit-message: + name: Check Commit Messages + runs-on: ubuntu-latest + steps: + - name: Check Package Prefix + uses: gsactions/commit-message-checker@v2 + with: + pattern: '^(\*|plumbing|utils|config|_examples|internal|storage|cli|build): .+' + error: | + Commit message(s) does not align with contribution acceptance criteria. + + Refer to https://github.com/go-git/go-git/blob/master/CONTRIBUTING.md#format-of-the-commit-message for more information. + excludeDescription: 'true' + excludeTitle: 'true' + checkAllCommitMessages: 'true' + accessToken: ${{ secrets.GITHUB_TOKEN }} From 129b709887b4528ced42c8d74f4c2609800a8942 Mon Sep 17 00:00:00 2001 From: Max Jonas Werner Date: Sat, 7 Oct 2023 00:02:01 +0200 Subject: [PATCH 16/56] plumbing: transport/common, Improve handling of remote errors Instead of simply returning the first line that the remote returned, go-git now actively searches all of stderr for lines that may contain a more actionable error message and returns that. In addition, this change adds a case to map the GitLab-specific error message to an ErrRepositoryNotFound error. Signed-off-by: Max Jonas Werner --- plumbing/transport/internal/common/common.go | 25 ++++++-- .../transport/internal/common/common_test.go | 57 +++++++++++++++++++ plumbing/transport/internal/common/mocks.go | 46 +++++++++++++++ 3 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 plumbing/transport/internal/common/mocks.go diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index 5fdf4250d..6574116b1 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "regexp" "strings" "time" @@ -28,6 +29,10 @@ const ( var ( ErrTimeoutExceeded = errors.New("timeout exceeded") + // stdErrSkipPattern is used for skipping lines from a command's stderr output. + // Any line matching this pattern will be skipped from further + // processing and not be returned to calling code. + stdErrSkipPattern = regexp.MustCompile("^remote:( =*){0,1}$") ) // Commander creates Command instances. This is the main entry point for @@ -149,10 +154,17 @@ func (c *client) listenFirstError(r io.Reader) chan string { errLine := make(chan string, 1) go func() { s := bufio.NewScanner(r) - if s.Scan() { - errLine <- s.Text() - } else { - close(errLine) + for { + if s.Scan() { + line := s.Text() + if !stdErrSkipPattern.MatchString(line) { + errLine <- line + break + } + } else { + close(errLine) + break + } } _, _ = io.Copy(io.Discard, r) @@ -393,6 +405,7 @@ var ( gitProtocolNoSuchErr = "ERR no such repository" gitProtocolAccessDeniedErr = "ERR access denied" gogsAccessDeniedErr = "Gogs: Repository does not exist or you do not have access" + gitlabRepoNotFoundErr = "remote: ERROR: The project you were looking for could not be found" ) func isRepoNotFoundError(s string) bool { @@ -424,6 +437,10 @@ func isRepoNotFoundError(s string) bool { return true } + if strings.HasPrefix(s, gitlabRepoNotFoundErr) { + return true + } + return false } diff --git a/plumbing/transport/internal/common/common_test.go b/plumbing/transport/internal/common/common_test.go index affa78706..f6f2f67d2 100644 --- a/plumbing/transport/internal/common/common_test.go +++ b/plumbing/transport/internal/common/common_test.go @@ -4,6 +4,7 @@ import ( "fmt" "testing" + "github.com/go-git/go-git/v5/plumbing/transport" . "gopkg.in/check.v1" ) @@ -77,6 +78,14 @@ func (s *CommonSuite) TestIsRepoNotFoundErrorForGogsAccessDenied(c *C) { c.Assert(isRepoNotFound, Equals, true) } +func (s *CommonSuite) TestIsRepoNotFoundErrorForGitlab(c *C) { + msg := fmt.Sprintf("%s : some error stuf", gitlabRepoNotFoundErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + func (s *CommonSuite) TestCheckNotFoundError(c *C) { firstErrLine := make(chan string, 1) @@ -90,3 +99,51 @@ func (s *CommonSuite) TestCheckNotFoundError(c *C) { c.Assert(err, IsNil) } + +func TestAdvertisedReferencesWithRemoteError(t *testing.T) { + tests := []struct { + name string + stderr string + wantErr error + }{ + { + name: "unknown error", + stderr: "something", + wantErr: fmt.Errorf("unknown error: something"), + }, + { + name: "GitLab: repository not found", + stderr: `remote: +remote: ======================================================================== +remote: +remote: ERROR: The project you were looking for could not be found or you don't have permission to view it. + +remote: +remote: ======================================================================== +remote:`, + wantErr: transport.ErrRepositoryNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := NewClient(MockCommander{stderr: tt.stderr}) + sess, err := client.NewUploadPackSession(nil, nil) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + _, err = sess.AdvertisedReferences() + + if tt.wantErr != nil { + if tt.wantErr != err { + if tt.wantErr.Error() != err.Error() { + t.Fatalf("expected a different error: got '%s', expected '%s'", err, tt.wantErr) + } + } + } else if err != nil { + t.Fatalf("unexpected error: %s", err) + } + }) + } +} diff --git a/plumbing/transport/internal/common/mocks.go b/plumbing/transport/internal/common/mocks.go new file mode 100644 index 000000000..bc18b27e8 --- /dev/null +++ b/plumbing/transport/internal/common/mocks.go @@ -0,0 +1,46 @@ +package common + +import ( + "bytes" + "io" + + gogitioutil "github.com/go-git/go-git/v5/utils/ioutil" + + "github.com/go-git/go-git/v5/plumbing/transport" +) + +type MockCommand struct { + stdin bytes.Buffer + stdout bytes.Buffer + stderr bytes.Buffer +} + +func (c MockCommand) StderrPipe() (io.Reader, error) { + return &c.stderr, nil +} + +func (c MockCommand) StdinPipe() (io.WriteCloser, error) { + return gogitioutil.WriteNopCloser(&c.stdin), nil +} + +func (c MockCommand) StdoutPipe() (io.Reader, error) { + return &c.stdout, nil +} + +func (c MockCommand) Start() error { + return nil +} + +func (c MockCommand) Close() error { + panic("not implemented") +} + +type MockCommander struct { + stderr string +} + +func (c MockCommander) Command(cmd string, ep *transport.Endpoint, auth transport.AuthMethod) (Command, error) { + return &MockCommand{ + stderr: *bytes.NewBufferString(c.stderr), + }, nil +} From 69b88d9bda44ebfe1d56a7624b956d9e20818c0e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 8 Oct 2023 15:49:51 +0100 Subject: [PATCH 17/56] plumbing: commitgraph, Add generation v2 support This PR adds in support for generation v2 support and a couple of new walkers to match --date-order etc options on log. This PR also fixes a bug in the chain code and adds more tests. Signed-off-by: Andrew Thornton --- go.mod | 2 +- go.sum | 4 +- plumbing/format/commitgraph/v2/chunk.go | 7 +- plumbing/format/commitgraph/v2/commitgraph.go | 17 ++ .../format/commitgraph/v2/commitgraph_test.go | 35 ++++ plumbing/format/commitgraph/v2/encoder.go | 84 ++++++-- plumbing/format/commitgraph/v2/file.go | 144 ++++++++++---- plumbing/format/commitgraph/v2/memory.go | 22 ++- plumbing/object/commitgraph/commitnode.go | 4 + .../object/commitgraph/commitnode_graph.go | 7 + .../object/commitgraph/commitnode_object.go | 7 + .../commitnode_walker_author_order.go | 61 ++++++ .../commitgraph/commitnode_walker_ctime.go | 3 +- .../commitnode_walker_date_order.go | 41 ++++ .../commitgraph/commitnode_walker_helper.go | 164 +++++++++++++++ .../commitgraph/commitnode_walker_test.go | 187 ++++++++++++++++++ .../commitnode_walker_topo_order.go | 161 +++++++++++++++ 17 files changed, 892 insertions(+), 58 deletions(-) create mode 100644 plumbing/object/commitgraph/commitnode_walker_author_order.go create mode 100644 plumbing/object/commitgraph/commitnode_walker_date_order.go create mode 100644 plumbing/object/commitgraph/commitnode_walker_helper.go create mode 100644 plumbing/object/commitgraph/commitnode_walker_test.go create mode 100644 plumbing/object/commitgraph/commitnode_walker_topo_order.go diff --git a/go.mod b/go.mod index 9df2020f1..6b7bbdb37 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/gliderlabs/ssh v0.3.5 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 github.com/go-git/go-billy/v5 v5.5.0 - github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389 + github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/go-cmp v0.5.9 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 diff --git a/go.sum b/go.sum index 7c62b1094..347da85ef 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389 h1:AlfdJ8f+G+4a4fXeHmAlKfyR3Yup4sVGCXlh+e+TrE8= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= diff --git a/plumbing/format/commitgraph/v2/chunk.go b/plumbing/format/commitgraph/v2/chunk.go index ab2432072..11f4d3163 100644 --- a/plumbing/format/commitgraph/v2/chunk.go +++ b/plumbing/format/commitgraph/v2/chunk.go @@ -3,7 +3,7 @@ package v2 import "bytes" const ( - chunkSigLen = 4 // Length of a chunk signature + szChunkSig = 4 // Length of a chunk signature chunkSigOffset = 4 // Offset of each chunk signature in chunkSignatures ) @@ -28,14 +28,15 @@ const ( BaseGraphsListChunk // "BASE" ZeroChunk // "\000\000\000\000" ) +const lenChunks = int(ZeroChunk) // ZeroChunk is not a valid chunk type, but it is used to determine the length of the chunk type list. // Signature returns the byte signature for the chunk type. func (ct ChunkType) Signature() []byte { if ct >= BaseGraphsListChunk || ct < 0 { // not a valid chunk type just return ZeroChunk - return chunkSignatures[ZeroChunk*chunkSigOffset : ZeroChunk*chunkSigOffset+chunkSigLen] + return chunkSignatures[ZeroChunk*chunkSigOffset : ZeroChunk*chunkSigOffset+szChunkSig] } - return chunkSignatures[ct*chunkSigOffset : ct*chunkSigOffset+chunkSigLen] + return chunkSignatures[ct*chunkSigOffset : ct*chunkSigOffset+szChunkSig] } // ChunkTypeFromBytes returns the chunk type for the given byte signature. diff --git a/plumbing/format/commitgraph/v2/commitgraph.go b/plumbing/format/commitgraph/v2/commitgraph.go index 7c67b6395..9c89cd9b4 100644 --- a/plumbing/format/commitgraph/v2/commitgraph.go +++ b/plumbing/format/commitgraph/v2/commitgraph.go @@ -2,6 +2,7 @@ package v2 import ( "io" + "math" "time" "github.com/go-git/go-git/v5/plumbing" @@ -19,10 +20,22 @@ type CommitData struct { // Generation number is the pre-computed generation in the commit graph // or zero if not available. Generation uint64 + // GenerationV2 stores the corrected commit date for the commits + // It combines the contents of the GDA2 and GDO2 sections of the commit-graph + // with the commit time portion of the CDAT section. + GenerationV2 uint64 // When is the timestamp of the commit. When time.Time } +// GenerationV2Data returns the corrected commit date for the commits +func (c *CommitData) GenerationV2Data() uint64 { + if c.GenerationV2 == 0 || c.GenerationV2 == math.MaxUint64 { + return 0 + } + return c.GenerationV2 - uint64(c.When.Unix()) +} + // Index represents a representation of commit graph that allows indexed // access to the nodes using commit object hash type Index interface { @@ -35,6 +48,10 @@ type Index interface { GetCommitDataByIndex(i uint32) (*CommitData, error) // Hashes returns all the hashes that are available in the index Hashes() []plumbing.Hash + // HasGenerationV2 returns true if the commit graph has the corrected commit date data + HasGenerationV2() bool + // MaximumNumberOfHashes returns the maximum number of hashes within the index + MaximumNumberOfHashes() uint32 io.Closer } diff --git a/plumbing/format/commitgraph/v2/commitgraph_test.go b/plumbing/format/commitgraph/v2/commitgraph_test.go index 69fdcd9fd..127840567 100644 --- a/plumbing/format/commitgraph/v2/commitgraph_test.go +++ b/plumbing/format/commitgraph/v2/commitgraph_test.go @@ -7,7 +7,11 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/util" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" + "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/filesystem" fixtures "github.com/go-git/go-git-fixtures/v4" . "gopkg.in/check.v1" @@ -76,6 +80,37 @@ func testDecodeHelper(c *C, index commitgraph.Index) { c.Assert(hashes[10].String(), Equals, "e713b52d7e13807e87a002e812041f248db3f643") } +func (s *CommitgraphSuite) TestDecodeMultiChain(c *C) { + fixtures.ByTag("commit-graph-chain-2").Test(c, func(f *fixtures.Fixture) { + dotgit := f.DotGit() + index, err := commitgraph.OpenChainOrFileIndex(dotgit) + c.Assert(err, IsNil) + defer index.Close() + storer := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) + p := f.Packfile() + defer p.Close() + packfile.UpdateObjectStorage(storer, p) + + for idx, hash := range index.Hashes() { + idx2, err := index.GetIndexByHash(hash) + c.Assert(err, IsNil) + c.Assert(idx2, Equals, uint32(idx)) + hash2, err := index.GetHashByIndex(idx2) + c.Assert(err, IsNil) + c.Assert(hash2.String(), Equals, hash.String()) + + commitData, err := index.GetCommitDataByIndex(uint32(idx)) + c.Assert(err, IsNil) + commit, err := object.GetCommit(storer, hash) + c.Assert(err, IsNil) + + for i, parent := range commit.ParentHashes { + c.Assert(hash.String()+":"+parent.String(), Equals, hash.String()+":"+commitData.ParentHashes[i].String()) + } + } + }) +} + func (s *CommitgraphSuite) TestDecode(c *C) { fixtures.ByTag("commit-graph").Test(c, func(f *fixtures.Fixture) { dotgit := f.DotGit() diff --git a/plumbing/format/commitgraph/v2/encoder.go b/plumbing/format/commitgraph/v2/encoder.go index d1e41f892..b79bc77f7 100644 --- a/plumbing/format/commitgraph/v2/encoder.go +++ b/plumbing/format/commitgraph/v2/encoder.go @@ -3,6 +3,7 @@ package v2 import ( "crypto" "io" + "math" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/hash" @@ -28,13 +29,21 @@ func (e *Encoder) Encode(idx Index) error { hashes := idx.Hashes() // Sort the inout and prepare helper structures we'll need for encoding - hashToIndex, fanout, extraEdgesCount := e.prepare(idx, hashes) + hashToIndex, fanout, extraEdgesCount, generationV2OverflowCount := e.prepare(idx, hashes) chunkSignatures := [][]byte{OIDFanoutChunk.Signature(), OIDLookupChunk.Signature(), CommitDataChunk.Signature()} - chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * (hash.Size + commitDataSize)} + chunkSizes := []uint64{szUint32 * lenFanout, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * (hash.Size + szCommitData)} if extraEdgesCount > 0 { chunkSignatures = append(chunkSignatures, ExtraEdgeListChunk.Signature()) - chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*4) + chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*szUint32) + } + if idx.HasGenerationV2() { + chunkSignatures = append(chunkSignatures, GenerationDataChunk.Signature()) + chunkSizes = append(chunkSizes, uint64(len(hashes))*szUint32) + if generationV2OverflowCount > 0 { + chunkSignatures = append(chunkSignatures, GenerationDataOverflowChunk.Signature()) + chunkSizes = append(chunkSizes, uint64(generationV2OverflowCount)*szUint64) + } } if err := e.encodeFileHeader(len(chunkSignatures)); err != nil { @@ -49,38 +58,52 @@ func (e *Encoder) Encode(idx Index) error { if err := e.encodeOidLookup(hashes); err != nil { return err } - if extraEdges, err := e.encodeCommitData(hashes, hashToIndex, idx); err == nil { - if err = e.encodeExtraEdges(extraEdges); err != nil { + + extraEdges, generationV2Data, err := e.encodeCommitData(hashes, hashToIndex, idx) + if err != nil { + return err + } + if err = e.encodeExtraEdges(extraEdges); err != nil { + return err + } + if idx.HasGenerationV2() { + overflows, err := e.encodeGenerationV2Data(generationV2Data) + if err != nil { + return err + } + if err = e.encodeGenerationV2Overflow(overflows); err != nil { return err } - } else { - return err } return e.encodeChecksum() } -func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[plumbing.Hash]uint32, fanout []uint32, extraEdgesCount uint32) { +func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[plumbing.Hash]uint32, fanout []uint32, extraEdgesCount uint32, generationV2OverflowCount uint32) { // Sort the hashes and build our index plumbing.HashesSort(hashes) hashToIndex = make(map[plumbing.Hash]uint32) - fanout = make([]uint32, 256) + fanout = make([]uint32, lenFanout) for i, hash := range hashes { hashToIndex[hash] = uint32(i) fanout[hash[0]]++ } // Convert the fanout to cumulative values - for i := 1; i <= 0xff; i++ { + for i := 1; i < lenFanout; i++ { fanout[i] += fanout[i-1] } + hasGenerationV2 := idx.HasGenerationV2() + // Find out if we will need extra edge table for i := 0; i < len(hashes); i++ { v, _ := idx.GetCommitDataByIndex(uint32(i)) if len(v.ParentHashes) > 2 { extraEdgesCount += uint32(len(v.ParentHashes) - 1) - break + } + if hasGenerationV2 && v.GenerationV2Data() > math.MaxUint32 { + generationV2OverflowCount++ } } @@ -100,7 +123,7 @@ func (e *Encoder) encodeFileHeader(chunkCount int) (err error) { func (e *Encoder) encodeChunkHeaders(chunkSignatures [][]byte, chunkSizes []uint64) (err error) { // 8 bytes of file header, 12 bytes for each chunk header and 12 byte for terminator - offset := uint64(8 + len(chunkSignatures)*12 + 12) + offset := uint64(szSignature + szHeader + (len(chunkSignatures)+1)*(szChunkSig+szUint64)) for i, signature := range chunkSignatures { if _, err = e.Write(signature); err == nil { err = binary.WriteUint64(e, offset) @@ -134,7 +157,10 @@ func (e *Encoder) encodeOidLookup(hashes []plumbing.Hash) (err error) { return } -func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumbing.Hash]uint32, idx Index) (extraEdges []uint32, err error) { +func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumbing.Hash]uint32, idx Index) (extraEdges []uint32, generationV2Data []uint64, err error) { + if idx.HasGenerationV2() { + generationV2Data = make([]uint64, 0, len(hashes)) + } for _, hash := range hashes { origIndex, _ := idx.GetIndexByHash(hash) commitData, _ := idx.GetCommitDataByIndex(origIndex) @@ -173,6 +199,9 @@ func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumb if err = binary.WriteUint64(e, unixTime); err != nil { return } + if generationV2Data != nil { + generationV2Data = append(generationV2Data, commitData.GenerationV2Data()) + } } return } @@ -186,6 +215,35 @@ func (e *Encoder) encodeExtraEdges(extraEdges []uint32) (err error) { return } +func (e *Encoder) encodeGenerationV2Data(generationV2Data []uint64) (overflows []uint64, err error) { + head := 0 + for _, data := range generationV2Data { + if data >= 0x80000000 { + // overflow + if err = binary.WriteUint32(e, uint32(head)|0x80000000); err != nil { + return nil, err + } + generationV2Data[head] = data + head++ + continue + } + if err = binary.WriteUint32(e, uint32(data)); err != nil { + return nil, err + } + } + + return generationV2Data[:head], nil +} + +func (e *Encoder) encodeGenerationV2Overflow(overflows []uint64) (err error) { + for _, overflow := range overflows { + if err = binary.WriteUint64(e, overflow); err != nil { + return + } + } + return +} + func (e *Encoder) encodeChecksum() error { _, err := e.Write(e.hash.Sum(nil)[:hash.Size]) return err diff --git a/plumbing/format/commitgraph/v2/file.go b/plumbing/format/commitgraph/v2/file.go index 69e02508e..c5f61e4de 100644 --- a/plumbing/format/commitgraph/v2/file.go +++ b/plumbing/format/commitgraph/v2/file.go @@ -34,14 +34,23 @@ var ( ) const ( - commitDataSize = 16 + szUint32 = 4 + szUint64 = 8 + + szSignature = 4 + szHeader = 4 + szCommitData = 2*szUint32 + szUint64 + + lenFanout = 256 ) type fileIndex struct { - reader ReaderAtCloser - fanout [256]uint32 - offsets [9]int64 - parent Index + reader ReaderAtCloser + fanout [lenFanout]uint32 + offsets [lenChunks]int64 + parent Index + hasGenerationV2 bool + minimumNumberOfHashes uint32 } // ReaderAtCloser is an interface that combines io.ReaderAt and io.Closer. @@ -74,6 +83,15 @@ func OpenFileIndexWithParent(reader ReaderAtCloser, parent Index) (Index, error) return nil, err } + fi.hasGenerationV2 = fi.offsets[GenerationDataChunk] > 0 + if fi.parent != nil { + fi.hasGenerationV2 = fi.hasGenerationV2 && fi.parent.HasGenerationV2() + } + + if fi.parent != nil { + fi.minimumNumberOfHashes = fi.parent.MaximumNumberOfHashes() + } + return fi, nil } @@ -94,7 +112,7 @@ func (fi *fileIndex) Close() (err error) { func (fi *fileIndex) verifyFileHeader() error { // Verify file signature - signature := make([]byte, 4) + signature := make([]byte, szSignature) if _, err := fi.reader.ReadAt(signature, 0); err != nil { return err } @@ -103,8 +121,8 @@ func (fi *fileIndex) verifyFileHeader() error { } // Read and verify the file header - header := make([]byte, 4) - if _, err := fi.reader.ReadAt(header, 4); err != nil { + header := make([]byte, szHeader) + if _, err := fi.reader.ReadAt(header, szHeader); err != nil { return err } if header[0] != 1 { @@ -120,10 +138,11 @@ func (fi *fileIndex) verifyFileHeader() error { } func (fi *fileIndex) readChunkHeaders() error { - chunkID := make([]byte, 4) + // The chunk table is a list of 4-byte chunk signatures and uint64 offsets into the file + chunkID := make([]byte, szChunkSig) for i := 0; ; i++ { - chunkHeader := io.NewSectionReader(fi.reader, 8+(int64(i)*12), 12) - if _, err := io.ReadAtLeast(chunkHeader, chunkID, 4); err != nil { + chunkHeader := io.NewSectionReader(fi.reader, szSignature+szHeader+(int64(i)*(szChunkSig+szUint64)), szChunkSig+szUint64) + if _, err := io.ReadAtLeast(chunkHeader, chunkID, szChunkSig); err != nil { return err } chunkOffset, err := binary.ReadUint64(chunkHeader) @@ -149,7 +168,9 @@ func (fi *fileIndex) readChunkHeaders() error { } func (fi *fileIndex) readFanout() error { - fanoutReader := io.NewSectionReader(fi.reader, fi.offsets[OIDFanoutChunk], 256*4) + // The Fanout table is a 256 entry table of the number (as uint32) of OIDs with first byte at most i. + // Thus F[255] stores the total number of commits (N) + fanoutReader := io.NewSectionReader(fi.reader, fi.offsets[OIDFanoutChunk], lenFanout*szUint32) for i := 0; i < 256; i++ { fanoutValue, err := binary.ReadUint32(fanoutReader) if err != nil { @@ -185,7 +206,7 @@ func (fi *fileIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) { if cmp < 0 { high = mid } else if cmp == 0 { - return mid, nil + return mid + fi.minimumNumberOfHashes, nil } else { low = mid + 1 } @@ -196,7 +217,7 @@ func (fi *fileIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) { if err != nil { return 0, err } - return idx + fi.fanout[0xff], nil + return idx, nil } return 0, plumbing.ErrObjectNotFound @@ -204,23 +225,24 @@ func (fi *fileIndex) GetIndexByHash(h plumbing.Hash) (uint32, error) { // GetCommitDataByIndex returns the commit data for the given index in the commit-graph. func (fi *fileIndex) GetCommitDataByIndex(idx uint32) (*CommitData, error) { - if idx >= fi.fanout[0xff] { + if idx < fi.minimumNumberOfHashes { if fi.parent != nil { - data, err := fi.parent.GetCommitDataByIndex(idx - fi.fanout[0xff]) + data, err := fi.parent.GetCommitDataByIndex(idx) if err != nil { return nil, err } - for i := range data.ParentIndexes { - data.ParentIndexes[i] += fi.fanout[0xff] - } return data, nil } return nil, plumbing.ErrObjectNotFound } + idx -= fi.minimumNumberOfHashes + if idx >= fi.fanout[0xff] { + return nil, plumbing.ErrObjectNotFound + } - offset := fi.offsets[CommitDataChunk] + int64(idx)*(hash.Size+commitDataSize) - commitDataReader := io.NewSectionReader(fi.reader, offset, hash.Size+commitDataSize) + offset := fi.offsets[CommitDataChunk] + int64(idx)*(hash.Size+szCommitData) + commitDataReader := io.NewSectionReader(fi.reader, offset, hash.Size+szCommitData) treeHash, err := binary.ReadHash(commitDataReader) if err != nil { @@ -241,10 +263,11 @@ func (fi *fileIndex) GetCommitDataByIndex(idx uint32) (*CommitData, error) { var parentIndexes []uint32 if parent2&parentOctopusUsed == parentOctopusUsed { - // Octopus merge + // Octopus merge - Look-up the extra parents from the extra edge list + // The extra edge list is a list of uint32s, each of which is an index into the Commit Data table, terminated by a index with the most significant bit on. parentIndexes = []uint32{parent1 & parentOctopusMask} - offset := fi.offsets[ExtraEdgeListChunk] + 4*int64(parent2&parentOctopusMask) - buf := make([]byte, 4) + offset := fi.offsets[ExtraEdgeListChunk] + szUint32*int64(parent2&parentOctopusMask) + buf := make([]byte, szUint32) for { _, err := fi.reader.ReadAt(buf, offset) if err != nil { @@ -252,7 +275,7 @@ func (fi *fileIndex) GetCommitDataByIndex(idx uint32) (*CommitData, error) { } parent := encbin.BigEndian.Uint32(buf) - offset += 4 + offset += szUint32 parentIndexes = append(parentIndexes, parent&parentOctopusMask) if parent&parentLast == parentLast { break @@ -269,23 +292,57 @@ func (fi *fileIndex) GetCommitDataByIndex(idx uint32) (*CommitData, error) { return nil, err } + generationV2 := uint64(0) + + if fi.hasGenerationV2 { + // set the GenerationV2 result to the commit time + generationV2 = uint64(genAndTime & 0x3FFFFFFFF) + + // Next read the generation (offset) data from the generation data chunk + offset := fi.offsets[GenerationDataChunk] + int64(idx)*szUint32 + buf := make([]byte, szUint32) + if _, err := fi.reader.ReadAt(buf, offset); err != nil { + return nil, err + } + genV2Data := encbin.BigEndian.Uint32(buf) + + // check if the data is an overflow that needs to be looked up in the overflow chunk + if genV2Data&0x80000000 > 0 { + // Overflow + offset := fi.offsets[GenerationDataOverflowChunk] + int64(genV2Data&0x7fffffff)*szUint64 + buf := make([]byte, 8) + if _, err := fi.reader.ReadAt(buf, offset); err != nil { + return nil, err + } + + generationV2 += encbin.BigEndian.Uint64(buf) + } else { + generationV2 += uint64(genV2Data) + } + } + return &CommitData{ TreeHash: treeHash, ParentIndexes: parentIndexes, ParentHashes: parentHashes, Generation: genAndTime >> 34, + GenerationV2: generationV2, When: time.Unix(int64(genAndTime&0x3FFFFFFFF), 0), }, nil } // GetHashByIndex looks up the hash for the given index in the commit-graph. func (fi *fileIndex) GetHashByIndex(idx uint32) (found plumbing.Hash, err error) { - if idx >= fi.fanout[0xff] { + if idx < fi.minimumNumberOfHashes { if fi.parent != nil { - return fi.parent.GetHashByIndex(idx - fi.fanout[0xff]) + return fi.parent.GetHashByIndex(idx) } return found, ErrMalformedCommitGraphFile } + idx -= fi.minimumNumberOfHashes + if idx >= fi.fanout[0xff] { + return found, ErrMalformedCommitGraphFile + } offset := fi.offsets[OIDLookupChunk] + int64(idx)*hash.Size if _, err := fi.reader.ReadAt(found[:], offset); err != nil { @@ -299,9 +356,9 @@ func (fi *fileIndex) getHashesFromIndexes(indexes []uint32) ([]plumbing.Hash, er hashes := make([]plumbing.Hash, len(indexes)) for i, idx := range indexes { - if idx >= fi.fanout[0xff] { + if idx < fi.minimumNumberOfHashes { if fi.parent != nil { - hash, err := fi.parent.GetHashByIndex(idx - fi.fanout[0xff]) + hash, err := fi.parent.GetHashByIndex(idx) if err != nil { return nil, err } @@ -312,6 +369,11 @@ func (fi *fileIndex) getHashesFromIndexes(indexes []uint32) ([]plumbing.Hash, er return nil, ErrMalformedCommitGraphFile } + idx -= fi.minimumNumberOfHashes + if idx >= fi.fanout[0xff] { + return nil, ErrMalformedCommitGraphFile + } + offset := fi.offsets[OIDLookupChunk] + int64(idx)*hash.Size if _, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil { return nil, err @@ -323,16 +385,28 @@ func (fi *fileIndex) getHashesFromIndexes(indexes []uint32) ([]plumbing.Hash, er // Hashes returns all the hashes that are available in the index. func (fi *fileIndex) Hashes() []plumbing.Hash { - hashes := make([]plumbing.Hash, fi.fanout[0xff]) + hashes := make([]plumbing.Hash, fi.fanout[0xff]+fi.minimumNumberOfHashes) + for i := uint32(0); i < fi.minimumNumberOfHashes; i++ { + hash, err := fi.parent.GetHashByIndex(i) + if err != nil { + return nil + } + hashes[i] = hash + } + for i := uint32(0); i < fi.fanout[0xff]; i++ { offset := fi.offsets[OIDLookupChunk] + int64(i)*hash.Size - if n, err := fi.reader.ReadAt(hashes[i][:], offset); err != nil || n < hash.Size { + if n, err := fi.reader.ReadAt(hashes[i+fi.minimumNumberOfHashes][:], offset); err != nil || n < hash.Size { return nil } } - if fi.parent != nil { - parentHashes := fi.parent.Hashes() - hashes = append(hashes, parentHashes...) - } return hashes } + +func (fi *fileIndex) HasGenerationV2() bool { + return fi.hasGenerationV2 +} + +func (fi *fileIndex) MaximumNumberOfHashes() uint32 { + return fi.minimumNumberOfHashes + fi.fanout[0xff] +} diff --git a/plumbing/format/commitgraph/v2/memory.go b/plumbing/format/commitgraph/v2/memory.go index ab7ddfad9..8de0c5f08 100644 --- a/plumbing/format/commitgraph/v2/memory.go +++ b/plumbing/format/commitgraph/v2/memory.go @@ -1,14 +1,17 @@ package v2 import ( + "math" + "github.com/go-git/go-git/v5/plumbing" ) // MemoryIndex provides a way to build the commit-graph in memory // for later encoding to file. type MemoryIndex struct { - commitData []commitData - indexMap map[plumbing.Hash]uint32 + commitData []commitData + indexMap map[plumbing.Hash]uint32 + hasGenerationV2 bool } type commitData struct { @@ -19,7 +22,8 @@ type commitData struct { // NewMemoryIndex creates in-memory commit graph representation func NewMemoryIndex() *MemoryIndex { return &MemoryIndex{ - indexMap: make(map[plumbing.Hash]uint32), + indexMap: make(map[plumbing.Hash]uint32), + hasGenerationV2: true, } } @@ -83,9 +87,21 @@ func (mi *MemoryIndex) Add(hash plumbing.Hash, data *CommitData) { data.ParentIndexes = nil mi.indexMap[hash] = uint32(len(mi.commitData)) mi.commitData = append(mi.commitData, commitData{Hash: hash, CommitData: data}) + if data.GenerationV2 == math.MaxUint64 { // if GenerationV2 is not available reset it to zero + data.GenerationV2 = 0 + } + mi.hasGenerationV2 = mi.hasGenerationV2 && data.GenerationV2 != 0 +} + +func (mi *MemoryIndex) HasGenerationV2() bool { + return mi.hasGenerationV2 } // Close closes the index func (mi *MemoryIndex) Close() error { return nil } + +func (mi *MemoryIndex) MaximumNumberOfHashes() uint32 { + return uint32(len(mi.indexMap)) +} diff --git a/plumbing/object/commitgraph/commitnode.go b/plumbing/object/commitgraph/commitnode.go index d92c9064f..47227d434 100644 --- a/plumbing/object/commitgraph/commitnode.go +++ b/plumbing/object/commitgraph/commitnode.go @@ -29,6 +29,10 @@ type CommitNode interface { // Generation returns the generation of the commit for reachability analysis. // Objects with newer generation are not reachable from objects of older generation. Generation() uint64 + // GenerationV2 stores the corrected commit date for the commits + // It combines the contents of the GDA2 and GDO2 sections of the commit-graph + // with the commit time portion of the CDAT section. + GenerationV2() uint64 // Commit returns the full commit object from the node Commit() (*object.Commit, error) } diff --git a/plumbing/object/commitgraph/commitnode_graph.go b/plumbing/object/commitgraph/commitnode_graph.go index 252b5181e..0f51e3be9 100644 --- a/plumbing/object/commitgraph/commitnode_graph.go +++ b/plumbing/object/commitgraph/commitnode_graph.go @@ -120,6 +120,13 @@ func (c *graphCommitNode) Generation() uint64 { return c.commitData.Generation } +func (c *graphCommitNode) GenerationV2() uint64 { + // If the commit-graph file was generated with older Git version that + // set the generation to zero for every commit the generation assumption + // is still valid. It is just less useful. + return c.commitData.GenerationV2 +} + func (c *graphCommitNode) Commit() (*object.Commit, error) { return object.GetCommit(c.gci.s, c.hash) } diff --git a/plumbing/object/commitgraph/commitnode_object.go b/plumbing/object/commitgraph/commitnode_object.go index 1bd37e3e0..7256bed2f 100644 --- a/plumbing/object/commitgraph/commitnode_object.go +++ b/plumbing/object/commitgraph/commitnode_object.go @@ -85,6 +85,13 @@ func (c *objectCommitNode) Generation() uint64 { return math.MaxUint64 } +func (c *objectCommitNode) GenerationV2() uint64 { + // Commit nodes representing objects outside of the commit graph can never + // be reached by objects from the commit-graph thus we return the highest + // possible value. + return math.MaxUint64 +} + func (c *objectCommitNode) Commit() (*object.Commit, error) { return c.commit, nil } diff --git a/plumbing/object/commitgraph/commitnode_walker_author_order.go b/plumbing/object/commitgraph/commitnode_walker_author_order.go new file mode 100644 index 000000000..f5b23cc51 --- /dev/null +++ b/plumbing/object/commitgraph/commitnode_walker_author_order.go @@ -0,0 +1,61 @@ +package commitgraph + +import ( + "github.com/go-git/go-git/v5/plumbing" + + "github.com/emirpasic/gods/trees/binaryheap" +) + +// NewCommitNodeIterAuthorDateOrder returns a CommitNodeIter that walks the commit history, +// starting at the given commit and visiting its parents in Author Time order but with the +// constraint that no parent is emitted before its children are emitted. +// +// This matches `git log --author-order` +// +// This ordering requires that commit objects need to be loaded into memory - thus this +// ordering is likely to be slower than other orderings. +func NewCommitNodeIterAuthorDateOrder(c CommitNode, + seenExternal map[plumbing.Hash]bool, + ignore []plumbing.Hash, +) CommitNodeIter { + seen := make(map[plumbing.Hash]struct{}) + for _, h := range ignore { + seen[h] = struct{}{} + } + for h, ext := range seenExternal { + if ext { + seen[h] = struct{}{} + } + } + inCounts := make(map[plumbing.Hash]int) + + exploreHeap := &commitNodeHeap{binaryheap.NewWith(generationAndDateOrderComparator)} + exploreHeap.Push(c) + + visitHeap := &commitNodeHeap{binaryheap.NewWith(func(left, right interface{}) int { + leftCommit, err := left.(CommitNode).Commit() + if err != nil { + return -1 + } + rightCommit, err := right.(CommitNode).Commit() + if err != nil { + return -1 + } + + switch { + case rightCommit.Author.When.Before(leftCommit.Author.When): + return -1 + case leftCommit.Author.When.Before(rightCommit.Author.When): + return 1 + } + return 0 + })} + visitHeap.Push(c) + + return &commitNodeIteratorTopological{ + exploreStack: exploreHeap, + visitStack: visitHeap, + inCounts: inCounts, + ignore: seen, + } +} diff --git a/plumbing/object/commitgraph/commitnode_walker_ctime.go b/plumbing/object/commitgraph/commitnode_walker_ctime.go index c26873ce5..3ab9e6e87 100644 --- a/plumbing/object/commitgraph/commitnode_walker_ctime.go +++ b/plumbing/object/commitgraph/commitnode_walker_ctime.go @@ -17,7 +17,8 @@ type commitNodeIteratorByCTime struct { // NewCommitNodeIterCTime returns a CommitNodeIter that walks the commit history, // starting at the given commit and visiting its parents while preserving Committer Time order. -// this appears to be the closest order to `git log` +// this is close in order to `git log` but does not guarantee topological order and will +// order things incorrectly occasionally. // The given callback will be called for each visited commit. Each commit will // be visited only once. If the callback returns an error, walking will stop // and will return the error. Other errors might be returned if the history diff --git a/plumbing/object/commitgraph/commitnode_walker_date_order.go b/plumbing/object/commitgraph/commitnode_walker_date_order.go new file mode 100644 index 000000000..659a4fa44 --- /dev/null +++ b/plumbing/object/commitgraph/commitnode_walker_date_order.go @@ -0,0 +1,41 @@ +package commitgraph + +import ( + "github.com/go-git/go-git/v5/plumbing" + + "github.com/emirpasic/gods/trees/binaryheap" +) + +// NewCommitNodeIterDateOrder returns a CommitNodeIter that walks the commit history, +// starting at the given commit and visiting its parents in Committer Time and Generation order, +// but with the constraint that no parent is emitted before its children are emitted. +// +// This matches `git log --date-order` +func NewCommitNodeIterDateOrder(c CommitNode, + seenExternal map[plumbing.Hash]bool, + ignore []plumbing.Hash, +) CommitNodeIter { + seen := make(map[plumbing.Hash]struct{}) + for _, h := range ignore { + seen[h] = struct{}{} + } + for h, ext := range seenExternal { + if ext { + seen[h] = struct{}{} + } + } + inCounts := make(map[plumbing.Hash]int) + + exploreHeap := &commitNodeHeap{binaryheap.NewWith(generationAndDateOrderComparator)} + exploreHeap.Push(c) + + visitHeap := &commitNodeHeap{binaryheap.NewWith(generationAndDateOrderComparator)} + visitHeap.Push(c) + + return &commitNodeIteratorTopological{ + exploreStack: exploreHeap, + visitStack: visitHeap, + inCounts: inCounts, + ignore: seen, + } +} diff --git a/plumbing/object/commitgraph/commitnode_walker_helper.go b/plumbing/object/commitgraph/commitnode_walker_helper.go new file mode 100644 index 000000000..c54f6caae --- /dev/null +++ b/plumbing/object/commitgraph/commitnode_walker_helper.go @@ -0,0 +1,164 @@ +package commitgraph + +import ( + "math" + + "github.com/go-git/go-git/v5/plumbing" + + "github.com/emirpasic/gods/trees/binaryheap" +) + +// commitNodeStackable represents a common interface between heaps and stacks +type commitNodeStackable interface { + Push(c CommitNode) + Pop() (CommitNode, bool) + Peek() (CommitNode, bool) + Size() int +} + +// commitNodeLifo is a stack implementation using an underlying slice +type commitNodeLifo struct { + l []CommitNode +} + +// Push pushes a new CommitNode to the stack +func (l *commitNodeLifo) Push(c CommitNode) { + l.l = append(l.l, c) +} + +// Pop pops the most recently added CommitNode from the stack +func (l *commitNodeLifo) Pop() (CommitNode, bool) { + if len(l.l) == 0 { + return nil, false + } + c := l.l[len(l.l)-1] + l.l = l.l[:len(l.l)-1] + return c, true +} + +// Peek returns the most recently added CommitNode from the stack without removing it +func (l *commitNodeLifo) Peek() (CommitNode, bool) { + if len(l.l) == 0 { + return nil, false + } + return l.l[len(l.l)-1], true +} + +// Size returns the number of CommitNodes in the stack +func (l *commitNodeLifo) Size() int { + return len(l.l) +} + +// commitNodeHeap is a stack implementation using an underlying binary heap +type commitNodeHeap struct { + *binaryheap.Heap +} + +// Push pushes a new CommitNode to the heap +func (h *commitNodeHeap) Push(c CommitNode) { + h.Heap.Push(c) +} + +// Pop removes top element on heap and returns it, or nil if heap is empty. +// Second return parameter is true, unless the heap was empty and there was nothing to pop. +func (h *commitNodeHeap) Pop() (CommitNode, bool) { + c, ok := h.Heap.Pop() + if !ok { + return nil, false + } + return c.(CommitNode), true +} + +// Peek returns top element on the heap without removing it, or nil if heap is empty. +// Second return parameter is true, unless the heap was empty and there was nothing to peek. +func (h *commitNodeHeap) Peek() (CommitNode, bool) { + c, ok := h.Heap.Peek() + if !ok { + return nil, false + } + return c.(CommitNode), true +} + +// Size returns number of elements within the heap. +func (h *commitNodeHeap) Size() int { + return h.Heap.Size() +} + +// generationAndDateOrderComparator compares two CommitNode objects based on their generation and commit time. +// If the left CommitNode object is in a higher generation or is newer than the right one, it returns a -1. +// If the left CommitNode object is in a lower generation or is older than the right one, it returns a 1. +// If the two CommitNode objects have the same commit time and generation, it returns 0. +func generationAndDateOrderComparator(left, right interface{}) int { + leftCommit := left.(CommitNode) + rightCommit := right.(CommitNode) + + // if GenerationV2 is MaxUint64, then the node is not in the graph + if leftCommit.GenerationV2() == math.MaxUint64 { + if rightCommit.GenerationV2() == math.MaxUint64 { + switch { + case rightCommit.CommitTime().Before(leftCommit.CommitTime()): + return -1 + case leftCommit.CommitTime().Before(rightCommit.CommitTime()): + return 1 + } + return 0 + } + // left is not in the graph, but right is, so it is newer than the right + return -1 + } + + if rightCommit.GenerationV2() == math.MaxInt64 { + // the right is not in the graph, therefore the left is before the right + return 1 + } + + if leftCommit.GenerationV2() == 0 || rightCommit.GenerationV2() == 0 { + // We need to assess generation and date + if leftCommit.Generation() < rightCommit.Generation() { + return 1 + } + if leftCommit.Generation() > rightCommit.Generation() { + return -1 + } + switch { + case rightCommit.CommitTime().Before(leftCommit.CommitTime()): + return -1 + case leftCommit.CommitTime().Before(rightCommit.CommitTime()): + return 1 + } + return 0 + } + + if leftCommit.GenerationV2() < rightCommit.GenerationV2() { + return 1 + } + if leftCommit.GenerationV2() > rightCommit.GenerationV2() { + return -1 + } + + return 0 +} + +// composeIgnores composes the ignore list with the provided seenExternal list +func composeIgnores(ignore []plumbing.Hash, seenExternal map[plumbing.Hash]bool) map[plumbing.Hash]struct{} { + if len(ignore) == 0 { + seen := make(map[plumbing.Hash]struct{}) + for h, ext := range seenExternal { + if ext { + seen[h] = struct{}{} + } + } + return seen + } + + seen := make(map[plumbing.Hash]struct{}) + for _, h := range ignore { + seen[h] = struct{}{} + } + for h, ext := range seenExternal { + if ext { + seen[h] = struct{}{} + } + } + return seen +} diff --git a/plumbing/object/commitgraph/commitnode_walker_test.go b/plumbing/object/commitgraph/commitnode_walker_test.go new file mode 100644 index 000000000..1e09c0be5 --- /dev/null +++ b/plumbing/object/commitgraph/commitnode_walker_test.go @@ -0,0 +1,187 @@ +package commitgraph + +import ( + "strings" + + "github.com/go-git/go-git/v5/plumbing" + commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" + + fixtures "github.com/go-git/go-git-fixtures/v4" + . "gopkg.in/check.v1" +) + +func (s *CommitNodeSuite) TestCommitNodeIter(c *C) { + f := fixtures.ByTag("commit-graph-chain-2").One() + + storer := unpackRepository(f) + + index, err := commitgraph.OpenChainOrFileIndex(storer.Filesystem()) + c.Assert(err, IsNil) + + nodeIndex := NewGraphCommitNodeIndex(index, storer) + + head, err := nodeIndex.Get(plumbing.NewHash("ec6f456c0e8c7058a29611429965aa05c190b54b")) + c.Assert(err, IsNil) + + testTopoOrder(c, head) + testDateOrder(c, head) + testAuthorDateOrder(c, head) +} + +func testTopoOrder(c *C, head CommitNode) { + iter := NewCommitNodeIterTopoOrder( + head, + nil, + nil, + ) + + var commits []string + iter.ForEach(func(c CommitNode) error { + commits = append(commits, c.ID().String()) + return nil + }) + c.Assert(commits, DeepEquals, strings.Split(`ec6f456c0e8c7058a29611429965aa05c190b54b +d82f291cde9987322c8a0c81a325e1ba6159684c +3048d280d2d5b258d9e582a226ff4bbed34fd5c9 +27aa8cdd2431068606741a589383c02c149ea625 +fa058d42fa3bc53f39108a56dad67157169b2191 +6c629843a1750a27c9af01ed2985f362f619c47a +d10a0e7c1f340a6cfc14540a5f8c508ce7e2eabf +d0a18ccd8eea3bdabc76d6dc5420af1ea30aae9f +cf2874632223220e0445abf0a7806dc772c0b37a +758ac33217f092bfcded4ad4774954ac054c9609 +214e1dca024fb6da5ed65564d2de734df5dc2127 +70923099e61fa33f0bc5256d2f938fa44c4df10e +bcaa1ac5644b16f1febb72f31e204720b7bb8934 +e1d8866ffa78fa16d2f39b0ba5344a7269ee5371 +2275fa7d0c75d20103f90b0e1616937d5a9fc5e6 +bdd9a92789d4a86b20a8d3df462df373f41acf23 +b359f11ea09e642695edcd114b463da4395b10c1 +6f43e8933ba3c04072d5d104acc6118aac3e52ee +ccafe8bd5f9dbfb8b98b0da03ced29608dcfdeec +939814f341fdd5d35e81a3845a33c4fedb19d2d2 +5f5ad88bf2babe506f927d64d2b7a1e1493dc2ae +a2014124ca3b3f9ff28fbab0a83ce3c71bf4622e +77906b653c3eb8a1cd5bd7254e161c00c6086d83 +465cba710284204f9851854587c2887c247222db +b9471b13256703d3f5eb88b280b4a16ce325ec1b +62925030859646daeeaf5a4d386a0c41e00dda8a +5f56aea0ca8b74215a5b982bca32236e1e28c76b +23148841baa5dbce48f6adcb7ddf83dcd97debb3 +c336d16298a017486c4164c40f8acb28afe64e84 +31eae7b619d166c366bf5df4991f04ba8cebea0a +d2a38b4a5965d529566566640519d03d2bd10f6c +b977a025ca21e3b5ca123d8093bd7917694f6da7 +35b585759cbf29f8ec428ef89da20705d59f99ec +c2bbf9fe8009b22d0f390f3c8c3f13937067590f +fc9f0643b21cfe571046e27e0c4565f3a1ee96c8 +c088fd6a7e1a38e9d5a9815265cb575bb08d08ff +5fddbeb678bd2c36c5e5c891ab8f2b143ced5baf +5d7303c49ac984a9fec60523f2d5297682e16646`, "\n")) +} + +func testDateOrder(c *C, head CommitNode) { + iter := NewCommitNodeIterDateOrder( + head, + nil, + nil, + ) + + var commits []string + iter.ForEach(func(c CommitNode) error { + commits = append(commits, c.ID().String()) + return nil + }) + + c.Assert(commits, DeepEquals, strings.Split(`ec6f456c0e8c7058a29611429965aa05c190b54b +3048d280d2d5b258d9e582a226ff4bbed34fd5c9 +d82f291cde9987322c8a0c81a325e1ba6159684c +27aa8cdd2431068606741a589383c02c149ea625 +fa058d42fa3bc53f39108a56dad67157169b2191 +d0a18ccd8eea3bdabc76d6dc5420af1ea30aae9f +6c629843a1750a27c9af01ed2985f362f619c47a +cf2874632223220e0445abf0a7806dc772c0b37a +d10a0e7c1f340a6cfc14540a5f8c508ce7e2eabf +758ac33217f092bfcded4ad4774954ac054c9609 +214e1dca024fb6da5ed65564d2de734df5dc2127 +70923099e61fa33f0bc5256d2f938fa44c4df10e +bcaa1ac5644b16f1febb72f31e204720b7bb8934 +e1d8866ffa78fa16d2f39b0ba5344a7269ee5371 +2275fa7d0c75d20103f90b0e1616937d5a9fc5e6 +bdd9a92789d4a86b20a8d3df462df373f41acf23 +b359f11ea09e642695edcd114b463da4395b10c1 +6f43e8933ba3c04072d5d104acc6118aac3e52ee +ccafe8bd5f9dbfb8b98b0da03ced29608dcfdeec +939814f341fdd5d35e81a3845a33c4fedb19d2d2 +5f5ad88bf2babe506f927d64d2b7a1e1493dc2ae +a2014124ca3b3f9ff28fbab0a83ce3c71bf4622e +77906b653c3eb8a1cd5bd7254e161c00c6086d83 +465cba710284204f9851854587c2887c247222db +b9471b13256703d3f5eb88b280b4a16ce325ec1b +62925030859646daeeaf5a4d386a0c41e00dda8a +5f56aea0ca8b74215a5b982bca32236e1e28c76b +23148841baa5dbce48f6adcb7ddf83dcd97debb3 +c336d16298a017486c4164c40f8acb28afe64e84 +31eae7b619d166c366bf5df4991f04ba8cebea0a +b977a025ca21e3b5ca123d8093bd7917694f6da7 +d2a38b4a5965d529566566640519d03d2bd10f6c +35b585759cbf29f8ec428ef89da20705d59f99ec +c2bbf9fe8009b22d0f390f3c8c3f13937067590f +fc9f0643b21cfe571046e27e0c4565f3a1ee96c8 +c088fd6a7e1a38e9d5a9815265cb575bb08d08ff +5fddbeb678bd2c36c5e5c891ab8f2b143ced5baf +5d7303c49ac984a9fec60523f2d5297682e16646`, "\n")) +} + +func testAuthorDateOrder(c *C, head CommitNode) { + iter := NewCommitNodeIterAuthorDateOrder( + head, + nil, + nil, + ) + + var commits []string + iter.ForEach(func(c CommitNode) error { + commits = append(commits, c.ID().String()) + return nil + }) + + c.Assert(commits, DeepEquals, strings.Split(`ec6f456c0e8c7058a29611429965aa05c190b54b +3048d280d2d5b258d9e582a226ff4bbed34fd5c9 +d82f291cde9987322c8a0c81a325e1ba6159684c +27aa8cdd2431068606741a589383c02c149ea625 +fa058d42fa3bc53f39108a56dad67157169b2191 +d0a18ccd8eea3bdabc76d6dc5420af1ea30aae9f +6c629843a1750a27c9af01ed2985f362f619c47a +cf2874632223220e0445abf0a7806dc772c0b37a +d10a0e7c1f340a6cfc14540a5f8c508ce7e2eabf +758ac33217f092bfcded4ad4774954ac054c9609 +214e1dca024fb6da5ed65564d2de734df5dc2127 +70923099e61fa33f0bc5256d2f938fa44c4df10e +bcaa1ac5644b16f1febb72f31e204720b7bb8934 +e1d8866ffa78fa16d2f39b0ba5344a7269ee5371 +2275fa7d0c75d20103f90b0e1616937d5a9fc5e6 +bdd9a92789d4a86b20a8d3df462df373f41acf23 +b359f11ea09e642695edcd114b463da4395b10c1 +6f43e8933ba3c04072d5d104acc6118aac3e52ee +ccafe8bd5f9dbfb8b98b0da03ced29608dcfdeec +939814f341fdd5d35e81a3845a33c4fedb19d2d2 +5f5ad88bf2babe506f927d64d2b7a1e1493dc2ae +a2014124ca3b3f9ff28fbab0a83ce3c71bf4622e +77906b653c3eb8a1cd5bd7254e161c00c6086d83 +465cba710284204f9851854587c2887c247222db +b9471b13256703d3f5eb88b280b4a16ce325ec1b +5f56aea0ca8b74215a5b982bca32236e1e28c76b +62925030859646daeeaf5a4d386a0c41e00dda8a +23148841baa5dbce48f6adcb7ddf83dcd97debb3 +c336d16298a017486c4164c40f8acb28afe64e84 +31eae7b619d166c366bf5df4991f04ba8cebea0a +b977a025ca21e3b5ca123d8093bd7917694f6da7 +d2a38b4a5965d529566566640519d03d2bd10f6c +35b585759cbf29f8ec428ef89da20705d59f99ec +c2bbf9fe8009b22d0f390f3c8c3f13937067590f +fc9f0643b21cfe571046e27e0c4565f3a1ee96c8 +c088fd6a7e1a38e9d5a9815265cb575bb08d08ff +5fddbeb678bd2c36c5e5c891ab8f2b143ced5baf +5d7303c49ac984a9fec60523f2d5297682e16646`, "\n")) +} diff --git a/plumbing/object/commitgraph/commitnode_walker_topo_order.go b/plumbing/object/commitgraph/commitnode_walker_topo_order.go new file mode 100644 index 000000000..29f4bb72e --- /dev/null +++ b/plumbing/object/commitgraph/commitnode_walker_topo_order.go @@ -0,0 +1,161 @@ +package commitgraph + +import ( + "io" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" + + "github.com/emirpasic/gods/trees/binaryheap" +) + +type commitNodeIteratorTopological struct { + exploreStack commitNodeStackable + visitStack commitNodeStackable + inCounts map[plumbing.Hash]int + + ignore map[plumbing.Hash]struct{} +} + +// NewCommitNodeIterTopoOrder returns a CommitNodeIter that walks the commit history, +// starting at the given commit and visiting its parents in a topological order but +// with the constraint that no parent is emitted before its children are emitted. +// +// This matches `git log --topo-order` +func NewCommitNodeIterTopoOrder(c CommitNode, + seenExternal map[plumbing.Hash]bool, + ignore []plumbing.Hash, +) CommitNodeIter { + seen := composeIgnores(ignore, seenExternal) + inCounts := make(map[plumbing.Hash]int) + + heap := &commitNodeHeap{binaryheap.NewWith(generationAndDateOrderComparator)} + heap.Push(c) + + lifo := &commitNodeLifo{make([]CommitNode, 0, 8)} + lifo.Push(c) + + return &commitNodeIteratorTopological{ + exploreStack: heap, + visitStack: lifo, + inCounts: inCounts, + ignore: seen, + } +} + +func (iter *commitNodeIteratorTopological) Next() (CommitNode, error) { + var next CommitNode + for { + var ok bool + next, ok = iter.visitStack.Pop() + if !ok { + return nil, io.EOF + } + + if iter.inCounts[next.ID()] == 0 { + break + } + } + + minimumLevel, generationV2 := next.GenerationV2(), true + if minimumLevel == 0 { + minimumLevel, generationV2 = next.Generation(), false + } + + parents := make([]CommitNode, 0, len(next.ParentHashes())) + for i := range next.ParentHashes() { + pc, err := next.ParentNode(i) + if err != nil { + return nil, err + } + + parents = append(parents, pc) + + if generationV2 { + if pc.GenerationV2() < minimumLevel { + minimumLevel = pc.GenerationV2() + } + continue + } + + if pc.Generation() < minimumLevel { + minimumLevel = pc.Generation() + } + } + + // EXPLORE + for { + toExplore, ok := iter.exploreStack.Peek() + if !ok { + break + } + + if toExplore.ID() != next.ID() && iter.exploreStack.Size() == 1 { + break + } + if generationV2 { + if toExplore.GenerationV2() < minimumLevel { + break + } + } else { + if toExplore.Generation() < minimumLevel { + break + } + } + + iter.exploreStack.Pop() + for i, h := range toExplore.ParentHashes() { + if _, has := iter.ignore[h]; has { + continue + } + iter.inCounts[h]++ + + if iter.inCounts[h] == 1 { + pc, err := toExplore.ParentNode(i) + if err != nil { + return nil, err + } + iter.exploreStack.Push(pc) + } + } + } + + // VISIT + for i, h := range next.ParentHashes() { + if _, has := iter.ignore[h]; has { + continue + } + iter.inCounts[h]-- + + if iter.inCounts[h] == 0 { + iter.visitStack.Push(parents[i]) + } + } + delete(iter.inCounts, next.ID()) + + return next, nil +} + +func (iter *commitNodeIteratorTopological) ForEach(cb func(CommitNode) error) error { + for { + obj, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + + return err + } + + if err := cb(obj); err != nil { + if err == storer.ErrStop { + return nil + } + + return err + } + } +} + +func (iter *commitNodeIteratorTopological) Close() { +} From 3ee0288b785e6e1070aa88c8fd5387b7afd80902 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 13 Oct 2023 22:27:48 +0100 Subject: [PATCH 18/56] build: bump golang.org/x/net from 0.15.0 to 0.17.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.15.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.15.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Paulo Gomes --- go.mod | 4 ++-- go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 9df2020f1..ff859fe8d 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,8 @@ require ( github.com/sergi/go-diff v1.1.0 github.com/skeema/knownhosts v1.2.0 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.13.0 - golang.org/x/net v0.15.0 + golang.org/x/crypto v0.14.0 + golang.org/x/net v0.17.0 golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 7c62b1094..c7e3c2bf4 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= @@ -98,8 +98,8 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -128,7 +128,7 @@ golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 136f6cc58f8469e8d5dfd1efeb6b1524215da22d Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 13 Oct 2023 22:51:43 +0100 Subject: [PATCH 19/56] build: Add dependabot configuration Configure dependabot to check for Go updates daily and GitHub actions weekly. The configuration sets the commit prefix, so that it aligns with the contributing guidelines. Signed-off-by: Paulo Gomes --- .github/dependabot.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/dependabot.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..6a7a66bfa --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,15 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "build" + + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "build" From f4b881ce59edb35afb6a375187953db5ea7cbd64 Mon Sep 17 00:00:00 2001 From: Aditya Sirish Date: Fri, 13 Oct 2023 18:37:50 -0400 Subject: [PATCH 20/56] build: Add 'git' as valid commit msg scheme Signed-off-by: Aditya Sirish --- .github/workflows/pr-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 04e704654..d7b115095 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -19,7 +19,7 @@ jobs: - name: Check Package Prefix uses: gsactions/commit-message-checker@v2 with: - pattern: '^(\*|plumbing|utils|config|_examples|internal|storage|cli|build): .+' + pattern: '^(\*|git|plumbing|utils|config|_examples|internal|storage|cli|build): .+' error: | Commit message(s) does not align with contribution acceptance criteria. From aca382bd2536df1533598bc1d0031cc8547b6cef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:19:50 +0000 Subject: [PATCH 21/56] build: bump github.com/google/go-cmp from 0.5.9 to 0.6.0 Bumps [github.com/google/go-cmp](https://github.com/google/go-cmp) from 0.5.9 to 0.6.0. - [Release notes](https://github.com/google/go-cmp/releases) - [Commits](https://github.com/google/go-cmp/compare/v0.5.9...v0.6.0) --- updated-dependencies: - dependency-name: github.com/google/go-cmp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 075446074..83bc402b0 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 github.com/jessevdk/go-flags v1.5.0 github.com/kevinburke/ssh_config v1.2.0 diff --git a/go.sum b/go.sum index 6baf1ba2d..f2bf9c11f 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= From 06df36caa7a98d76c5555ee1ec7bc1e04ddd4a89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:30:55 +0000 Subject: [PATCH 22/56] build: bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 2 +- .github/workflows/git.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index fbb867cc7..bfe9879af 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index 68776fce6..c6eff0868 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -22,7 +22,7 @@ jobs: go-version: 1.21.x - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install build dependencies run: sudo apt-get update && sudo apt-get install gettext libcurl4-openssl-dev diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74c9bdeb5..1ceee007c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure known hosts if: matrix.platform != 'ubuntu-latest' From fa818ac30356d8b124f8a0a2df296d211ce5ebb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:30:59 +0000 Subject: [PATCH 23/56] build: bump actions/setup-go from 3 to 4 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/git.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index 68776fce6..821987fd7 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.21.x diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74c9bdeb5..ef9624ac4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} From 32212de859ea5df73933efd304cb8e3c076fd6a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 05:36:40 +0000 Subject: [PATCH 24/56] build: bump github.com/skeema/knownhosts from 1.2.0 to 1.2.1 Bumps [github.com/skeema/knownhosts](https://github.com/skeema/knownhosts) from 1.2.0 to 1.2.1. - [Commits](https://github.com/skeema/knownhosts/compare/v1.2.0...v1.2.1) --- updated-dependencies: - dependency-name: github.com/skeema/knownhosts dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 83bc402b0..25be1e1d4 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/kevinburke/ssh_config v1.2.0 github.com/pjbgf/sha1cd v0.3.0 github.com/sergi/go-diff v1.1.0 - github.com/skeema/knownhosts v1.2.0 + github.com/skeema/knownhosts v1.2.1 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 diff --git a/go.sum b/go.sum index f2bf9c11f..9a2760a1c 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,8 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= -github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= +github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= From 4862643548f115dff8d402faf610e50408458cdf Mon Sep 17 00:00:00 2001 From: Aditya Sirish Date: Thu, 12 Oct 2023 00:15:14 -0400 Subject: [PATCH 25/56] git: remote, flip branch check to not-tag check In remote.updateLocalReferenceStorage, this commit updates a check to see if the reference is for a branch (in refs/heads) to checking the reference is not a tag. This change ensures that the subsequent fast-forward only handling clauses apply to references that are not standard branches stored in refs/heads. Signed-off-by: Aditya Sirish --- remote.go | 6 +-- remote_test.go | 141 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/remote.go b/remote.go index 2ffffe7b6..bbe275d5b 100644 --- a/remote.go +++ b/remote.go @@ -1198,9 +1198,9 @@ func (r *Remote) updateLocalReferenceStorage( old, _ := storer.ResolveReference(r.s, localName) new := plumbing.NewHashReference(localName, ref.Hash()) - // If the ref exists locally as a branch and force is not specified, - // only update if the new ref is an ancestor of the old - if old != nil && old.Name().IsBranch() && !force && !spec.IsForceUpdate() { + // If the ref exists locally as a non-tag and force is not + // specified, only update if the new ref is an ancestor of the old + if old != nil && !old.Name().IsTag() && !force && !spec.IsForceUpdate() { ff, err := isFastForward(r.s, old.Hash(), new.Hash()) if err != nil { return updated, err diff --git a/remote_test.go b/remote_test.go index e0c333294..81c60bcb8 100644 --- a/remote_test.go +++ b/remote_test.go @@ -4,16 +4,20 @@ import ( "bytes" "context" "errors" + "fmt" "io" "os" "path/filepath" "runtime" "strings" + "testing" "time" + "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/protocol/packp" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" "github.com/go-git/go-git/v5/plumbing/storer" @@ -1555,3 +1559,140 @@ func (s *RemoteSuite) TestFetchAfterShallowClone(c *C) { plumbing.NewSymbolicReference("HEAD", "refs/heads/master"), }) } + +func TestFetchFastForwardForCustomRef(t *testing.T) { + customRef := "refs/custom/branch" + // 1. Set up a remote with a URL + remoteURL := t.TempDir() + remoteRepo, err := PlainInit(remoteURL, true) + if err != nil { + t.Fatal(err) + } + + // 2. Add a commit with an empty tree to master and custom ref, also set HEAD + emptyTreeID := writeEmptyTree(t, remoteRepo) + writeCommitToRef(t, remoteRepo, "refs/heads/master", emptyTreeID, time.Now()) + writeCommitToRef(t, remoteRepo, customRef, emptyTreeID, time.Now()) + if err := remoteRepo.Storer.SetReference(plumbing.NewSymbolicReference(plumbing.HEAD, "refs/heads/master")); err != nil { + t.Fatal(err) + } + + // 3. Clone repo, then fetch the custom ref + // Note that using custom ref in ReferenceName has an IsBranch issue + localRepo, err := Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ + URL: remoteURL, + }) + if err != nil { + t.Fatal(err) + } + if err := localRepo.Fetch(&FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec(fmt.Sprintf("%s:%s", customRef, customRef)), + }, + }); err != nil { + t.Fatal(err) + } + + // 4. Make divergent changes + remoteCommitID := writeCommitToRef(t, remoteRepo, customRef, emptyTreeID, time.Now()) + // Consecutive calls to writeCommitToRef with time.Now() might have the same + // time value, explicitly set distinct ones to ensure the commit hashes + // differ + writeCommitToRef(t, localRepo, customRef, emptyTreeID, time.Now().Add(time.Second)) + + // 5. Try to fetch with fast-forward only mode + remote, err := localRepo.Remote(DefaultRemoteName) + if err != nil { + t.Fatal(err) + } + + err = remote.Fetch(&FetchOptions{RefSpecs: []config.RefSpec{ + config.RefSpec(fmt.Sprintf("%s:%s", customRef, customRef)), + }}) + if !errors.Is(err, ErrForceNeeded) { + t.Errorf("expected %v, got %v", ErrForceNeeded, err) + } + + // 6. Fetch with force + err = remote.Fetch(&FetchOptions{RefSpecs: []config.RefSpec{ + config.RefSpec(fmt.Sprintf("+%s:%s", customRef, customRef)), + }}) + if err != nil { + t.Errorf("unexpected error %v", err) + } + + // 7. Assert commit ID matches + ref, err := localRepo.Reference(plumbing.ReferenceName(customRef), true) + if err != nil { + t.Fatal(err) + } + if remoteCommitID != ref.Hash() { + t.Errorf("expected %s, got %s", remoteCommitID.String(), ref.Hash().String()) + } +} + +func writeEmptyTree(t *testing.T, repo *Repository) plumbing.Hash { + t.Helper() + + obj := repo.Storer.NewEncodedObject() + obj.SetType(plumbing.TreeObject) + + tree := object.Tree{Entries: nil} + if err := tree.Encode(obj); err != nil { + t.Fatal(err) + } + + treeID, err := repo.Storer.SetEncodedObject(obj) + if err != nil { + t.Fatal(err) + } + + return treeID +} + +func writeCommitToRef(t *testing.T, repo *Repository, refName string, treeID plumbing.Hash, when time.Time) plumbing.Hash { + t.Helper() + + ref, err := repo.Reference(plumbing.ReferenceName(refName), true) + if err != nil { + if errors.Is(err, plumbing.ErrReferenceNotFound) { + if err := repo.Storer.SetReference(plumbing.NewHashReference(plumbing.ReferenceName(refName), plumbing.ZeroHash)); err != nil { + t.Fatal(err) + } + + ref, err = repo.Reference(plumbing.ReferenceName(refName), true) + if err != nil { + t.Fatal(err) + } + } else { + t.Fatal(err) + } + } + + commit := &object.Commit{ + TreeHash: treeID, + Author: object.Signature{ + When: when, + }, + } + if !ref.Hash().IsZero() { + commit.ParentHashes = []plumbing.Hash{ref.Hash()} + } + + obj := repo.Storer.NewEncodedObject() + if err := commit.Encode(obj); err != nil { + t.Fatal(err) + } + + commitID, err := repo.Storer.SetEncodedObject(obj) + if err != nil { + t.Fatal(err) + } + + newRef := plumbing.NewHashReference(plumbing.ReferenceName(refName), commitID) + if err := repo.Storer.CheckAndSetReference(newRef, ref); err != nil { + t.Fatal(err) + } + + return commitID +} From 814abc098d033f77315d3bfb89ae5991aae10457 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 27 Jun 2023 21:33:01 +0100 Subject: [PATCH 26/56] *: Improve BenchmarkPlainClone The changes aim to make that specific benchmark more reliable for setting a baseline which can later be use to compare against future changes on the most basic feature of go-git: plain cloning repositories. --- repository_test.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/repository_test.go b/repository_test.go index f6839b6da..35a62f131 100644 --- a/repository_test.go +++ b/repository_test.go @@ -3311,20 +3311,25 @@ func BenchmarkObjects(b *testing.B) { } func BenchmarkPlainClone(b *testing.B) { - for i := 0; i < b.N; i++ { - t, err := os.MkdirTemp("", "") - if err != nil { - b.Fatal(err) - } - _, err = PlainClone(t, false, &CloneOptions{ - URL: "https://github.com/knqyf263/vuln-list", - Depth: 1, + b.StopTimer() + clone := func(b *testing.B) { + _, err := PlainClone(b.TempDir(), true, &CloneOptions{ + URL: "https://github.com/go-git/go-git.git", + Depth: 1, + Tags: NoTags, + SingleBranch: true, }) if err != nil { b.Error(err) } - b.StopTimer() - os.RemoveAll(t) - b.StartTimer() + } + + // Warm-up as the initial clone could have a higher cost which + // may skew results. + clone(b) + + b.StartTimer() + for i := 0; i < b.N; i++ { + clone(b) } } From 1c361adbc1f4b0e3d0743d11f187fb0b3ac4cb4d Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Thu, 20 Jul 2023 21:19:28 +0100 Subject: [PATCH 27/56] plumbing: Optimise memory consumption for filesystem storage Previously, as part of building the index representation, the resolveObject func would create an interim plumbing.MemoryObject, which would then be saved into storage via storage.SetEncodedObject. This meant that objects would be unnecessarily loaded into memory, to then be saved into disk. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The changes streamlines this process by: - Introducing the LazyObjectWriter interface which enables the write operation to take places directly against the filesystem-based storage. - Leverage multi-writers to process the input data once, while targeting multiple writers (e.g. hasher and storage). An additional change relates to the caching of object info children within Parser.get. The cache is now skipped when a seekable filesystem is being used. The impact of the changes can be observed when using seekable filesystem storages, especially when cloning large repositories. The stats below were captured by adapting the BenchmarkPlainClone test to clone https://github.com/torvalds/linux.git: pkg: github.com/go-git/go-git/v5 cpu: Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz │ /tmp/old │ /tmp/new │ │ sec/op │ sec/op vs base │ PlainClone-16 41.68 ± 17% 48.04 ± 9% +15.27% (p=0.015 n=6) │ /tmp/old │ /tmp/new │ │ B/op │ B/op vs base │ PlainClone-16 1127.8Mi ± 7% 256.7Mi ± 50% -77.23% (p=0.002 n=6) │ /tmp/old │ /tmp/new │ │ allocs/op │ allocs/op vs base │ PlainClone-16 3.125M ± 0% 3.800M ± 0% +21.60% (p=0.002 n=6) Notice that on average the memory consumption per operation is over 75% smaller. The time per operation increased by 15%, which may actual be less on long running applications, due to the decreased GC pressure and the garbage collection costs. Signed-off-by: Paulo Gomes --- plumbing/format/packfile/parser.go | 195 ++++++++++++++++++------ plumbing/format/packfile/patch_delta.go | 102 +++++++++++++ storage/filesystem/object.go | 13 ++ 3 files changed, 260 insertions(+), 50 deletions(-) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index edbc0e796..62f1d13cb 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -3,6 +3,7 @@ package packfile import ( "bytes" "errors" + "fmt" "io" "github.com/go-git/go-git/v5/plumbing" @@ -174,13 +175,25 @@ func (p *Parser) init() error { return nil } +type objectHeaderWriter func(typ plumbing.ObjectType, sz int64) error + +type lazyObjectWriter interface { + // LazyWriter enables an object to be lazily written. + // It returns: + // - w: a writer to receive the object's content. + // - lwh: a func to write the object header. + // - err: any error from the initial writer creation process. + // + // Note that if the object header is not written BEFORE the writer + // is used, this will result in an invalid object. + LazyWriter() (w io.WriteCloser, lwh objectHeaderWriter, err error) +} + func (p *Parser) indexObjects() error { buf := sync.GetBytesBuffer() defer sync.PutBytesBuffer(buf) for i := uint32(0); i < p.count; i++ { - buf.Reset() - oh, err := p.scanner.NextObjectHeader() if err != nil { return err @@ -220,21 +233,60 @@ func (p *Parser) indexObjects() error { ota = newBaseObject(oh.Offset, oh.Length, t) } - buf.Grow(int(oh.Length)) - _, crc, err := p.scanner.NextObject(buf) + hasher := plumbing.NewHasher(oh.Type, oh.Length) + writers := []io.Writer{hasher} + var obj *plumbing.MemoryObject + + // Lazy writing is only available for non-delta objects. + if p.storage != nil && !delta { + // When a storage is set and supports lazy writing, + // use that instead of creating a memory object. + if low, ok := p.storage.(lazyObjectWriter); ok { + ow, lwh, err := low.LazyWriter() + if err != nil { + return err + } + + if err = lwh(oh.Type, oh.Length); err != nil { + return err + } + + defer ow.Close() + writers = append(writers, ow) + } else { + obj = new(plumbing.MemoryObject) + obj.SetSize(oh.Length) + obj.SetType(oh.Type) + + writers = append(writers, obj) + } + } + if delta && !p.scanner.IsSeekable { + buf.Reset() + buf.Grow(int(oh.Length)) + writers = append(writers, buf) + } + + mw := io.MultiWriter(writers...) + + _, crc, err := p.scanner.NextObject(mw) if err != nil { return err } + // Non delta objects needs to be added into the storage. This + // is only required when lazy writing is not supported. + if obj != nil { + if _, err := p.storage.SetEncodedObject(obj); err != nil { + return err + } + } + ota.Crc32 = crc ota.Length = oh.Length - data := buf.Bytes() if !delta { - sha1, err := getSHA1(ota.Type, data) - if err != nil { - return err - } + sha1 := hasher.Sum() // Move children of placeholder parent into actual parent, in case this // was a non-external delta reference. @@ -249,20 +301,8 @@ func (p *Parser) indexObjects() error { p.oiByHash[ota.SHA1] = ota } - if p.storage != nil && !delta { - obj := new(plumbing.MemoryObject) - obj.SetSize(oh.Length) - obj.SetType(oh.Type) - if _, err := obj.Write(data); err != nil { - return err - } - - if _, err := p.storage.SetEncodedObject(obj); err != nil { - return err - } - } - if delta && !p.scanner.IsSeekable { + data := buf.Bytes() p.deltas[oh.Offset] = make([]byte, len(data)) copy(p.deltas[oh.Offset], data) } @@ -280,23 +320,29 @@ func (p *Parser) resolveDeltas() error { for _, obj := range p.oi { buf.Reset() + buf.Grow(int(obj.Length)) err := p.get(obj, buf) if err != nil { return err } - content := buf.Bytes() if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil { return err } - if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil { + if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, nil); err != nil { return err } if !obj.IsDelta() && len(obj.Children) > 0 { + // Dealing with an io.ReaderAt object, means we can + // create it once and reuse across all children. + r := bytes.NewReader(buf.Bytes()) for _, child := range obj.Children { - if err := p.resolveObject(io.Discard, child, content); err != nil { + // Even though we are discarding the output, we still need to read it to + // so that the scanner can advance to the next object, and the SHA1 can be + // calculated. + if err := p.resolveObject(io.Discard, child, r); err != nil { return err } p.resolveExternalRef(child) @@ -361,13 +407,13 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { if o.DiskType.IsDelta() { b := sync.GetBytesBuffer() defer sync.PutBytesBuffer(b) + buf.Grow(int(o.Length)) err := p.get(o.Parent, b) if err != nil { return err } - base := b.Bytes() - err = p.resolveObject(buf, o, base) + err = p.resolveObject(buf, o, bytes.NewReader(b.Bytes())) if err != nil { return err } @@ -378,6 +424,13 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { } } + // If the scanner is seekable, caching this data into + // memory by offset seems wasteful. + // There is a trade-off to be considered here in terms + // of execution time vs memory consumption. + // + // TODO: improve seekable execution time, so that we can + // skip this cache. if len(o.Children) > 0 { data := make([]byte, buf.Len()) copy(data, buf.Bytes()) @@ -386,10 +439,25 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { return nil } +// resolveObject resolves an object from base, using information +// provided by o. +// +// This call has the side-effect of changing field values +// from the object info o: +// - Type: OFSDeltaObject may become the target type (e.g. Blob). +// - Size: The size may be update with the target size. +// - Hash: Zero hashes will be calculated as part of the object +// resolution. Hence why this process can't be avoided even when w +// is an io.Discard. +// +// base must be an io.ReaderAt, which is a requirement from +// patchDeltaStream. The main reason being that reversing an +// delta object may lead to going backs and forths within base, +// which is not supported by io.Reader. func (p *Parser) resolveObject( w io.Writer, o *objectInfo, - base []byte, + base io.ReaderAt, ) error { if !o.DiskType.IsDelta() { return nil @@ -400,26 +468,46 @@ func (p *Parser) resolveObject( if err != nil { return err } - data := buf.Bytes() - data, err = applyPatchBase(o, data, base) + writers := []io.Writer{w} + var obj *plumbing.MemoryObject + var lwh objectHeaderWriter + + if p.storage != nil { + if low, ok := p.storage.(lazyObjectWriter); ok { + ow, wh, err := low.LazyWriter() + if err != nil { + return err + } + lwh = wh + + defer ow.Close() + writers = append(writers, ow) + } else { + obj = new(plumbing.MemoryObject) + ow, err := obj.Writer() + if err != nil { + return err + } + + writers = append(writers, ow) + } + } + + mw := io.MultiWriter(writers...) + + err = applyPatchBase(o, base, buf, mw, lwh) if err != nil { return err } - if p.storage != nil { - obj := new(plumbing.MemoryObject) - obj.SetSize(o.Size()) + if obj != nil { obj.SetType(o.Type) - if _, err := obj.Write(data); err != nil { - return err - } - + obj.SetSize(o.Size()) // Size here is correct as it was populated by applyPatchBase. if _, err := p.storage.SetEncodedObject(obj); err != nil { return err } } - _, err = w.Write(data) return err } @@ -443,24 +531,31 @@ func (p *Parser) readData(w io.Writer, o *objectInfo) error { return nil } -func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { - patched, err := PatchDelta(base, data) - if err != nil { - return nil, err +// applyPatchBase applies the patch to target. +// +// Note that ota will be updated based on the description in resolveObject. +func applyPatchBase(ota *objectInfo, base io.ReaderAt, delta io.Reader, target io.Writer, wh objectHeaderWriter) error { + if target == nil { + return fmt.Errorf("cannot apply patch against nil target") } + typ := ota.Type if ota.SHA1 == plumbing.ZeroHash { - ota.Type = ota.Parent.Type - sha1, err := getSHA1(ota.Type, patched) - if err != nil { - return nil, err - } + typ = ota.Parent.Type + } + + sz, h, err := patchDeltaWriter(target, base, delta, typ, wh) + if err != nil { + return err + } - ota.SHA1 = sha1 - ota.Length = int64(len(patched)) + if ota.SHA1 == plumbing.ZeroHash { + ota.Type = typ + ota.Length = int64(sz) + ota.SHA1 = h } - return patched, nil + return nil } func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) { diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index f00562d63..67c20ff78 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "errors" + "fmt" "io" "math" @@ -265,6 +266,107 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error { return nil } +func patchDeltaWriter(dst io.Writer, base io.ReaderAt, delta io.Reader, + typ plumbing.ObjectType, writeHeader objectHeaderWriter) (uint, plumbing.Hash, error) { + deltaBuf := bufio.NewReaderSize(delta, 1024) + srcSz, err := decodeLEB128ByteReader(deltaBuf) + if err != nil { + if err == io.EOF { + return 0, plumbing.ZeroHash, ErrInvalidDelta + } + return 0, plumbing.ZeroHash, err + } + + if r, ok := base.(*bytes.Reader); ok && srcSz != uint(r.Size()) { + return 0, plumbing.ZeroHash, ErrInvalidDelta + } + + targetSz, err := decodeLEB128ByteReader(deltaBuf) + if err != nil { + if err == io.EOF { + return 0, plumbing.ZeroHash, ErrInvalidDelta + } + return 0, plumbing.ZeroHash, err + } + + // If header still needs to be written, caller will provide + // a LazyObjectWriterHeader. This seems to be the case when + // dealing with thin-packs. + if writeHeader != nil { + err = writeHeader(typ, int64(targetSz)) + if err != nil { + return 0, plumbing.ZeroHash, fmt.Errorf("could not lazy write header: %w", err) + } + } + + remainingTargetSz := targetSz + + hasher := plumbing.NewHasher(typ, int64(targetSz)) + mw := io.MultiWriter(dst, hasher) + + bufp := sync.GetByteSlice() + defer sync.PutByteSlice(bufp) + + sr := io.NewSectionReader(base, int64(0), int64(srcSz)) + // Keep both the io.LimitedReader types, so we can reset N. + baselr := io.LimitReader(sr, 0).(*io.LimitedReader) + deltalr := io.LimitReader(deltaBuf, 0).(*io.LimitedReader) + + for { + buf := *bufp + cmd, err := deltaBuf.ReadByte() + if err == io.EOF { + return 0, plumbing.ZeroHash, ErrInvalidDelta + } + if err != nil { + return 0, plumbing.ZeroHash, err + } + + if isCopyFromSrc(cmd) { + offset, err := decodeOffsetByteReader(cmd, deltaBuf) + if err != nil { + return 0, plumbing.ZeroHash, err + } + sz, err := decodeSizeByteReader(cmd, deltaBuf) + if err != nil { + return 0, plumbing.ZeroHash, err + } + + if invalidSize(sz, targetSz) || + invalidOffsetSize(offset, sz, srcSz) { + return 0, plumbing.ZeroHash, err + } + + if _, err := sr.Seek(int64(offset), io.SeekStart); err != nil { + return 0, plumbing.ZeroHash, err + } + baselr.N = int64(sz) + if _, err := io.CopyBuffer(mw, baselr, buf); err != nil { + return 0, plumbing.ZeroHash, err + } + remainingTargetSz -= sz + } else if isCopyFromDelta(cmd) { + sz := uint(cmd) // cmd is the size itself + if invalidSize(sz, targetSz) { + return 0, plumbing.ZeroHash, ErrInvalidDelta + } + deltalr.N = int64(sz) + if _, err := io.CopyBuffer(mw, deltalr, buf); err != nil { + return 0, plumbing.ZeroHash, err + } + + remainingTargetSz -= sz + } else { + return 0, plumbing.ZeroHash, err + } + if remainingTargetSz <= 0 { + break + } + } + + return targetSz, hasher.Sum(), nil +} + // Decodes a number encoded as an unsigned LEB128 at the start of some // binary data and returns the decoded number and the rest of the // stream. diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 846a7b860..e812fe934 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -146,6 +146,19 @@ func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (h plumbing.H return o.Hash(), err } +// LazyWriter returns a lazy ObjectWriter that is bound to a DotGit file. +// It first write the header passing on the object type and size, so +// that the object contents can be written later, without the need to +// create a MemoryObject and buffering its entire contents into memory. +func (s *ObjectStorage) LazyWriter() (w io.WriteCloser, wh func(typ plumbing.ObjectType, sz int64) error, err error) { + ow, err := s.dir.NewObject() + if err != nil { + return nil, nil, err + } + + return ow, ow.WriteHeader, nil +} + // HasEncodedObject returns nil if the object exists, without actually // reading the object data from storage. func (s *ObjectStorage) HasEncodedObject(h plumbing.Hash) (err error) { From 501e9ad99aa3d18b7711d8cea94e0e9b68883b6b Mon Sep 17 00:00:00 2001 From: Anand Francis Joseph Date: Thu, 2 Nov 2023 22:48:52 +0530 Subject: [PATCH 28/56] plumbing: transport/ssh, Fix nil pointer dereference caused when an unreachable proxy server is set. Fixes #900 Signed-off-by: Anand Francis Joseph --- plumbing/transport/ssh/common.go | 10 +++++----- plumbing/transport/ssh/common_test.go | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/plumbing/transport/ssh/common.go b/plumbing/transport/ssh/common.go index 15316038b..46fda73fa 100644 --- a/plumbing/transport/ssh/common.go +++ b/plumbing/transport/ssh/common.go @@ -168,7 +168,7 @@ func dial(network, addr string, proxyOpts transport.ProxyOptions, config *ssh.Cl defer cancel() var conn net.Conn - var err error + var dialErr error if proxyOpts.URL != "" { proxyUrl, err := proxyOpts.FullURL() @@ -186,12 +186,12 @@ func dial(network, addr string, proxyOpts transport.ProxyOptions, config *ssh.Cl return nil, fmt.Errorf("expected ssh proxy dialer to be of type %s; got %s", reflect.TypeOf(ctxDialer), reflect.TypeOf(dialer)) } - conn, err = ctxDialer.DialContext(ctx, "tcp", addr) + conn, dialErr = ctxDialer.DialContext(ctx, "tcp", addr) } else { - conn, err = proxy.Dial(ctx, network, addr) + conn, dialErr = proxy.Dial(ctx, network, addr) } - if err != nil { - return nil, err + if dialErr != nil { + return nil, dialErr } c, chans, reqs, err := ssh.NewClientConn(conn, addr, config) diff --git a/plumbing/transport/ssh/common_test.go b/plumbing/transport/ssh/common_test.go index 496e82d17..4cc2a0693 100644 --- a/plumbing/transport/ssh/common_test.go +++ b/plumbing/transport/ssh/common_test.go @@ -172,6 +172,28 @@ func (s *SuiteCommon) TestIssue70(c *C) { c.Assert(err, IsNil) } +/* +Given, an endpoint to a git server with a socks5 proxy URL, +When, the socks5 proxy server is not reachable, +Then, there should not be any panic and an error with appropriate message should be returned. +Related issue : https://github.com/go-git/go-git/pull/900 +*/ +func (s *SuiteCommon) TestInvalidSocks5Proxy(c *C) { + ep, err := transport.NewEndpoint("git@github.com:foo/bar.git") + c.Assert(err, IsNil) + ep.Proxy.URL = "socks5://127.0.0.1:1080" + + auth, err := NewPublicKeys("foo", testdata.PEMBytes["rsa"], "") + c.Assert(err, IsNil) + c.Assert(auth, NotNil) + + ps, err := DefaultClient.NewUploadPackSession(ep, auth) + //Since the proxy server is not running, we expect an error. + c.Assert(ps, IsNil) + c.Assert(err, NotNil) + c.Assert(err, ErrorMatches, "socks connect .* dial tcp 127.0.0.1:1080: .*") +} + type mockSSHConfig struct { Values map[string]map[string]string } From ce0b76e7674d683db547103bc773305129a0ded4 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Sun, 29 Oct 2023 16:35:09 -0400 Subject: [PATCH 29/56] git: implement upload-server-info. Fixes #731 This adds UpdateServerInfo along with a new go-git command to generate info files to help git dumb http serve refs and their objects. This also updates the docs to reflect this. Docs: https://git-scm.com/docs/git-update-server-info Fixes: https://github.com/go-git/go-git/issues/731 --- COMPATIBILITY.md | 8 +- cli/go-git/main.go | 1 + cli/go-git/update_server_info.go | 34 +++++ internal/reference/sort.go | 14 ++ plumbing/serverinfo/serverinfo.go | 94 +++++++++++++ plumbing/serverinfo/serverinfo_test.go | 185 +++++++++++++++++++++++++ 6 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 cli/go-git/update_server_info.go create mode 100644 internal/reference/sort.go create mode 100644 plumbing/serverinfo/serverinfo.go create mode 100644 plumbing/serverinfo/serverinfo_test.go diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index bbffea522..c1f280d4d 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -109,10 +109,10 @@ compatibility status with go-git. ## Server admin -| Feature | Sub-feature | Status | Notes | Examples | -| -------------------- | ----------- | ------ | ----- | -------- | -| `daemon` | | ❌ | | | -| `update-server-info` | | ❌ | | | +| Feature | Sub-feature | Status | Notes | Examples | +| -------------------- | ----------- | ------ | ----- | ----------------------------------------- | +| `daemon` | | ❌ | | | +| `update-server-info` | | ✅ | | [cli](./cli/go-git/update_server_info.go) | ## Advanced diff --git a/cli/go-git/main.go b/cli/go-git/main.go index 97b8c3e2a..0a5ad2c26 100644 --- a/cli/go-git/main.go +++ b/cli/go-git/main.go @@ -22,6 +22,7 @@ func main() { } parser := flags.NewNamedParser(bin, flags.Default) + parser.AddCommand("update-server-info", "", "", &CmdUpdateServerInfo{}) parser.AddCommand("receive-pack", "", "", &CmdReceivePack{}) parser.AddCommand("upload-pack", "", "", &CmdUploadPack{}) parser.AddCommand("version", "Show the version information.", "", &CmdVersion{}) diff --git a/cli/go-git/update_server_info.go b/cli/go-git/update_server_info.go new file mode 100644 index 000000000..a7f3e3e39 --- /dev/null +++ b/cli/go-git/update_server_info.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "os" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/serverinfo" + "github.com/go-git/go-git/v5/storage/filesystem" +) + +// CmdUpdateServerInfo command updates the server info files in the repository. +// This is used by git http transport (dumb) to generate a list of available +// refs for the repository. See: +// https://git-scm.com/docs/git-update-server-info +type CmdUpdateServerInfo struct { + cmd +} + +// Usage returns the usage of the command. +func (CmdUpdateServerInfo) Usage() string { + return fmt.Sprintf("within a git repository run: %s", os.Args[0]) +} + +// Execute runs the command. +func (c *CmdUpdateServerInfo) Execute(args []string) error { + r, err := git.PlainOpen(".") + if err != nil { + return err + } + + fs := r.Storer.(*filesystem.Storage).Filesystem() + return serverinfo.UpdateServerInfo(r.Storer, fs) +} diff --git a/internal/reference/sort.go b/internal/reference/sort.go new file mode 100644 index 000000000..726edbdd3 --- /dev/null +++ b/internal/reference/sort.go @@ -0,0 +1,14 @@ +package reference + +import ( + "sort" + + "github.com/go-git/go-git/v5/plumbing" +) + +// Sort sorts the references by name to ensure a consistent order. +func Sort(refs []*plumbing.Reference) { + sort.Slice(refs, func(i, j int) bool { + return refs[i].Name() < refs[j].Name() + }) +} diff --git a/plumbing/serverinfo/serverinfo.go b/plumbing/serverinfo/serverinfo.go new file mode 100644 index 000000000..d7ea7ef06 --- /dev/null +++ b/plumbing/serverinfo/serverinfo.go @@ -0,0 +1,94 @@ +package serverinfo + +import ( + "fmt" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/internal/reference" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/storage" +) + +// UpdateServerInfo updates the server info files in the repository. +// +// It generates a list of available refs for the repository. +// Used by git http transport (dumb), for more information refer to: +// https://git-scm.com/book/id/v2/Git-Internals-Transfer-Protocols#_the_dumb_protocol +func UpdateServerInfo(s storage.Storer, fs billy.Filesystem) error { + pos, ok := s.(storer.PackedObjectStorer) + if !ok { + return git.ErrPackedObjectsNotSupported + } + + infoRefs, err := fs.Create("info/refs") + if err != nil { + return err + } + + defer infoRefs.Close() + + refsIter, err := s.IterReferences() + if err != nil { + return err + } + + defer refsIter.Close() + + var refs []*plumbing.Reference + if err := refsIter.ForEach(func(ref *plumbing.Reference) error { + refs = append(refs, ref) + return nil + }); err != nil { + return err + } + + reference.Sort(refs) + for _, ref := range refs { + name := ref.Name() + hash := ref.Hash() + switch ref.Type() { + case plumbing.SymbolicReference: + if name == plumbing.HEAD { + continue + } + ref, err := s.Reference(ref.Target()) + if err != nil { + return err + } + + hash = ref.Hash() + fallthrough + case plumbing.HashReference: + fmt.Fprintf(infoRefs, "%s\t%s\n", hash, name) + if name.IsTag() { + tag, err := object.GetTag(s, hash) + if err == nil { + fmt.Fprintf(infoRefs, "%s\t%s^{}\n", tag.Target, name) + } + } + } + } + + infoPacks, err := fs.Create("objects/info/packs") + if err != nil { + return err + } + + defer infoPacks.Close() + + packs, err := pos.ObjectPacks() + if err != nil { + return err + } + + for _, p := range packs { + fmt.Fprintf(infoPacks, "P pack-%s.pack\n", p) + } + + fmt.Fprintln(infoPacks) + + return nil +} diff --git a/plumbing/serverinfo/serverinfo_test.go b/plumbing/serverinfo/serverinfo_test.go new file mode 100644 index 000000000..0a52ea2f9 --- /dev/null +++ b/plumbing/serverinfo/serverinfo_test.go @@ -0,0 +1,185 @@ +package serverinfo + +import ( + "io" + "strings" + "testing" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + fixtures "github.com/go-git/go-git-fixtures/v4" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/storage" + "github.com/go-git/go-git/v5/storage/memory" + . "gopkg.in/check.v1" +) + +type ServerInfoSuite struct{} + +var _ = Suite(&ServerInfoSuite{}) + +func Test(t *testing.T) { TestingT(t) } + +func (s *ServerInfoSuite) TestUpdateServerInfoInit(c *C) { + fs := memfs.New() + st := memory.NewStorage() + r, err := git.Init(st, fs) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + err = UpdateServerInfo(st, fs) + c.Assert(err, IsNil) +} + +func assertInfoRefs(c *C, st storage.Storer, fs billy.Filesystem) { + refsFile, err := fs.Open("info/refs") + c.Assert(err, IsNil) + + defer refsFile.Close() + bts, err := io.ReadAll(refsFile) + c.Assert(err, IsNil) + + localRefs := make(map[plumbing.ReferenceName]plumbing.Hash) + for _, line := range strings.Split(string(bts), "\n") { + if line == "" { + continue + } + parts := strings.Split(line, "\t") + c.Assert(parts, HasLen, 2) + hash := plumbing.NewHash(parts[0]) + name := plumbing.ReferenceName(parts[1]) + localRefs[name] = hash + } + + refs, err := st.IterReferences() + c.Assert(err, IsNil) + + err = refs.ForEach(func(ref *plumbing.Reference) error { + name := ref.Name() + hash := ref.Hash() + switch ref.Type() { + case plumbing.SymbolicReference: + if name == plumbing.HEAD { + return nil + } + ref, err := st.Reference(ref.Target()) + c.Assert(err, IsNil) + hash = ref.Hash() + fallthrough + case plumbing.HashReference: + h, ok := localRefs[name] + c.Assert(ok, Equals, true) + c.Assert(h, Equals, hash) + if name.IsTag() { + tag, err := object.GetTag(st, hash) + if err == nil { + t, ok := localRefs[name+"^{}"] + c.Assert(ok, Equals, true) + c.Assert(t, Equals, tag.Target) + } + } + } + return nil + }) + + c.Assert(err, IsNil) +} + +func assertObjectPacks(c *C, st storage.Storer, fs billy.Filesystem) { + infoPacks, err := fs.Open("objects/info/packs") + c.Assert(err, IsNil) + + defer infoPacks.Close() + bts, err := io.ReadAll(infoPacks) + c.Assert(err, IsNil) + + pos, ok := st.(storer.PackedObjectStorer) + c.Assert(ok, Equals, true) + localPacks := make(map[string]struct{}) + packs, err := pos.ObjectPacks() + c.Assert(err, IsNil) + + for _, line := range strings.Split(string(bts), "\n") { + if line == "" { + continue + } + parts := strings.Split(line, " ") + c.Assert(parts, HasLen, 2) + pack := strings.TrimPrefix(parts[1], "pack-") + pack = strings.TrimSuffix(pack, ".pack") + localPacks[pack] = struct{}{} + } + + for _, p := range packs { + _, ok := localPacks[p.String()] + c.Assert(ok, Equals, true) + } +} + +func (s *ServerInfoSuite) TestUpdateServerInfoTags(c *C) { + fs := memfs.New() + st := memory.NewStorage() + r, err := git.Clone(st, fs, &git.CloneOptions{ + URL: fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().URL, + }) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + err = UpdateServerInfo(st, fs) + c.Assert(err, IsNil) + + assertInfoRefs(c, st, fs) + assertObjectPacks(c, st, fs) +} + +func (s *ServerInfoSuite) TestUpdateServerInfoBasic(c *C) { + fs := memfs.New() + st := memory.NewStorage() + r, err := git.Clone(st, fs, &git.CloneOptions{ + URL: fixtures.Basic().One().URL, + }) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + err = UpdateServerInfo(st, fs) + c.Assert(err, IsNil) + + assertInfoRefs(c, st, fs) + assertObjectPacks(c, st, fs) +} + +func (s *ServerInfoSuite) TestUpdateServerInfoBasicChange(c *C) { + fs := memfs.New() + st := memory.NewStorage() + r, err := git.Clone(st, fs, &git.CloneOptions{ + URL: fixtures.Basic().One().URL, + }) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + err = UpdateServerInfo(st, fs) + c.Assert(err, IsNil) + + assertInfoRefs(c, st, fs) + assertObjectPacks(c, st, fs) + + head, err := r.Head() + c.Assert(err, IsNil) + + ref := plumbing.NewHashReference("refs/heads/my-branch", head.Hash()) + err = r.Storer.SetReference(ref) + c.Assert(err, IsNil) + + _, err = r.CreateTag("test-tag", head.Hash(), &git.CreateTagOptions{ + Message: "test-tag", + }) + c.Assert(err, IsNil) + + err = UpdateServerInfo(st, fs) + + assertInfoRefs(c, st, fs) + assertObjectPacks(c, st, fs) +} From 71db7fa9023ed5e0b9b8a6b6cac81f9d3c787972 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:21:52 +0000 Subject: [PATCH 30/56] build: bump golang.org/x/sys from 0.13.0 to 0.14.0 Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.13.0 to 0.14.0. - [Commits](https://github.com/golang/sys/compare/v0.13.0...v0.14.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 25be1e1d4..383a7cb84 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 - golang.org/x/sys v0.13.0 + golang.org/x/sys v0.14.0 golang.org/x/text v0.13.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) diff --git a/go.sum b/go.sum index 9a2760a1c..010c325e5 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 3c3e49e73a2531e2cf89e314d1766fb7f8e0fee4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:19:42 +0000 Subject: [PATCH 31/56] build: bump golang.org/x/text from 0.13.0 to 0.14.0 Bumps [golang.org/x/text](https://github.com/golang/text) from 0.13.0 to 0.14.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.13.0...v0.14.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 383a7cb84..9be458cf4 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 golang.org/x/sys v0.14.0 - golang.org/x/text v0.13.0 + golang.org/x/text v0.14.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) diff --git a/go.sum b/go.sum index 010c325e5..b0817f8b9 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From f7cd844ced42685ed1e6676b0598f064ea5a5bc8 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Mon, 6 Nov 2023 21:31:10 +0000 Subject: [PATCH 32/56] plumbing: format/packfile, Refactor patch delta The changes aim to simplify the code and reduce duplication. Signed-off-by: Paulo Gomes --- plumbing/format/packfile/diff_delta.go | 5 +- plumbing/format/packfile/patch_delta.go | 187 ++++++++++-------------- 2 files changed, 79 insertions(+), 113 deletions(-) diff --git a/plumbing/format/packfile/diff_delta.go b/plumbing/format/packfile/diff_delta.go index 2c7a33581..8898e5830 100644 --- a/plumbing/format/packfile/diff_delta.go +++ b/plumbing/format/packfile/diff_delta.go @@ -17,8 +17,11 @@ const ( s = 16 // https://github.com/git/git/blob/f7466e94375b3be27f229c78873f0acf8301c0a5/diff-delta.c#L428 - // Max size of a copy operation (64KB) + // Max size of a copy operation (64KB). maxCopySize = 64 * 1024 + + // Min size of a copy operation. + minCopySize = 4 ) // GetDelta returns an EncodedObject of type OFSDeltaObject. Base and Target object, diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 67c20ff78..960769c7c 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -18,7 +18,33 @@ import ( // and https://github.com/tarruda/node-git-core/blob/master/src/js/delta.js // for details about the delta format. -const deltaSizeMin = 4 +var ( + ErrInvalidDelta = errors.New("invalid delta") + ErrDeltaCmd = errors.New("wrong delta command") +) + +const ( + payload = 0x7f // 0111 1111 + continuation = 0x80 // 1000 0000 +) + +type offset struct { + mask byte + shift uint +} + +var offsets = []offset{ + {mask: 0x01, shift: 0}, + {mask: 0x02, shift: 8}, + {mask: 0x04, shift: 16}, + {mask: 0x08, shift: 24}, +} + +var sizes = []offset{ + {mask: 0x10, shift: 0}, + {mask: 0x20, shift: 8}, + {mask: 0x40, shift: 16}, +} // ApplyDelta writes to target the result of applying the modification deltas in delta to base. func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { @@ -59,11 +85,6 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { return err } -var ( - ErrInvalidDelta = errors.New("invalid delta") - ErrDeltaCmd = errors.New("wrong delta command") -) - // PatchDelta returns the result of applying the modification deltas in delta to src. // An error will be returned if delta is corrupted (ErrDeltaLen) or an action command // is not copy from source or copy from delta (ErrDeltaCmd). @@ -121,7 +142,8 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo return } - if isCopyFromSrc(cmd) { + switch { + case isCopyFromSrc(cmd): offset, err := decodeOffsetByteReader(cmd, deltaBuf) if err != nil { _ = dstWr.CloseWithError(err) @@ -174,7 +196,8 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo } remainingTargetSz -= sz basePos += sz - } else if isCopyFromDelta(cmd) { + + case isCopyFromDelta(cmd): sz := uint(cmd) // cmd is the size itself if invalidSize(sz, targetSz) { _ = dstWr.CloseWithError(ErrInvalidDelta) @@ -186,10 +209,12 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo } remainingTargetSz -= sz - } else { + + default: _ = dstWr.CloseWithError(ErrDeltaCmd) return } + if remainingTargetSz <= 0 { _ = dstWr.Close() return @@ -201,7 +226,7 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo } func patchDelta(dst *bytes.Buffer, src, delta []byte) error { - if len(delta) < deltaSizeMin { + if len(delta) < minCopySize { return ErrInvalidDelta } @@ -222,7 +247,9 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error { cmd = delta[0] delta = delta[1:] - if isCopyFromSrc(cmd) { + + switch { + case isCopyFromSrc(cmd): var offset, sz uint var err error offset, delta, err = decodeOffset(cmd, delta) @@ -241,7 +268,8 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error { } dst.Write(src[offset : offset+sz]) remainingTargetSz -= sz - } else if isCopyFromDelta(cmd) { + + case isCopyFromDelta(cmd): sz := uint(cmd) // cmd is the size itself if invalidSize(sz, targetSz) { return ErrInvalidDelta @@ -254,7 +282,8 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error { dst.Write(delta[0:sz]) remainingTargetSz -= sz delta = delta[sz:] - } else { + + default: return ErrDeltaCmd } @@ -408,48 +437,24 @@ func decodeLEB128ByteReader(input io.ByteReader) (uint, error) { return num, nil } -const ( - payload = 0x7f // 0111 1111 - continuation = 0x80 // 1000 0000 -) - func isCopyFromSrc(cmd byte) bool { - return (cmd & 0x80) != 0 + return (cmd & continuation) != 0 } func isCopyFromDelta(cmd byte) bool { - return (cmd&0x80) == 0 && cmd != 0 + return (cmd&continuation) == 0 && cmd != 0 } func decodeOffsetByteReader(cmd byte, delta io.ByteReader) (uint, error) { var offset uint - if (cmd & 0x01) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err - } - offset = uint(next) - } - if (cmd & 0x02) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err - } - offset |= uint(next) << 8 - } - if (cmd & 0x04) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err - } - offset |= uint(next) << 16 - } - if (cmd & 0x08) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err + for _, o := range offsets { + if (cmd & o.mask) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + offset |= uint(next) << o.shift } - offset |= uint(next) << 24 } return offset, nil @@ -457,33 +462,14 @@ func decodeOffsetByteReader(cmd byte, delta io.ByteReader) (uint, error) { func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) { var offset uint - if (cmd & 0x01) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta - } - offset = uint(delta[0]) - delta = delta[1:] - } - if (cmd & 0x02) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta - } - offset |= uint(delta[0]) << 8 - delta = delta[1:] - } - if (cmd & 0x04) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta - } - offset |= uint(delta[0]) << 16 - delta = delta[1:] - } - if (cmd & 0x08) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta + for _, o := range offsets { + if (cmd & o.mask) != 0 { + if len(delta) == 0 { + return 0, nil, ErrInvalidDelta + } + offset |= uint(delta[0]) << o.shift + delta = delta[1:] } - offset |= uint(delta[0]) << 24 - delta = delta[1:] } return offset, delta, nil @@ -491,29 +477,18 @@ func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) { func decodeSizeByteReader(cmd byte, delta io.ByteReader) (uint, error) { var sz uint - if (cmd & 0x10) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err - } - sz = uint(next) - } - if (cmd & 0x20) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err - } - sz |= uint(next) << 8 - } - if (cmd & 0x40) != 0 { - next, err := delta.ReadByte() - if err != nil { - return 0, err + for _, s := range sizes { + if (cmd & s.mask) != 0 { + next, err := delta.ReadByte() + if err != nil { + return 0, err + } + sz |= uint(next) << s.shift } - sz |= uint(next) << 16 } + if sz == 0 { - sz = 0x10000 + sz = maxCopySize } return sz, nil @@ -521,29 +496,17 @@ func decodeSizeByteReader(cmd byte, delta io.ByteReader) (uint, error) { func decodeSize(cmd byte, delta []byte) (uint, []byte, error) { var sz uint - if (cmd & 0x10) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta - } - sz = uint(delta[0]) - delta = delta[1:] - } - if (cmd & 0x20) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta - } - sz |= uint(delta[0]) << 8 - delta = delta[1:] - } - if (cmd & 0x40) != 0 { - if len(delta) == 0 { - return 0, nil, ErrInvalidDelta + for _, s := range sizes { + if (cmd & s.mask) != 0 { + if len(delta) == 0 { + return 0, nil, ErrInvalidDelta + } + sz |= uint(delta[0]) << s.shift + delta = delta[1:] } - sz |= uint(delta[0]) << 16 - delta = delta[1:] } if sz == 0 { - sz = 0x10000 + sz = maxCopySize } return sz, delta, nil From 242eac6741c4bf82c8e53d69ac0ae2ebf9c789d8 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 7 Nov 2023 12:07:14 -0500 Subject: [PATCH 33/56] cli: separate go module for cli --- cli/go-git/go.mod | 33 +++++++++++ cli/go-git/go.sum | 137 ++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 - go.sum | 3 - 4 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 cli/go-git/go.mod create mode 100644 cli/go-git/go.sum diff --git a/cli/go-git/go.mod b/cli/go-git/go.mod new file mode 100644 index 000000000..bde9014cc --- /dev/null +++ b/cli/go-git/go.mod @@ -0,0 +1,33 @@ +module github.com/go-git/v5/go-git/cli/go-git + +go 1.19 + +require ( + github.com/go-git/go-git/v5 v5.10.1-0.20231107163107-e54a6ae399e9 + github.com/jessevdk/go-flags v1.5.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/skeema/knownhosts v1.2.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/tools v0.13.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect +) diff --git a/cli/go-git/go.sum b/cli/go-git/go.sum new file mode 100644 index 000000000..00a9140dc --- /dev/null +++ b/cli/go-git/go.sum @@ -0,0 +1,137 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= +github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git/v5 v5.10.1-0.20231107163107-e54a6ae399e9 h1:uNsBlaMgh9sjbk+uvOJSJm+ixmoJoXbeDFNBCgYqUXo= +github.com/go-git/go-git/v5 v5.10.1-0.20231107163107-e54a6ae399e9/go.mod h1:vQMZfh4ro7khF6K55zuBL6sU45QYjjppZ6Cot47a9t4= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= +github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.mod b/go.mod index 9be458cf4..51fc3e834 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/go-cmp v0.6.0 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 - github.com/jessevdk/go-flags v1.5.0 github.com/kevinburke/ssh_config v1.2.0 github.com/pjbgf/sha1cd v0.3.0 github.com/sergi/go-diff v1.1.0 diff --git a/go.sum b/go.sum index b0817f8b9..a6f00e58a 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -108,7 +106,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From e40360644f4a6e368e916db5a930c17c47f64a78 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 7 Nov 2023 12:11:03 -0500 Subject: [PATCH 34/56] build: add cli/go-git to dependabot --- .github/dependabot.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 6a7a66bfa..403f428e4 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -13,3 +13,10 @@ updates: interval: "daily" commit-message: prefix: "build" + + - package-ecosystem: "gomod" + directory: "/cli/go-git" + schedule: + interval: "daily" + commit-message: + prefix: "build" From fbc6d7f5d37846133ca92dabf2e2115fcee06e19 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 7 Nov 2023 12:42:10 -0500 Subject: [PATCH 35/56] build: fix go-git binary build --- plumbing/transport/file/common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plumbing/transport/file/common_test.go b/plumbing/transport/file/common_test.go index 7e033a80b..a217e9716 100644 --- a/plumbing/transport/file/common_test.go +++ b/plumbing/transport/file/common_test.go @@ -29,8 +29,8 @@ func (s *CommonSuite) SetUpSuite(c *C) { s.ReceivePackBin = filepath.Join(s.tmpDir, "git-receive-pack") s.UploadPackBin = filepath.Join(s.tmpDir, "git-upload-pack") bin := filepath.Join(s.tmpDir, "go-git") - cmd := exec.Command("go", "build", "-o", bin, - "../../../cli/go-git/...") + cmd := exec.Command("go", "build", "-o", bin) + cmd.Dir = "../../../cli/go-git" c.Assert(cmd.Run(), IsNil) c.Assert(os.Symlink(bin, s.ReceivePackBin), IsNil) c.Assert(os.Symlink(bin, s.UploadPackBin), IsNil) From a0bdfd93726140d0bccc403584f8e09aacd915ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:07:27 +0000 Subject: [PATCH 36/56] build: bump golang.org/x/crypto from 0.14.0 to 0.15.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.15.0. - [Commits](https://github.com/golang/crypto/compare/v0.14.0...v0.15.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 51fc3e834..203dd58a0 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/sergi/go-diff v1.1.0 github.com/skeema/knownhosts v1.2.1 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.15.0 golang.org/x/net v0.17.0 golang.org/x/sys v0.14.0 golang.org/x/text v0.14.0 diff --git a/go.sum b/go.sum index a6f00e58a..6b1bc44f7 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= @@ -125,7 +125,7 @@ golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From f1485ab725f8ecbaf61ca252d9237b1132206275 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:10:21 +0000 Subject: [PATCH 37/56] build: bump golang.org/x/net from 0.17.0 to 0.18.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.17.0 to 0.18.0. - [Commits](https://github.com/golang/net/compare/v0.17.0...v0.18.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 203dd58a0..c7bfb1437 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/skeema/knownhosts v1.2.1 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.15.0 - golang.org/x/net v0.17.0 + golang.org/x/net v0.18.0 golang.org/x/sys v0.14.0 golang.org/x/text v0.14.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 6b1bc44f7..4b75ae7b8 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,8 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From a28a4ac965f714e7506ba26212556fd3d9817712 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Mon, 13 Nov 2023 17:53:46 -0500 Subject: [PATCH 38/56] utils: remove ioutil.Pipe and use std library io.Pipe ioutil.Pipe literally calls io.Pipe. --- go.mod | 1 - go.sum | 4 ---- plumbing/transport/file/client.go | 3 +-- plumbing/transport/server/server.go | 2 +- remote.go | 2 +- utils/ioutil/common.go | 12 +----------- utils/ioutil/pipe.go | 9 --------- utils/ioutil/pipe_js.go | 9 --------- 8 files changed, 4 insertions(+), 38 deletions(-) delete mode 100644 utils/ioutil/pipe.go delete mode 100644 utils/ioutil/pipe_js.go diff --git a/go.mod b/go.mod index c7bfb1437..18b60da9e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ go 1.19 require ( dario.cat/mergo v1.0.0 github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 - github.com/acomagu/bufpipe v1.0.4 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a github.com/emirpasic/gods v1.18.1 diff --git a/go.sum b/go.sum index 4b75ae7b8..f5af02373 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -50,8 +48,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= diff --git a/plumbing/transport/file/client.go b/plumbing/transport/file/client.go index 6f0a38012..38714e2ad 100644 --- a/plumbing/transport/file/client.go +++ b/plumbing/transport/file/client.go @@ -11,7 +11,6 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/internal/common" - "github.com/go-git/go-git/v5/utils/ioutil" "golang.org/x/sys/execabs" ) @@ -112,7 +111,7 @@ func (c *command) Start() error { func (c *command) StderrPipe() (io.Reader, error) { // Pipe returned by Command.StderrPipe has a race with Read + Command.Wait. // We use an io.Pipe and close it after the command finishes. - r, w := ioutil.Pipe() + r, w := io.Pipe() c.cmd.Stderr = w c.stderrCloser = r return r, nil diff --git a/plumbing/transport/server/server.go b/plumbing/transport/server/server.go index 11fa0c801..cf5d6f43f 100644 --- a/plumbing/transport/server/server.go +++ b/plumbing/transport/server/server.go @@ -166,7 +166,7 @@ func (s *upSession) UploadPack(ctx context.Context, req *packp.UploadPackRequest return nil, err } - pr, pw := ioutil.Pipe() + pr, pw := io.Pipe() e := packfile.NewEncoder(pw, s.storer, false) go func() { // TODO: plumb through a pack window. diff --git a/remote.go b/remote.go index bbe275d5b..2b4122e96 100644 --- a/remote.go +++ b/remote.go @@ -1387,7 +1387,7 @@ func pushHashes( allDelete bool, ) (*packp.ReportStatus, error) { - rd, wr := ioutil.Pipe() + rd, wr := io.Pipe() config, err := s.Config() if err != nil { diff --git a/utils/ioutil/common.go b/utils/ioutil/common.go index b0ace4e62..235af717b 100644 --- a/utils/ioutil/common.go +++ b/utils/ioutil/common.go @@ -195,7 +195,7 @@ func NewWriterOnError(w io.Writer, notify func(error)) io.Writer { } // NewWriteCloserOnError returns a io.WriteCloser that call the notify function -//when an unexpected (!io.EOF) error happens, after call Write function. +// when an unexpected (!io.EOF) error happens, after call Write function. func NewWriteCloserOnError(w io.WriteCloser, notify func(error)) io.WriteCloser { return NewWriteCloser(NewWriterOnError(w, notify), w) } @@ -208,13 +208,3 @@ func (r *writerOnError) Write(p []byte) (n int, err error) { return } - -type PipeReader interface { - io.ReadCloser - CloseWithError(err error) error -} - -type PipeWriter interface { - io.WriteCloser - CloseWithError(err error) error -} diff --git a/utils/ioutil/pipe.go b/utils/ioutil/pipe.go deleted file mode 100644 index f30c452fa..000000000 --- a/utils/ioutil/pipe.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !js - -package ioutil - -import "io" - -func Pipe() (PipeReader, PipeWriter) { - return io.Pipe() -} diff --git a/utils/ioutil/pipe_js.go b/utils/ioutil/pipe_js.go deleted file mode 100644 index cf102e6ef..000000000 --- a/utils/ioutil/pipe_js.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build js - -package ioutil - -import "github.com/acomagu/bufpipe" - -func Pipe() (PipeReader, PipeWriter) { - return bufpipe.New(nil) -} From a8e17350b8b1c97e252964c863fca0fac9ff5cec Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 8 Nov 2023 18:40:06 -0500 Subject: [PATCH 39/56] internal: add trace package This adds a generic tracing package to log messages to output if target is enabled. --- internal/trace/trace.go | 55 +++++++++++++++++++++ internal/trace/trace_test.go | 95 ++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 internal/trace/trace.go create mode 100644 internal/trace/trace_test.go diff --git a/internal/trace/trace.go b/internal/trace/trace.go new file mode 100644 index 000000000..3e15c5b9f --- /dev/null +++ b/internal/trace/trace.go @@ -0,0 +1,55 @@ +package trace + +import ( + "fmt" + "log" + "os" + "sync/atomic" +) + +var ( + // logger is the logger to use for tracing. + logger = newLogger() + + // current is the targets that are enabled for tracing. + current atomic.Int32 +) + +func newLogger() *log.Logger { + return log.New(os.Stderr, "", log.Ltime|log.Lmicroseconds|log.Lshortfile) +} + +// Target is a tracing target. +type Target int32 + +const ( + // General traces general operations. + General Target = 1 << iota + + // Packet traces git packets. + Packet +) + +// SetTarget sets the tracing targets. +func SetTarget(target Target) { + current.Store(int32(target)) +} + +// SetLogger sets the logger to use for tracing. +func SetLogger(l *log.Logger) { + logger = l +} + +// Print prints the given message only if the target is enabled. +func (t Target) Print(args ...interface{}) { + if int32(t)¤t.Load() != 0 { + logger.Output(2, fmt.Sprint(args...)) // nolint: errcheck + } +} + +// Printf prints the given message only if the target is enabled. +func (t Target) Printf(format string, args ...interface{}) { + if int32(t)¤t.Load() != 0 { + logger.Output(2, fmt.Sprintf(format, args...)) // nolint: errcheck + } +} diff --git a/internal/trace/trace_test.go b/internal/trace/trace_test.go new file mode 100644 index 000000000..6f8f14094 --- /dev/null +++ b/internal/trace/trace_test.go @@ -0,0 +1,95 @@ +package trace + +import ( + "bytes" + "io" + "log" + "testing" +) + +func TestMain(m *testing.M) { + defer SetLogger(newLogger()) + if code := m.Run(); code != 0 { + panic(code) + } +} + +func setUpTest(t testing.TB, buf *bytes.Buffer) { + t.Cleanup(func() { + if buf != nil { + buf.Reset() + } + SetTarget(0) + }) + w := io.Discard + if buf != nil { + w = buf + } + SetLogger(log.New(w, "", 0)) +} + +func TestEmpty(t *testing.T) { + var buf bytes.Buffer + setUpTest(t, &buf) + General.Print("test") + if buf.String() != "" { + t.Error("expected empty string") + } +} + +func TestOneTarget(t *testing.T) { + var buf bytes.Buffer + setUpTest(t, &buf) + SetTarget(General) + General.Print("test") + if buf.String() != "test\n" { + t.Error("expected 'test'") + } +} + +func TestMultipleTargets(t *testing.T) { + var buf bytes.Buffer + setUpTest(t, &buf) + SetTarget(General | Packet) + General.Print("a") + Packet.Print("b") + if buf.String() != "a\nb\n" { + t.Error("expected 'a\nb\n'") + } +} + +func TestPrintf(t *testing.T) { + var buf bytes.Buffer + setUpTest(t, &buf) + SetTarget(General) + General.Printf("a %d", 1) + if buf.String() != "a 1\n" { + t.Error("expected 'a 1\n'") + } +} + +func TestDisabledMultipleTargets(t *testing.T) { + var buf bytes.Buffer + setUpTest(t, &buf) + SetTarget(General) + General.Print("a") + Packet.Print("b") + if buf.String() != "a\n" { + t.Error("expected 'a\n'") + } +} + +func BenchmarkDisabledTarget(b *testing.B) { + setUpTest(b, nil) + for i := 0; i < b.N; i++ { + General.Print("test") + } +} + +func BenchmarkEnabledTarget(b *testing.B) { + setUpTest(b, nil) + SetTarget(General) + for i := 0; i < b.N; i++ { + General.Print("test") + } +} From c3843453da23043b4a4dcb9f7968f72c1e4ae8cc Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 15 Nov 2023 16:53:42 -0500 Subject: [PATCH 40/56] plumbing: format/pktline: trace packets --- plumbing/format/pktline/encoder.go | 4 ++++ plumbing/format/pktline/scanner.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/plumbing/format/pktline/encoder.go b/plumbing/format/pktline/encoder.go index 6d409795b..1b9b84e8a 100644 --- a/plumbing/format/pktline/encoder.go +++ b/plumbing/format/pktline/encoder.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "io" + + "github.com/go-git/go-git/v5/internal/trace" ) // An Encoder writes pkt-lines to an output stream. @@ -43,6 +45,7 @@ func NewEncoder(w io.Writer) *Encoder { // Flush encodes a flush-pkt to the output stream. func (e *Encoder) Flush() error { + defer trace.Packet.Print("packet: > 0000") _, err := e.w.Write(FlushPkt) return err } @@ -70,6 +73,7 @@ func (e *Encoder) encodeLine(p []byte) error { } n := len(p) + 4 + defer trace.Packet.Printf("packet: > %04x %s", n, p) if _, err := e.w.Write(asciiHex16(n)); err != nil { return err } diff --git a/plumbing/format/pktline/scanner.go b/plumbing/format/pktline/scanner.go index 99aab46e8..86068af79 100644 --- a/plumbing/format/pktline/scanner.go +++ b/plumbing/format/pktline/scanner.go @@ -3,6 +3,8 @@ package pktline import ( "errors" "io" + + "github.com/go-git/go-git/v5/internal/trace" ) const ( @@ -65,6 +67,7 @@ func (s *Scanner) Scan() bool { return false } s.payload = s.payload[:l] + trace.Packet.Printf("packet: < %04x %s", l, s.payload) return true } From c62aa3e780da6f85f6e1ddd7628ab30ec9d92b53 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 16 Nov 2023 11:15:10 -0500 Subject: [PATCH 41/56] utils: move trace to utils Without exposing `trace`, we can't set a target to enable tracing from out of go-git. Fixes: https://github.com/go-git/go-git/pull/916 --- plumbing/format/pktline/encoder.go | 2 +- plumbing/format/pktline/scanner.go | 2 +- {internal => utils}/trace/trace.go | 0 {internal => utils}/trace/trace_test.go | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename {internal => utils}/trace/trace.go (100%) rename {internal => utils}/trace/trace_test.go (100%) diff --git a/plumbing/format/pktline/encoder.go b/plumbing/format/pktline/encoder.go index 1b9b84e8a..b6144faf5 100644 --- a/plumbing/format/pktline/encoder.go +++ b/plumbing/format/pktline/encoder.go @@ -8,7 +8,7 @@ import ( "fmt" "io" - "github.com/go-git/go-git/v5/internal/trace" + "github.com/go-git/go-git/v5/utils/trace" ) // An Encoder writes pkt-lines to an output stream. diff --git a/plumbing/format/pktline/scanner.go b/plumbing/format/pktline/scanner.go index 86068af79..5e85ed092 100644 --- a/plumbing/format/pktline/scanner.go +++ b/plumbing/format/pktline/scanner.go @@ -4,7 +4,7 @@ import ( "errors" "io" - "github.com/go-git/go-git/v5/internal/trace" + "github.com/go-git/go-git/v5/utils/trace" ) const ( diff --git a/internal/trace/trace.go b/utils/trace/trace.go similarity index 100% rename from internal/trace/trace.go rename to utils/trace/trace.go diff --git a/internal/trace/trace_test.go b/utils/trace/trace_test.go similarity index 100% rename from internal/trace/trace_test.go rename to utils/trace/trace_test.go From 05551b70f689944ab558a2b326b2174f313d372a Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 16 Nov 2023 11:28:02 -0500 Subject: [PATCH 42/56] plumbing: fix empty uploadpack request error If we have all what we asked for, finish the session and handle error. This is equivalent of git "Already up to date." message. Fixes: https://github.com/go-git/go-git/issues/328 Fixes: https://github.com/go-git/go-git/issues/638 Fixes: https://github.com/go-git/go-git/issues/157 --- plumbing/transport/internal/common/common.go | 6 ++++++ remote.go | 4 ++++ worktree_test.go | 14 ++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index 6574116b1..da1c2acce 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -245,6 +245,12 @@ func (s *session) handleAdvRefDecodeError(err error) error { // returned with the packfile content. The reader must be closed after reading. func (s *session) UploadPack(ctx context.Context, req *packp.UploadPackRequest) (*packp.UploadPackResponse, error) { if req.IsEmpty() { + // XXX: IsEmpty means haves are a subset of wants, in that case we have + // everything we asked for. Close the connection and return nil. + if err := s.finish(); err != nil { + return nil, err + } + // TODO:(v6) return nil here return nil, transport.ErrEmptyUploadPackRequest } diff --git a/remote.go b/remote.go index 2b4122e96..8ca9b102f 100644 --- a/remote.go +++ b/remote.go @@ -552,6 +552,10 @@ func (r *Remote) fetchPack(ctx context.Context, o *FetchOptions, s transport.Upl reader, err := s.UploadPack(ctx, req) if err != nil { + if errors.Is(err, transport.ErrEmptyUploadPackRequest) { + // XXX: no packfile provided, everything is up-to-date. + return nil + } return err } diff --git a/worktree_test.go b/worktree_test.go index 712695ae2..180bfb0b4 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -299,6 +299,20 @@ func (s *WorktreeSuite) TestPullAlreadyUptodate(c *C) { c.Assert(err, Equals, NoErrAlreadyUpToDate) } +func (s *WorktreeSuite) TestPullDepth(c *C) { + r, err := Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ + URL: fixtures.Basic().One().URL, + Depth: 1, + }) + + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + err = w.Pull(&PullOptions{}) + c.Assert(err, Equals, nil) +} + func (s *WorktreeSuite) TestCheckout(c *C) { fs := memfs.New() w := &Worktree{ From 2e14e3ab84edd9b0f54b355aeddfcadf1383ec67 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Mon, 9 Oct 2023 21:37:49 +0100 Subject: [PATCH 43/56] plumbing: transport/git, Improve tests error message When running the tests in an install in which git daemon is not installed (e.g. openSUSE), all the git transport tests will fail with a connection refused error. This was due the git server not being running in the first place. The tests will now fail with 'git daemon cannot be found'. Signed-off-by: Paulo Gomes --- plumbing/transport/git/common_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/plumbing/transport/git/common_test.go b/plumbing/transport/git/common_test.go index 73899198f..3cab93314 100644 --- a/plumbing/transport/git/common_test.go +++ b/plumbing/transport/git/common_test.go @@ -1,6 +1,7 @@ package git import ( + "bytes" "fmt" "net" "os" @@ -32,7 +33,12 @@ func (s *BaseSuite) SetUpTest(c *C) { See https://github.com/git-for-windows/git/issues/907`) } - var err error + cmd := exec.Command("git", "daemon", "--help") + output, err := cmd.CombinedOutput() + if err != nil && bytes.Contains(output, []byte("'daemon' is not a git command")) { + c.Fatal("git daemon cannot be found") + } + s.port, err = freePort() c.Assert(err, IsNil) @@ -85,11 +91,15 @@ func (s *BaseSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *t } func (s *BaseSuite) TearDownTest(c *C) { - _ = s.daemon.Process.Signal(os.Kill) - _ = s.daemon.Wait() + if s.daemon != nil { + _ = s.daemon.Process.Signal(os.Kill) + _ = s.daemon.Wait() + } - err := os.RemoveAll(s.base) - c.Assert(err, IsNil) + if s.base != "" { + err := os.RemoveAll(s.base) + c.Assert(err, IsNil) + } } func freePort() (int, error) { From 5349b8a45ed2712302e85b258d3789808e1bd96d Mon Sep 17 00:00:00 2001 From: steiler Date: Thu, 16 Nov 2023 12:08:26 +0100 Subject: [PATCH 44/56] utils: merkletrie, Skip loading sockets as filesystem nodes. Fixes #312 --- utils/merkletrie/filesystem/node.go | 4 ++++ utils/merkletrie/filesystem/node_test.go | 26 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/utils/merkletrie/filesystem/node.go b/utils/merkletrie/filesystem/node.go index ad169ff4a..7bba0d03e 100644 --- a/utils/merkletrie/filesystem/node.go +++ b/utils/merkletrie/filesystem/node.go @@ -103,6 +103,10 @@ func (n *node) calculateChildren() error { continue } + if file.Mode()&os.ModeSocket != 0 { + continue + } + c, err := n.newChildNode(file) if err != nil { return err diff --git a/utils/merkletrie/filesystem/node_test.go b/utils/merkletrie/filesystem/node_test.go index 159e63dcd..b76abc412 100644 --- a/utils/merkletrie/filesystem/node_test.go +++ b/utils/merkletrie/filesystem/node_test.go @@ -2,9 +2,12 @@ package filesystem import ( "bytes" + "fmt" "io" + "net" "os" "path" + "runtime" "testing" "github.com/go-git/go-git/v5/plumbing" @@ -13,6 +16,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-billy/v5/osfs" . "gopkg.in/check.v1" ) @@ -196,6 +200,28 @@ func (s *NoderSuite) TestDiffDirectory(c *C) { c.Assert(a, Equals, merkletrie.Modify) } +func (s *NoderSuite) TestSocket(c *C) { + if runtime.GOOS == "windows" { + c.Skip("socket files do not exist on windows") + } + + td, err := os.MkdirTemp("", "socket-test") + defer os.RemoveAll(td) + c.Assert(err, IsNil) + + sock, err := net.ListenUnix("unix", &net.UnixAddr{Name: fmt.Sprintf("%s/socket", td), Net: "unix"}) + c.Assert(err, IsNil) + defer sock.Close() + + fsA := osfs.New(td) + WriteFile(fsA, "foo", []byte("foo"), 0644) + + noder := NewRootNode(fsA, nil) + childs, err := noder.Children() + c.Assert(err, IsNil) + c.Assert(childs, HasLen, 1) +} + func WriteFile(fs billy.Filesystem, filename string, data []byte, perm os.FileMode) error { f, err := fs.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { From e1875336041ead704df7f481f8691452ef939556 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Fri, 17 Nov 2023 15:27:49 -0500 Subject: [PATCH 45/56] plumbing: add git-proto-request type This the request used for the git transport protocol feat: nil writer --- plumbing/protocol/packp/common.go | 5 ++ plumbing/protocol/packp/gitproto.go | 115 ++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 plumbing/protocol/packp/gitproto.go diff --git a/plumbing/protocol/packp/common.go b/plumbing/protocol/packp/common.go index fef50a450..a858323e7 100644 --- a/plumbing/protocol/packp/common.go +++ b/plumbing/protocol/packp/common.go @@ -48,6 +48,11 @@ func isFlush(payload []byte) bool { return len(payload) == 0 } +var ( + // ErrNilWriter is returned when a nil writer is passed to the encoder. + ErrNilWriter = fmt.Errorf("nil writer") +) + // ErrUnexpectedData represents an unexpected data decoding a message type ErrUnexpectedData struct { Msg string diff --git a/plumbing/protocol/packp/gitproto.go b/plumbing/protocol/packp/gitproto.go new file mode 100644 index 000000000..806e44ce9 --- /dev/null +++ b/plumbing/protocol/packp/gitproto.go @@ -0,0 +1,115 @@ +package packp + +import ( + "fmt" + "io" + "strings" + + "github.com/go-git/go-git/v5/plumbing/format/pktline" +) + +var ( + // ErrInvalidGitProtoRequest is returned by Decode if the input is not a + // valid git protocol request. + ErrInvalidGitProtoRequest = fmt.Errorf("invalid git protocol request") +) + +// GitProtoRequest is a command request for the git protocol. +// It is used to send the command, endpoint, and extra parameters to the +// remote. +// See https://git-scm.com/docs/pack-protocol#_git_transport +type GitProtoRequest struct { + RequestCommand string + Pathname string + + // Optional + Host string + + // Optional + ExtraParams []string +} + +// validate validates the request. +func (g *GitProtoRequest) validate() error { + if g.RequestCommand == "" { + return fmt.Errorf("%w: empty request command", ErrInvalidGitProtoRequest) + } + + if g.Pathname == "" { + return fmt.Errorf("%w: empty pathname", ErrInvalidGitProtoRequest) + } + + return nil +} + +// Encode encodes the request into the writer. +func (g *GitProtoRequest) Encode(w io.Writer) error { + if w == nil { + return ErrNilWriter + } + + if err := g.validate(); err != nil { + return err + } + + p := pktline.NewEncoder(w) + req := fmt.Sprintf("%s %s\x00", g.RequestCommand, g.Pathname) + if host := g.Host; host != "" { + req += fmt.Sprintf("host=%s\x00", host) + } + + if len(g.ExtraParams) > 0 { + req += "\x00" + for _, param := range g.ExtraParams { + req += param + "\x00" + } + } + + if err := p.Encode([]byte(req)); err != nil { + return err + } + + return nil +} + +// Decode decodes the request from the reader. +func (g *GitProtoRequest) Decode(r io.Reader) error { + s := pktline.NewScanner(r) + if !s.Scan() { + return s.Err() + } + + line := string(s.Bytes()) + if len(line) == 0 { + return io.EOF + } + + if line[len(line)-1] != 0 { + return fmt.Errorf("%w: missing null terminator", ErrInvalidGitProtoRequest) + } + + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return fmt.Errorf("%w: short request", ErrInvalidGitProtoRequest) + } + + g.RequestCommand = parts[0] + params := strings.Split(parts[1], string(null)) + if len(params) < 1 { + return fmt.Errorf("%w: missing pathname", ErrInvalidGitProtoRequest) + } + + if len(params) > 1 { + g.Host = params[1] + } + + if len(params) > 2 { + for _, param := range params[2:] { + if param != "" { + g.ExtraParams = append(g.ExtraParams, param) + } + } + } + + return nil +} From e2c6ae3333a3facd13aa52e1986a2ba2dbc56a9d Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Fri, 17 Nov 2023 15:28:25 -0500 Subject: [PATCH 46/56] plumbing: handle pktline erro-line as errors Error when encountering an error-line See https://git-scm.com/docs/pack-protocol Update plumbing/format/pktline/error.go Co-authored-by: Paulo Gomes Update plumbing/format/pktline/error.go Co-authored-by: Paulo Gomes feat: format newline --- plumbing/format/pktline/error.go | 51 ++++++++++++++++++++ plumbing/format/pktline/error_test.go | 68 +++++++++++++++++++++++++++ plumbing/format/pktline/scanner.go | 9 ++++ 3 files changed, 128 insertions(+) create mode 100644 plumbing/format/pktline/error.go create mode 100644 plumbing/format/pktline/error_test.go diff --git a/plumbing/format/pktline/error.go b/plumbing/format/pktline/error.go new file mode 100644 index 000000000..2c0e5a72a --- /dev/null +++ b/plumbing/format/pktline/error.go @@ -0,0 +1,51 @@ +package pktline + +import ( + "bytes" + "errors" + "io" + "strings" +) + +var ( + // ErrInvalidErrorLine is returned by Decode when the packet line is not an + // error line. + ErrInvalidErrorLine = errors.New("expected an error-line") + + errPrefix = []byte("ERR ") +) + +// ErrorLine is a packet line that contains an error message. +// Once this packet is sent by client or server, the data transfer process is +// terminated. +// See https://git-scm.com/docs/pack-protocol#_pkt_line_format +type ErrorLine struct { + Text string +} + +// Error implements the error interface. +func (e *ErrorLine) Error() string { + return e.Text +} + +// Encode encodes the ErrorLine into a packet line. +func (e *ErrorLine) Encode(w io.Writer) error { + p := NewEncoder(w) + return p.Encodef("%s%s\n", string(errPrefix), e.Text) +} + +// Decode decodes a packet line into an ErrorLine. +func (e *ErrorLine) Decode(r io.Reader) error { + s := NewScanner(r) + if !s.Scan() { + return s.Err() + } + + line := s.Bytes() + if !bytes.HasPrefix(line, errPrefix) { + return ErrInvalidErrorLine + } + + e.Text = strings.TrimSpace(string(line[4:])) + return nil +} diff --git a/plumbing/format/pktline/error_test.go b/plumbing/format/pktline/error_test.go new file mode 100644 index 000000000..3cffd20d1 --- /dev/null +++ b/plumbing/format/pktline/error_test.go @@ -0,0 +1,68 @@ +package pktline + +import ( + "bytes" + "errors" + "io" + "testing" +) + +func TestEncodeEmptyErrorLine(t *testing.T) { + e := &ErrorLine{} + err := e.Encode(io.Discard) + if err != nil { + t.Fatal(err) + } +} + +func TestEncodeErrorLine(t *testing.T) { + e := &ErrorLine{ + Text: "something", + } + var buf bytes.Buffer + err := e.Encode(&buf) + if err != nil { + t.Fatal(err) + } + if buf.String() != "0012ERR something\n" { + t.Fatalf("unexpected encoded error line: %q", buf.String()) + } +} + +func TestDecodeEmptyErrorLine(t *testing.T) { + var buf bytes.Buffer + e := &ErrorLine{} + err := e.Decode(&buf) + if err != nil { + t.Fatal(err) + } + if e.Text != "" { + t.Fatalf("unexpected error line: %q", e.Text) + } +} + +func TestDecodeErrorLine(t *testing.T) { + var buf bytes.Buffer + buf.WriteString("000eERR foobar") + var e *ErrorLine + err := e.Decode(&buf) + if !errors.As(err, &e) { + t.Fatalf("expected error line, got: %T: %v", err, err) + } + if e.Text != "foobar" { + t.Fatalf("unexpected error line: %q", e.Text) + } +} + +func TestDecodeErrorLineLn(t *testing.T) { + var buf bytes.Buffer + buf.WriteString("000fERR foobar\n") + var e *ErrorLine + err := e.Decode(&buf) + if !errors.As(err, &e) { + t.Fatalf("expected error line, got: %T: %v", err, err) + } + if e.Text != "foobar" { + t.Fatalf("unexpected error line: %q", e.Text) + } +} diff --git a/plumbing/format/pktline/scanner.go b/plumbing/format/pktline/scanner.go index 5e85ed092..fbb137de0 100644 --- a/plumbing/format/pktline/scanner.go +++ b/plumbing/format/pktline/scanner.go @@ -1,8 +1,10 @@ package pktline import ( + "bytes" "errors" "io" + "strings" "github.com/go-git/go-git/v5/utils/trace" ) @@ -69,6 +71,13 @@ func (s *Scanner) Scan() bool { s.payload = s.payload[:l] trace.Packet.Printf("packet: < %04x %s", l, s.payload) + if bytes.HasPrefix(s.payload, errPrefix) { + s.err = &ErrorLine{ + Text: strings.TrimSpace(string(s.payload[4:])), + } + return false + } + return true } From f46d04a18759071e45b057908050a01e061f1ddd Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Fri, 17 Nov 2023 15:29:33 -0500 Subject: [PATCH 47/56] plumbing: transport: use git-proto-request and decode error-line errors --- plumbing/protocol/packp/gitproto.go | 9 +- plumbing/protocol/packp/gitproto_test.go | 99 +++++++++++++++++++ plumbing/transport/git/common.go | 26 +++-- plumbing/transport/internal/common/common.go | 77 +++++++-------- .../transport/internal/common/common_test.go | 60 +---------- 5 files changed, 157 insertions(+), 114 deletions(-) create mode 100644 plumbing/protocol/packp/gitproto_test.go diff --git a/plumbing/protocol/packp/gitproto.go b/plumbing/protocol/packp/gitproto.go index 806e44ce9..0b7ff8f82 100644 --- a/plumbing/protocol/packp/gitproto.go +++ b/plumbing/protocol/packp/gitproto.go @@ -76,7 +76,11 @@ func (g *GitProtoRequest) Encode(w io.Writer) error { func (g *GitProtoRequest) Decode(r io.Reader) error { s := pktline.NewScanner(r) if !s.Scan() { - return s.Err() + err := s.Err() + if err == nil { + return ErrInvalidGitProtoRequest + } + return err } line := string(s.Bytes()) @@ -99,8 +103,9 @@ func (g *GitProtoRequest) Decode(r io.Reader) error { return fmt.Errorf("%w: missing pathname", ErrInvalidGitProtoRequest) } + g.Pathname = params[0] if len(params) > 1 { - g.Host = params[1] + g.Host = strings.TrimPrefix(params[1], "host=") } if len(params) > 2 { diff --git a/plumbing/protocol/packp/gitproto_test.go b/plumbing/protocol/packp/gitproto_test.go new file mode 100644 index 000000000..9cf1049fd --- /dev/null +++ b/plumbing/protocol/packp/gitproto_test.go @@ -0,0 +1,99 @@ +package packp + +import ( + "bytes" + "testing" +) + +func TestEncodeEmptyGitProtoRequest(t *testing.T) { + var buf bytes.Buffer + var p GitProtoRequest + err := p.Encode(&buf) + if err == nil { + t.Fatal("expected error") + } +} + +func TestEncodeGitProtoRequest(t *testing.T) { + var buf bytes.Buffer + p := GitProtoRequest{ + RequestCommand: "command", + Pathname: "pathname", + Host: "host", + ExtraParams: []string{"param1", "param2"}, + } + err := p.Encode(&buf) + if err != nil { + t.Fatal(err) + } + expected := "002ecommand pathname\x00host=host\x00\x00param1\x00param2\x00" + if buf.String() != expected { + t.Fatalf("expected %q, got %q", expected, buf.String()) + } +} + +func TestEncodeInvalidGitProtoRequest(t *testing.T) { + var buf bytes.Buffer + p := GitProtoRequest{ + RequestCommand: "command", + } + err := p.Encode(&buf) + if err == nil { + t.Fatal("expected error") + } +} + +func TestDecodeEmptyGitProtoRequest(t *testing.T) { + var buf bytes.Buffer + var p GitProtoRequest + err := p.Decode(&buf) + if err == nil { + t.Fatal("expected error") + } +} + +func TestDecodeGitProtoRequest(t *testing.T) { + var buf bytes.Buffer + buf.WriteString("002ecommand pathname\x00host=host\x00\x00param1\x00param2\x00") + var p GitProtoRequest + err := p.Decode(&buf) + if err != nil { + t.Fatal(err) + } + expected := GitProtoRequest{ + RequestCommand: "command", + Pathname: "pathname", + Host: "host", + ExtraParams: []string{"param1", "param2"}, + } + if p.RequestCommand != expected.RequestCommand { + t.Fatalf("expected %q, got %q", expected.RequestCommand, p.RequestCommand) + } + if p.Pathname != expected.Pathname { + t.Fatalf("expected %q, got %q", expected.Pathname, p.Pathname) + } + if p.Host != expected.Host { + t.Fatalf("expected %q, got %q", expected.Host, p.Host) + } + if len(p.ExtraParams) != len(expected.ExtraParams) { + t.Fatalf("expected %d, got %d", len(expected.ExtraParams), len(p.ExtraParams)) + } +} + +func TestDecodeInvalidGitProtoRequest(t *testing.T) { + var buf bytes.Buffer + buf.WriteString("0026command \x00host=host\x00\x00param1\x00param2") + var p GitProtoRequest + err := p.Decode(&buf) + if err == nil { + t.Fatal("expected error") + } +} + +func TestValidateEmptyGitProtoRequest(t *testing.T) { + var p GitProtoRequest + err := p.validate() + if err == nil { + t.Fatal("expected error") + } +} diff --git a/plumbing/transport/git/common.go b/plumbing/transport/git/common.go index 92fc0becc..2b878b035 100644 --- a/plumbing/transport/git/common.go +++ b/plumbing/transport/git/common.go @@ -2,12 +2,11 @@ package git import ( - "fmt" "io" "net" "strconv" - "github.com/go-git/go-git/v5/plumbing/format/pktline" + "github.com/go-git/go-git/v5/plumbing/protocol/packp" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/internal/common" "github.com/go-git/go-git/v5/utils/ioutil" @@ -42,10 +41,18 @@ type command struct { // Start executes the command sending the required message to the TCP connection func (c *command) Start() error { - cmd := endpointToCommand(c.command, c.endpoint) + req := packp.GitProtoRequest{ + RequestCommand: c.command, + Pathname: c.endpoint.Path, + } + host := c.endpoint.Host + if c.endpoint.Port != DefaultPort { + host = net.JoinHostPort(c.endpoint.Host, strconv.Itoa(c.endpoint.Port)) + } + + req.Host = host - e := pktline.NewEncoder(c.conn) - return e.Encode([]byte(cmd)) + return req.Encode(c.conn) } func (c *command) connect() error { @@ -90,15 +97,6 @@ func (c *command) StdoutPipe() (io.Reader, error) { return c.conn, nil } -func endpointToCommand(cmd string, ep *transport.Endpoint) string { - host := ep.Host - if ep.Port != DefaultPort { - host = net.JoinHostPort(ep.Host, strconv.Itoa(ep.Port)) - } - - return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path, 0, host, 0) -} - // Close closes the TCP connection and connection. func (c *command) Close() error { if !c.connected { diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index da1c2acce..9e1d02357 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -203,9 +203,22 @@ func (s *session) AdvertisedReferencesContext(ctx context.Context) (*packp.AdvRe } func (s *session) handleAdvRefDecodeError(err error) error { + var errLine *pktline.ErrorLine + if errors.As(err, &errLine) { + if isRepoNotFoundError(errLine.Text) { + return transport.ErrRepositoryNotFound + } + + return errLine + } + // If repository is not found, we get empty stdout and server writes an // error to stderr. - if err == packp.ErrEmptyInput { + if errors.Is(err, packp.ErrEmptyInput) { + // TODO:(v6): handle this error in a better way. + // Instead of checking the stderr output for a specific error message, + // define an ExitError and embed the stderr output and exit (if one + // exists) in the error struct. Just like exec.ExitError. s.finished = true if err := s.checkNotFoundError(); err != nil { return err @@ -399,59 +412,43 @@ func (s *session) checkNotFoundError() error { return transport.ErrRepositoryNotFound } + // TODO:(v6): return server error just as it is without a prefix return fmt.Errorf("unknown error: %s", line) } } -var ( - githubRepoNotFoundErr = "ERROR: Repository not found." - bitbucketRepoNotFoundErr = "conq: repository does not exist." +const ( + githubRepoNotFoundErr = "Repository not found." + bitbucketRepoNotFoundErr = "repository does not exist." localRepoNotFoundErr = "does not appear to be a git repository" - gitProtocolNotFoundErr = "ERR \n Repository not found." - gitProtocolNoSuchErr = "ERR no such repository" - gitProtocolAccessDeniedErr = "ERR access denied" - gogsAccessDeniedErr = "Gogs: Repository does not exist or you do not have access" - gitlabRepoNotFoundErr = "remote: ERROR: The project you were looking for could not be found" + gitProtocolNotFoundErr = "Repository not found." + gitProtocolNoSuchErr = "no such repository" + gitProtocolAccessDeniedErr = "access denied" + gogsAccessDeniedErr = "Repository does not exist or you do not have access" + gitlabRepoNotFoundErr = "The project you were looking for could not be found" ) func isRepoNotFoundError(s string) bool { - if strings.HasPrefix(s, githubRepoNotFoundErr) { - return true - } - - if strings.HasPrefix(s, bitbucketRepoNotFoundErr) { - return true - } - - if strings.HasSuffix(s, localRepoNotFoundErr) { - return true - } - - if strings.HasPrefix(s, gitProtocolNotFoundErr) { - return true - } - - if strings.HasPrefix(s, gitProtocolNoSuchErr) { - return true - } - - if strings.HasPrefix(s, gitProtocolAccessDeniedErr) { - return true - } - - if strings.HasPrefix(s, gogsAccessDeniedErr) { - return true - } - - if strings.HasPrefix(s, gitlabRepoNotFoundErr) { - return true + for _, err := range []string{ + githubRepoNotFoundErr, + bitbucketRepoNotFoundErr, + localRepoNotFoundErr, + gitProtocolNotFoundErr, + gitProtocolNoSuchErr, + gitProtocolAccessDeniedErr, + gogsAccessDeniedErr, + gitlabRepoNotFoundErr, + } { + if strings.Contains(s, err) { + return true + } } return false } // uploadPack implements the git-upload-pack protocol. -func uploadPack(w io.WriteCloser, r io.Reader, req *packp.UploadPackRequest) error { +func uploadPack(w io.WriteCloser, _ io.Reader, req *packp.UploadPackRequest) error { // TODO support multi_ack mode // TODO support multi_ack_detailed mode // TODO support acks for common objects diff --git a/plumbing/transport/internal/common/common_test.go b/plumbing/transport/internal/common/common_test.go index f6f2f67d2..9344bb62b 100644 --- a/plumbing/transport/internal/common/common_test.go +++ b/plumbing/transport/internal/common/common_test.go @@ -22,64 +22,8 @@ func (s *CommonSuite) TestIsRepoNotFoundErrorForUnknownSource(c *C) { c.Assert(isRepoNotFound, Equals, false) } -func (s *CommonSuite) TestIsRepoNotFoundErrorForGithub(c *C) { - msg := fmt.Sprintf("%s : some error stuf", githubRepoNotFoundErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForBitBucket(c *C) { - msg := fmt.Sprintf("%s : some error stuf", bitbucketRepoNotFoundErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForLocal(c *C) { - msg := fmt.Sprintf("some error stuf : %s", localRepoNotFoundErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForGitProtocolNotFound(c *C) { - msg := fmt.Sprintf("%s : some error stuf", gitProtocolNotFoundErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForGitProtocolNoSuch(c *C) { - msg := fmt.Sprintf("%s : some error stuf", gitProtocolNoSuchErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForGitProtocolAccessDenied(c *C) { - msg := fmt.Sprintf("%s : some error stuf", gitProtocolAccessDeniedErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForGogsAccessDenied(c *C) { - msg := fmt.Sprintf("%s : some error stuf", gogsAccessDeniedErr) - - isRepoNotFound := isRepoNotFoundError(msg) - - c.Assert(isRepoNotFound, Equals, true) -} - -func (s *CommonSuite) TestIsRepoNotFoundErrorForGitlab(c *C) { - msg := fmt.Sprintf("%s : some error stuf", gitlabRepoNotFoundErr) +func (s *CommonSuite) TestIsRepoNotFoundError(c *C) { + msg := "no such repository : some error stuf" isRepoNotFound := isRepoNotFoundError(msg) From 861009f70a5b68d1ee134fea7bc1d893088388bb Mon Sep 17 00:00:00 2001 From: Dan Hoizner Date: Wed, 22 Nov 2023 12:51:45 -0500 Subject: [PATCH 48/56] git: stop iterating at oldest shallow when pulling. Fixes #305 --- remote.go | 23 ++++++++++++++++++----- worktree.go | 13 +++++++++++-- worktree_test.go | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/remote.go b/remote.go index 8ca9b102f..0cb70bc00 100644 --- a/remote.go +++ b/remote.go @@ -1070,7 +1070,7 @@ func checkFastForwardUpdate(s storer.EncodedObjectStorer, remoteRefs storer.Refe return fmt.Errorf("non-fast-forward update: %s", cmd.Name.String()) } - ff, err := isFastForward(s, cmd.Old, cmd.New) + ff, err := isFastForward(s, cmd.Old, cmd.New, nil) if err != nil { return err } @@ -1082,14 +1082,28 @@ func checkFastForwardUpdate(s storer.EncodedObjectStorer, remoteRefs storer.Refe return nil } -func isFastForward(s storer.EncodedObjectStorer, old, new plumbing.Hash) (bool, error) { +func isFastForward(s storer.EncodedObjectStorer, old, new plumbing.Hash, earliestShallow *plumbing.Hash) (bool, error) { c, err := object.GetCommit(s, new) if err != nil { return false, err } + parentsToIgnore := []plumbing.Hash{} + if earliestShallow != nil { + earliestCommit, err := object.GetCommit(s, *earliestShallow) + if err != nil { + return false, err + } + + parentsToIgnore = earliestCommit.ParentHashes + } + found := false - iter := object.NewCommitPreorderIter(c, nil, nil) + // stop iterating at the earlist shallow commit, ignoring its parents + // note: when pull depth is smaller than the number of new changes on the remote, this fails due to missing parents. + // as far as i can tell, without the commits in-between the shallow pull and the earliest shallow, there's no + // real way of telling whether it will be a fast-forward merge. + iter := object.NewCommitPreorderIter(c, nil, parentsToIgnore) err = iter.ForEach(func(c *object.Commit) error { if c.Hash != old { return nil @@ -1205,7 +1219,7 @@ func (r *Remote) updateLocalReferenceStorage( // If the ref exists locally as a non-tag and force is not // specified, only update if the new ref is an ancestor of the old if old != nil && !old.Name().IsTag() && !force && !spec.IsForceUpdate() { - ff, err := isFastForward(r.s, old.Hash(), new.Hash()) + ff, err := isFastForward(r.s, old.Hash(), new.Hash(), nil) if err != nil { return updated, err } @@ -1390,7 +1404,6 @@ func pushHashes( useRefDeltas bool, allDelete bool, ) (*packp.ReportStatus, error) { - rd, wr := io.Pipe() config, err := s.Config() diff --git a/worktree.go b/worktree.go index f8b854dda..974d2336a 100644 --- a/worktree.go +++ b/worktree.go @@ -95,7 +95,15 @@ func (w *Worktree) PullContext(ctx context.Context, o *PullOptions) error { head, err := w.r.Head() if err == nil { - headAheadOfRef, err := isFastForward(w.r.Storer, ref.Hash(), head.Hash()) + // if we don't have a shallows list, just ignore it + shallowList, _ := w.r.Storer.Shallow() + + var earliestShallow *plumbing.Hash + if len(shallowList) > 0 { + earliestShallow = &shallowList[0] + } + + headAheadOfRef, err := isFastForward(w.r.Storer, ref.Hash(), head.Hash(), earliestShallow) if err != nil { return err } @@ -104,7 +112,7 @@ func (w *Worktree) PullContext(ctx context.Context, o *PullOptions) error { return NoErrAlreadyUpToDate } - ff, err := isFastForward(w.r.Storer, head.Hash(), ref.Hash()) + ff, err := isFastForward(w.r.Storer, head.Hash(), ref.Hash(), earliestShallow) if err != nil { return err } @@ -188,6 +196,7 @@ func (w *Worktree) Checkout(opts *CheckoutOptions) error { return w.Reset(ro) } + func (w *Worktree) createBranch(opts *CheckoutOptions) error { _, err := w.r.Storer.Reference(opts.Branch) if err == nil { diff --git a/worktree_test.go b/worktree_test.go index 180bfb0b4..8805079d8 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -313,6 +313,42 @@ func (s *WorktreeSuite) TestPullDepth(c *C) { c.Assert(err, Equals, nil) } +func (s *WorktreeSuite) TestPullAfterShallowClone(c *C) { + tempDir, clean := s.TemporalDir() + defer clean() + remoteURL := filepath.Join(tempDir, "remote") + repoDir := filepath.Join(tempDir, "repo") + + remote, err := PlainInit(remoteURL, false) + c.Assert(err, IsNil) + c.Assert(remote, NotNil) + + _ = CommitNewFile(c, remote, "File1") + _ = CommitNewFile(c, remote, "File2") + + repo, err := PlainClone(repoDir, false, &CloneOptions{ + URL: remoteURL, + Depth: 1, + Tags: NoTags, + SingleBranch: true, + ReferenceName: "master", + }) + c.Assert(err, IsNil) + + _ = CommitNewFile(c, remote, "File3") + _ = CommitNewFile(c, remote, "File4") + + w, err := repo.Worktree() + c.Assert(err, IsNil) + + err = w.Pull(&PullOptions{ + RemoteName: DefaultRemoteName, + SingleBranch: true, + ReferenceName: plumbing.NewBranchReferenceName("master"), + }) + c.Assert(err, IsNil) +} + func (s *WorktreeSuite) TestCheckout(c *C) { fs := memfs.New() w := &Worktree{ From aebf868d2a557384539cf91476d21901ab378643 Mon Sep 17 00:00:00 2001 From: Daniel Moch Date: Fri, 24 Nov 2023 12:29:45 -0500 Subject: [PATCH 49/56] plumbing: object, enable renames in getFileStatsFromFilePatches Diff has handled renames by default since 2020. This change sets Name for the renamed file in a manner similar to diffstat. --- plumbing/object/patch.go | 4 +-- plumbing/object/patch_stats_test.go | 54 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 plumbing/object/patch_stats_test.go diff --git a/plumbing/object/patch.go b/plumbing/object/patch.go index 06bc35bbc..dd8fef447 100644 --- a/plumbing/object/patch.go +++ b/plumbing/object/patch.go @@ -317,8 +317,8 @@ func getFileStatsFromFilePatches(filePatches []fdiff.FilePatch) FileStats { // File is deleted. cs.Name = from.Path() } else if from.Path() != to.Path() { - // File is renamed. Not supported. - // cs.Name = fmt.Sprintf("%s => %s", from.Path(), to.Path()) + // File is renamed. + cs.Name = fmt.Sprintf("%s => %s", from.Path(), to.Path()) } else { cs.Name = from.Path() } diff --git a/plumbing/object/patch_stats_test.go b/plumbing/object/patch_stats_test.go new file mode 100644 index 000000000..f393c30c4 --- /dev/null +++ b/plumbing/object/patch_stats_test.go @@ -0,0 +1,54 @@ +package object_test + +import ( + "time" + + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-billy/v5/util" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" + + fixtures "github.com/go-git/go-git-fixtures/v4" + . "gopkg.in/check.v1" +) + +type PatchStatsSuite struct { + fixtures.Suite +} + +var _ = Suite(&PatchStatsSuite{}) + +func (s *PatchStatsSuite) TestStatsWithRename(c *C) { + cm := &git.CommitOptions{ + Author: &object.Signature{Name: "Foo", Email: "foo@example.local", When: time.Now()}, + } + + fs := memfs.New() + r, err := git.Init(memory.NewStorage(), fs) + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + util.WriteFile(fs, "foo", []byte("foo\nbar\n"), 0644) + + _, err = w.Add("foo") + c.Assert(err, IsNil) + + _, err = w.Commit("foo\n", cm) + c.Assert(err, IsNil) + + _, err = w.Move("foo", "bar") + c.Assert(err, IsNil) + + hash, err := w.Commit("rename foo to bar", cm) + c.Assert(err, IsNil) + + commit, err := r.CommitObject(hash) + c.Assert(err, IsNil) + + fileStats, err := commit.Stats() + c.Assert(err, IsNil) + c.Assert(fileStats[0].Name, Equals, "foo => bar") +} From 93b009cae97a84586b944f34006fcb178e77ed10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:51:23 +0000 Subject: [PATCH 50/56] build: bump golang.org/x/crypto from 0.15.0 to 0.16.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.15.0 to 0.16.0. - [Commits](https://github.com/golang/crypto/compare/v0.15.0...v0.16.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 18b60da9e..7bdee7b7f 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,9 @@ require ( github.com/sergi/go-diff v1.1.0 github.com/skeema/knownhosts v1.2.1 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.15.0 + golang.org/x/crypto v0.16.0 golang.org/x/net v0.18.0 - golang.org/x/sys v0.14.0 + golang.org/x/sys v0.15.0 golang.org/x/text v0.14.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) diff --git a/go.sum b/go.sum index f5af02373..6f4755b5d 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= @@ -113,15 +113,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From a24037512f4edd5a591a2c8b2077519424225783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:16:26 +0000 Subject: [PATCH 51/56] build: bump golang.org/x/net from 0.18.0 to 0.19.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.18.0 to 0.19.0. - [Commits](https://github.com/golang/net/compare/v0.18.0...v0.19.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7bdee7b7f..9a18c7a49 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/skeema/knownhosts v1.2.1 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.16.0 - golang.org/x/net v0.18.0 + golang.org/x/net v0.19.0 golang.org/x/sys v0.15.0 golang.org/x/text v0.14.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 6f4755b5d..4be7ec4eb 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From d85bc9920e5d0d9f023f7462096378a71429c549 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:43:14 +0000 Subject: [PATCH 52/56] build: bump github.com/go-git/go-git/v5 in /cli/go-git Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.10.1-0.20231107163107-e54a6ae399e9 to 5.10.1. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/commits/v5.10.1) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- cli/go-git/go.mod | 7 +++---- cli/go-git/go.sum | 18 +++++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/cli/go-git/go.mod b/cli/go-git/go.mod index bde9014cc..5cc0d18ad 100644 --- a/cli/go-git/go.mod +++ b/cli/go-git/go.mod @@ -3,7 +3,7 @@ module github.com/go-git/v5/go-git/cli/go-git go 1.19 require ( - github.com/go-git/go-git/v5 v5.10.1-0.20231107163107-e54a6ae399e9 + github.com/go-git/go-git/v5 v5.10.1 github.com/jessevdk/go-flags v1.5.0 ) @@ -11,7 +11,6 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -24,9 +23,9 @@ require ( github.com/sergi/go-diff v1.1.0 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/cli/go-git/go.sum b/cli/go-git/go.sum index 00a9140dc..45c055264 100644 --- a/cli/go-git/go.sum +++ b/cli/go-git/go.sum @@ -5,8 +5,6 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -26,8 +24,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git/v5 v5.10.1-0.20231107163107-e54a6ae399e9 h1:uNsBlaMgh9sjbk+uvOJSJm+ixmoJoXbeDFNBCgYqUXo= -github.com/go-git/go-git/v5 v5.10.1-0.20231107163107-e54a6ae399e9/go.mod h1:vQMZfh4ro7khF6K55zuBL6sU45QYjjppZ6Cot47a9t4= +github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk= +github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -42,8 +40,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= @@ -69,8 +65,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= @@ -82,8 +78,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -109,7 +105,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From de1d5a5978b9599ca3dacd58bbf699e4bb4cf6bd Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 28 Nov 2023 14:31:04 -0500 Subject: [PATCH 53/56] git: validate reference names Check reference names format before creating branches/tags/remotes. This should probably be in a lower level somewhere in `plumbing`. Validating the names under `plumbing.NewReference*` is not possible since these functions don't return errors. Fixes: https://github.com/go-git/go-git/issues/929 --- config/branch.go | 2 +- config/config.go | 3 +- plumbing/reference.go | 89 ++++++++++++++++++++++++++++++++++++++ plumbing/reference_test.go | 59 +++++++++++++++++++++++++ repository.go | 9 +++- repository_test.go | 37 ++++++++++++++++ worktree.go | 4 ++ worktree_test.go | 24 ++++++++++ 8 files changed, 224 insertions(+), 3 deletions(-) diff --git a/config/branch.go b/config/branch.go index 652270a28..db2cb499a 100644 --- a/config/branch.go +++ b/config/branch.go @@ -54,7 +54,7 @@ func (b *Branch) Validate() error { return errBranchInvalidRebase } - return nil + return plumbing.NewBranchReferenceName(b.Name).Validate() } func (b *Branch) marshal() *format.Subsection { diff --git a/config/config.go b/config/config.go index da425a784..6d41c15dc 100644 --- a/config/config.go +++ b/config/config.go @@ -13,6 +13,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/internal/url" + "github.com/go-git/go-git/v5/plumbing" format "github.com/go-git/go-git/v5/plumbing/format/config" ) @@ -614,7 +615,7 @@ func (c *RemoteConfig) Validate() error { c.Fetch = []RefSpec{RefSpec(fmt.Sprintf(DefaultFetchRefSpec, c.Name))} } - return nil + return plumbing.NewRemoteHEADReferenceName(c.Name).Validate() } func (c *RemoteConfig) unmarshal(s *format.Subsection) error { diff --git a/plumbing/reference.go b/plumbing/reference.go index 5a67f69e7..ddba93029 100644 --- a/plumbing/reference.go +++ b/plumbing/reference.go @@ -3,6 +3,7 @@ package plumbing import ( "errors" "fmt" + "regexp" "strings" ) @@ -29,6 +30,9 @@ var RefRevParseRules = []string{ var ( ErrReferenceNotFound = errors.New("reference not found") + + // ErrInvalidReferenceName is returned when a reference name is invalid. + ErrInvalidReferenceName = errors.New("invalid reference name") ) // ReferenceType reference type's @@ -124,6 +128,91 @@ func (r ReferenceName) Short() string { return res } +var ( + ctrlSeqs = regexp.MustCompile(`[\000-\037\177]`) +) + +// Validate validates a reference name. +// This follows the git-check-ref-format rules. +// See https://git-scm.com/docs/git-check-ref-format +// +// It is important to note that this function does not check if the reference +// exists in the repository. +// It only checks if the reference name is valid. +// This functions does not support the --refspec-pattern, --normalize, and +// --allow-onelevel options. +// +// Git imposes the following rules on how references are named: +// +// 1. They can include slash / for hierarchical (directory) grouping, but no +// slash-separated component can begin with a dot . or end with the +// sequence .lock. +// 2. They must contain at least one /. This enforces the presence of a +// category like heads/, tags/ etc. but the actual names are not +// restricted. If the --allow-onelevel option is used, this rule is +// waived. +// 3. They cannot have two consecutive dots .. anywhere. +// 4. They cannot have ASCII control characters (i.e. bytes whose values are +// lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : +// anywhere. +// 5. They cannot have question-mark ?, asterisk *, or open bracket [ +// anywhere. See the --refspec-pattern option below for an exception to this +// rule. +// 6. They cannot begin or end with a slash / or contain multiple consecutive +// slashes (see the --normalize option below for an exception to this rule). +// 7. They cannot end with a dot .. +// 8. They cannot contain a sequence @{. +// 9. They cannot be the single character @. +// 10. They cannot contain a \. +func (r ReferenceName) Validate() error { + s := string(r) + if len(s) == 0 { + return ErrInvalidReferenceName + } + + // HEAD is a special case + if r == HEAD { + return nil + } + + // rule 7 + if strings.HasSuffix(s, ".") { + return ErrInvalidReferenceName + } + + // rule 2 + parts := strings.Split(s, "/") + if len(parts) < 2 { + return ErrInvalidReferenceName + } + + isBranch := r.IsBranch() + isTag := r.IsTag() + for _, part := range parts { + // rule 6 + if len(part) == 0 { + return ErrInvalidReferenceName + } + + if strings.HasPrefix(part, ".") || // rule 1 + strings.Contains(part, "..") || // rule 3 + ctrlSeqs.MatchString(part) || // rule 4 + strings.ContainsAny(part, "~^:?*[ \t\n") || // rule 4 & 5 + strings.Contains(part, "@{") || // rule 8 + part == "@" || // rule 9 + strings.Contains(part, "\\") || // rule 10 + strings.HasSuffix(part, ".lock") { // rule 1 + return ErrInvalidReferenceName + } + + if (isBranch || isTag) && strings.HasPrefix(part, "-") { // branches & tags can't start with - + return ErrInvalidReferenceName + } + } + + return nil +} + const ( HEAD ReferenceName = "HEAD" Master ReferenceName = "refs/heads/master" diff --git a/plumbing/reference_test.go b/plumbing/reference_test.go index 04dfef93d..ce570752f 100644 --- a/plumbing/reference_test.go +++ b/plumbing/reference_test.go @@ -103,6 +103,65 @@ func (s *ReferenceSuite) TestIsTag(c *C) { c.Assert(r.IsTag(), Equals, true) } +func (s *ReferenceSuite) TestValidReferenceNames(c *C) { + valid := []ReferenceName{ + "refs/heads/master", + "refs/notes/commits", + "refs/remotes/origin/master", + "HEAD", + "refs/tags/v3.1.1", + "refs/pulls/1/head", + "refs/pulls/1/merge", + "refs/pulls/1/abc.123", + "refs/pulls", + "refs/-", // should this be allowed? + } + for _, v := range valid { + c.Assert(v.Validate(), IsNil) + } + + invalid := []ReferenceName{ + "refs", + "refs/", + "refs//", + "refs/heads/\\", + "refs/heads/\\foo", + "refs/heads/\\foo/bar", + "abc", + "", + "refs/heads/ ", + "refs/heads/ /", + "refs/heads/ /foo", + "refs/heads/.", + "refs/heads/..", + "refs/heads/foo..", + "refs/heads/foo.lock", + "refs/heads/foo@{bar}", + "refs/heads/foo[", + "refs/heads/foo~", + "refs/heads/foo^", + "refs/heads/foo:", + "refs/heads/foo?", + "refs/heads/foo*", + "refs/heads/foo[bar", + "refs/heads/foo\t", + "refs/heads/@", + "refs/heads/@{bar}", + "refs/heads/\n", + "refs/heads/-foo", + "refs/heads/foo..bar", + "refs/heads/-", + "refs/tags/-", + "refs/tags/-foo", + } + + for i, v := range invalid { + comment := Commentf("invalid reference name case %d: %s", i, v) + c.Assert(v.Validate(), NotNil, comment) + c.Assert(v.Validate(), ErrorMatches, "invalid reference name", comment) + } +} + func benchMarkReferenceString(r *Reference, b *testing.B) { for n := 0; n < b.N; n++ { _ = r.String() diff --git a/repository.go b/repository.go index 48988383d..1524a6913 100644 --- a/repository.go +++ b/repository.go @@ -98,6 +98,10 @@ func InitWithOptions(s storage.Storer, worktree billy.Filesystem, options InitOp options.DefaultBranch = plumbing.Master } + if err := options.DefaultBranch.Validate(); err != nil { + return nil, err + } + r := newRepository(s, worktree) _, err := r.Reference(plumbing.HEAD, false) switch err { @@ -724,7 +728,10 @@ func (r *Repository) DeleteBranch(name string) error { // CreateTag creates a tag. If opts is included, the tag is an annotated tag, // otherwise a lightweight tag is created. func (r *Repository) CreateTag(name string, hash plumbing.Hash, opts *CreateTagOptions) (*plumbing.Reference, error) { - rname := plumbing.ReferenceName(path.Join("refs", "tags", name)) + rname := plumbing.NewTagReferenceName(name) + if err := rname.Validate(); err != nil { + return nil, err + } _, err := r.Storer.Reference(rname) switch err { diff --git a/repository_test.go b/repository_test.go index 35a62f131..51df84512 100644 --- a/repository_test.go +++ b/repository_test.go @@ -75,6 +75,13 @@ func (s *RepositorySuite) TestInitWithOptions(c *C) { } +func (s *RepositorySuite) TestInitWithInvalidDefaultBranch(c *C) { + _, err := InitWithOptions(memory.NewStorage(), memfs.New(), InitOptions{ + DefaultBranch: "foo", + }) + c.Assert(err, NotNil) +} + func createCommit(c *C, r *Repository) { // Create a commit so there is a HEAD to check wt, err := r.Worktree() @@ -391,6 +398,22 @@ func (s *RepositorySuite) TestDeleteRemote(c *C) { c.Assert(alt, IsNil) } +func (s *RepositorySuite) TestEmptyCreateBranch(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.CreateBranch(&config.Branch{}) + + c.Assert(err, NotNil) +} + +func (s *RepositorySuite) TestInvalidCreateBranch(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.CreateBranch(&config.Branch{ + Name: "-foo", + }) + + c.Assert(err, NotNil) +} + func (s *RepositorySuite) TestCreateBranchAndBranch(c *C) { r, _ := Init(memory.NewStorage(), nil) testBranch := &config.Branch{ @@ -2797,6 +2820,20 @@ func (s *RepositorySuite) TestDeleteTagAnnotatedUnpacked(c *C) { c.Assert(err, Equals, plumbing.ErrObjectNotFound) } +func (s *RepositorySuite) TestInvalidTagName(c *C) { + r, err := Init(memory.NewStorage(), nil) + c.Assert(err, IsNil) + for i, name := range []string{ + "", + "foo bar", + "foo\tbar", + "foo\nbar", + } { + _, err = r.CreateTag(name, plumbing.ZeroHash, nil) + c.Assert(err, NotNil, Commentf("case %d %q", i, name)) + } +} + func (s *RepositorySuite) TestBranches(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/root-references.git").One() sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) diff --git a/worktree.go b/worktree.go index f8b854dda..51795e635 100644 --- a/worktree.go +++ b/worktree.go @@ -189,6 +189,10 @@ func (w *Worktree) Checkout(opts *CheckoutOptions) error { return w.Reset(ro) } func (w *Worktree) createBranch(opts *CheckoutOptions) error { + if err := opts.Branch.Validate(); err != nil { + return err + } + _, err := w.r.Storer.Reference(opts.Branch) if err == nil { return fmt.Errorf("a branch named %q already exists", opts.Branch) diff --git a/worktree_test.go b/worktree_test.go index 180bfb0b4..5c9a4eb3a 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -785,6 +785,30 @@ func (s *WorktreeSuite) TestCheckoutCreateMissingBranch(c *C) { c.Assert(err, Equals, ErrCreateRequiresBranch) } +func (s *WorktreeSuite) TestCheckoutCreateInvalidBranch(c *C) { + w := &Worktree{ + r: s.Repository, + Filesystem: memfs.New(), + } + + for _, name := range []plumbing.ReferenceName{ + "foo", + "-", + "-foo", + "refs/heads//", + "refs/heads/..", + "refs/heads/a..b", + "refs/heads/.", + } { + err := w.Checkout(&CheckoutOptions{ + Create: true, + Branch: name, + }) + + c.Assert(err, Equals, plumbing.ErrInvalidReferenceName) + } +} + func (s *WorktreeSuite) TestCheckoutTag(c *C) { f := fixtures.ByTag("tags").One() r := s.NewRepositoryWithEmptyWorktree(f) From 8b47ceb1aa854f3c3bfa1c347157a04324fcd51e Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Sat, 2 Dec 2023 10:30:30 +0000 Subject: [PATCH 54/56] storage: filesystem, Add option to set a specific FS for alternates Introduces the option to set a FS for alternates, enabling more flexible cross FS sharing of alternates. If none is set, falls back to the current FS used for the object storage. The changes only process a given path once, and if an alternates dir is not valid, exits with error - aligning behaviour with upstream. Signed-off-by: Paulo Gomes --- go.mod | 4 + go.sum | 1 + storage/filesystem/dotgit/dotgit.go | 64 ++++++--- storage/filesystem/dotgit/dotgit_test.go | 162 +++++++++++++++++------ storage/filesystem/storage.go | 5 + worktree_test.go | 31 +++-- 6 files changed, 203 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index 9a18c7a49..5247d063c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 github.com/sergi/go-diff v1.1.0 github.com/skeema/knownhosts v1.2.1 + github.com/stretchr/testify v1.8.4 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.16.0 golang.org/x/net v0.19.0 @@ -33,10 +34,13 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4be7ec4eb..c0593765b 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 3080e4acc..31c469481 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -10,18 +10,19 @@ import ( "os" "path" "path/filepath" + "reflect" "runtime" "sort" "strings" "time" - "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/utils/ioutil" "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/helper/chroot" ) const ( @@ -81,6 +82,10 @@ type Options struct { // KeepDescriptors makes the file descriptors to be reused but they will // need to be manually closed calling Close(). KeepDescriptors bool + // AlternatesFS provides the billy filesystem to be used for Git Alternates. + // If none is provided, it falls back to using the underlying instance used for + // DotGit. + AlternatesFS billy.Filesystem } // The DotGit type represents a local git repository on disk. This @@ -1146,28 +1151,55 @@ func (d *DotGit) Alternates() ([]*DotGit, error) { } defer f.Close() + fs := d.options.AlternatesFS + if fs == nil { + fs = d.fs + } + var alternates []*DotGit + seen := make(map[string]struct{}) // Read alternate paths line-by-line and create DotGit objects. scanner := bufio.NewScanner(f) for scanner.Scan() { path := scanner.Text() - if !filepath.IsAbs(path) { - // For relative paths, we can perform an internal conversion to - // slash so that they work cross-platform. - slashPath := filepath.ToSlash(path) - // If the path is not absolute, it must be relative to object - // database (.git/objects/info). - // https://www.kernel.org/pub/software/scm/git/docs/gitrepository-layout.html - // Hence, derive a path relative to DotGit's root. - // "../../../reponame/.git/" -> "../../reponame/.git" - // Remove the first ../ - relpath := filepath.Join(strings.Split(slashPath, "/")[1:]...) - normalPath := filepath.FromSlash(relpath) - path = filepath.Join(d.fs.Root(), normalPath) + + // Avoid creating multiple dotgits for the same alternative path. + if _, ok := seen[path]; ok { + continue + } + + seen[path] = struct{}{} + + if filepath.IsAbs(path) { + // Handling absolute paths should be straight-forward. However, the default osfs (Chroot) + // tries to concatenate an abs path with the root path in some operations (e.g. Stat), + // which leads to unexpected errors. Therefore, make the path relative to the current FS instead. + if reflect.TypeOf(fs) == reflect.TypeOf(&chroot.ChrootHelper{}) { + path, err = filepath.Rel(fs.Root(), path) + if err != nil { + return nil, fmt.Errorf("cannot make path %q relative: %w", path, err) + } + } + } else { + // By Git conventions, relative paths should be based on the object database (.git/objects/info) + // location as per: https://www.kernel.org/pub/software/scm/git/docs/gitrepository-layout.html + // However, due to the nature of go-git and its filesystem handling via Billy, paths cannot + // cross its "chroot boundaries". Therefore, ignore any "../" and treat the path from the + // fs root. If this is not correct based on the dotgit fs, set a different one via AlternatesFS. + abs := filepath.Join(string(filepath.Separator), filepath.ToSlash(path)) + path = filepath.FromSlash(abs) + } + + // Aligns with upstream behavior: exit if target path is not a valid directory. + if fi, err := fs.Stat(path); err != nil || !fi.IsDir() { + return nil, fmt.Errorf("invalid object directory %q: %w", path, err) + } + afs, err := fs.Chroot(filepath.Dir(path)) + if err != nil { + return nil, fmt.Errorf("cannot chroot %q: %w", path, err) } - fs := osfs.New(filepath.Dir(path)) - alternates = append(alternates, New(fs)) + alternates = append(alternates, New(afs)) } if err = scanner.Err(); err != nil { diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 1b6c11351..2cbdb0c9f 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "regexp" "runtime" "strings" "testing" @@ -15,6 +16,7 @@ import ( "github.com/go-git/go-billy/v5/util" fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" . "gopkg.in/check.v1" ) @@ -810,53 +812,139 @@ func (s *SuiteDotGit) TestPackRefs(c *C) { c.Assert(ref.Hash().String(), Equals, "b8d3ffab552895c19b9fcf7aa264d277cde33881") } -func (s *SuiteDotGit) TestAlternates(c *C) { - fs, clean := s.TemporalFilesystem() - defer clean() +func TestAlternatesDefault(t *testing.T) { + // Create a new dotgit object. + dotFS := osfs.New(t.TempDir()) - // Create a new dotgit object and initialize. - dir := New(fs) - err := dir.Initialize() - c.Assert(err, IsNil) + testAlternates(t, dotFS, dotFS) +} - // Create alternates file. - altpath := fs.Join("objects", "info", "alternates") - f, err := fs.Create(altpath) - c.Assert(err, IsNil) +func TestAlternatesWithFS(t *testing.T) { + // Create a new dotgit object with a specific FS for alternates. + altFS := osfs.New(t.TempDir()) + dotFS, _ := altFS.Chroot("repo2") - // Multiple alternates. - var strContent string - if runtime.GOOS == "windows" { - strContent = "C:\\Users\\username\\repo1\\.git\\objects\r\n..\\..\\..\\rep2\\.git\\objects" - } else { - strContent = "/Users/username/rep1//.git/objects\n../../../rep2//.git/objects" + testAlternates(t, dotFS, altFS) +} + +func TestAlternatesWithBoundOS(t *testing.T) { + // Create a new dotgit object with a specific FS for alternates. + altFS := osfs.New(t.TempDir(), osfs.WithBoundOS()) + dotFS, _ := altFS.Chroot("repo2") + + testAlternates(t, dotFS, altFS) +} + +func testAlternates(t *testing.T, dotFS, altFS billy.Filesystem) { + tests := []struct { + name string + in []string + inWindows []string + setup func() + wantErr bool + wantRoots []string + }{ + { + name: "no alternates", + }, + { + name: "abs path", + in: []string{filepath.Join(altFS.Root(), "./repo1/.git/objects")}, + inWindows: []string{filepath.Join(altFS.Root(), ".\\repo1\\.git\\objects")}, + setup: func() { + err := altFS.MkdirAll(filepath.Join("repo1", ".git", "objects"), 0o700) + assert.NoError(t, err) + }, + wantRoots: []string{filepath.Join("repo1", ".git")}, + }, + { + name: "rel path", + in: []string{"../../../repo3//.git/objects"}, + inWindows: []string{"..\\..\\..\\repo3\\.git\\objects"}, + setup: func() { + err := altFS.MkdirAll(filepath.Join("repo3", ".git", "objects"), 0o700) + assert.NoError(t, err) + }, + wantRoots: []string{filepath.Join("repo3", ".git")}, + }, + { + name: "invalid abs path", + in: []string{"/alt/target2"}, + inWindows: []string{"\\alt\\target2"}, + wantErr: true, + }, + { + name: "invalid rel path", + in: []string{"../../../alt/target3"}, + inWindows: []string{"..\\..\\..\\alt\\target3"}, + wantErr: true, + }, } - content := []byte(strContent) - f.Write(content) - f.Close() - dotgits, err := dir.Alternates() - c.Assert(err, IsNil) - if runtime.GOOS == "windows" { - c.Assert(dotgits[0].fs.Root(), Equals, "C:\\Users\\username\\repo1\\.git") - } else { - c.Assert(dotgits[0].fs.Root(), Equals, "/Users/username/rep1/.git") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := NewWithOptions(dotFS, Options{AlternatesFS: altFS}) + err := dir.Initialize() + assert.NoError(t, err) + + content := strings.Join(tc.in, "\n") + if runtime.GOOS == "windows" { + content = strings.Join(tc.inWindows, "\r\n") + } + + // Create alternates file. + altpath := dotFS.Join("objects", "info", "alternates") + f, err := dotFS.Create(altpath) + assert.NoError(t, err) + f.Write([]byte(content)) + f.Close() + + if tc.setup != nil { + tc.setup() + } + + dotgits, err := dir.Alternates() + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + for i, d := range dotgits { + assert.Regexp(t, "^"+regexp.QuoteMeta(altFS.Root()), d.fs.Root()) + assert.Regexp(t, regexp.QuoteMeta(tc.wantRoots[i])+"$", d.fs.Root()) + } + }) } +} - // For relative path: - // /some/absolute/path/to/dot-git -> /some/absolute/path - pathx := strings.Split(fs.Root(), string(filepath.Separator)) - pathx = pathx[:len(pathx)-2] - // Use string.Join() to avoid malformed absolutepath on windows - // C:Users\\User\\... instead of C:\\Users\\appveyor\\... . - resolvedPath := strings.Join(pathx, string(filepath.Separator)) - // Append the alternate path to the resolvedPath - expectedPath := fs.Join(string(filepath.Separator), resolvedPath, "rep2", ".git") +func TestAlternatesDupes(t *testing.T) { + dotFS := osfs.New(t.TempDir()) + dir := New(dotFS) + err := dir.Initialize() + assert.NoError(t, err) + + path := filepath.Join(dotFS.Root(), "target3") + dupes := []string{path, path, path, path, path} + + content := strings.Join(dupes, "\n") if runtime.GOOS == "windows" { - expectedPath = fs.Join(resolvedPath, "rep2", ".git") + content = strings.Join(dupes, "\r\n") } - c.Assert(dotgits[1].fs.Root(), Equals, expectedPath) + err = dotFS.MkdirAll("target3", 0o700) + assert.NoError(t, err) + + // Create alternates file. + altpath := dotFS.Join("objects", "info", "alternates") + f, err := dotFS.Create(altpath) + assert.NoError(t, err) + f.Write([]byte(content)) + f.Close() + + dotgits, err := dir.Alternates() + assert.NoError(t, err) + assert.Len(t, dotgits, 1) } type norwfs struct { diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 2069d3a6f..951ea00c8 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -37,6 +37,10 @@ type Options struct { // LargeObjectThreshold maximum object size (in bytes) that will be read in to memory. // If left unset or set to 0 there is no limit LargeObjectThreshold int64 + // AlternatesFS provides the billy filesystem to be used for Git Alternates. + // If none is provided, it falls back to using the underlying instance used for + // DotGit. + AlternatesFS billy.Filesystem } // NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. @@ -49,6 +53,7 @@ func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { dirOps := dotgit.Options{ ExclusiveAccess: ops.ExclusiveAccess, + AlternatesFS: ops.AlternatesFS, } dir := dotgit.NewWithOptions(fs, dirOps) diff --git a/worktree_test.go b/worktree_test.go index 3136e5988..3e057a79d 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -16,11 +16,14 @@ import ( fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/go-git/go-git/v5/plumbing/format/index" "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/filesystem" "github.com/go-git/go-git/v5/storage/memory" + "github.com/stretchr/testify/assert" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/osfs" @@ -2198,34 +2201,40 @@ func (s *WorktreeSuite) TestCleanBare(c *C) { c.Assert(err, IsNil) } -func (s *WorktreeSuite) TestAlternatesRepo(c *C) { +func TestAlternatesRepo(t *testing.T) { fs := fixtures.ByTag("alternates").One().Worktree() // Open 1st repo. rep1fs, err := fs.Chroot("rep1") - c.Assert(err, IsNil) + assert.NoError(t, err) rep1, err := PlainOpen(rep1fs.Root()) - c.Assert(err, IsNil) + assert.NoError(t, err) // Open 2nd repo. rep2fs, err := fs.Chroot("rep2") - c.Assert(err, IsNil) - rep2, err := PlainOpen(rep2fs.Root()) - c.Assert(err, IsNil) + assert.NoError(t, err) + d, _ := rep2fs.Chroot(GitDirName) + storer := filesystem.NewStorageWithOptions(d, + cache.NewObjectLRUDefault(), filesystem.Options{ + AlternatesFS: fs, + }) + rep2, err := Open(storer, rep2fs) + + assert.NoError(t, err) // Get the HEAD commit from the main repo. h, err := rep1.Head() - c.Assert(err, IsNil) + assert.NoError(t, err) commit1, err := rep1.CommitObject(h.Hash()) - c.Assert(err, IsNil) + assert.NoError(t, err) // Get the HEAD commit from the shared repo. h, err = rep2.Head() - c.Assert(err, IsNil) + assert.NoError(t, err) commit2, err := rep2.CommitObject(h.Hash()) - c.Assert(err, IsNil) + assert.NoError(t, err) - c.Assert(commit1.String(), Equals, commit2.String()) + assert.Equal(t, commit1.String(), commit2.String()) } func (s *WorktreeSuite) TestGrep(c *C) { From b2c19824771bbcbb21abb51abb319c1a610aa6b3 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 8 Dec 2023 08:55:30 +0000 Subject: [PATCH 55/56] git: worktree, Align validation with upstream rules Some worktree validation rules observed upstream are not checked by go-git, leading to scenarios in which what seems to be a valid repository for go-git is not necessarily the case for the git cli. Signed-off-by: Paulo Gomes --- worktree.go | 107 +++++++++++++++++++++++++++++++++++++++++++++++ worktree_test.go | 75 +++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/worktree.go b/worktree.go index 66cb3d29c..ad525c1a4 100644 --- a/worktree.go +++ b/worktree.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "github.com/go-git/go-billy/v5" @@ -394,6 +395,9 @@ func (w *Worktree) resetWorktree(t *object.Tree) error { b := newIndexBuilder(idx) for _, ch := range changes { + if err := w.validChange(ch); err != nil { + return err + } if err := w.checkoutChange(ch, t, b); err != nil { return err } @@ -403,6 +407,104 @@ func (w *Worktree) resetWorktree(t *object.Tree) error { return w.r.Storer.SetIndex(idx) } +// worktreeDeny is a list of paths that are not allowed +// to be used when resetting the worktree. +var worktreeDeny = map[string]struct{}{ + // .git + GitDirName: {}, + + // For other historical reasons, file names that do not conform to the 8.3 + // format (up to eight characters for the basename, three for the file + // extension, certain characters not allowed such as `+`, etc) are associated + // with a so-called "short name", at least on the `C:` drive by default. + // Which means that `git~1/` is a valid way to refer to `.git/`. + "git~1": {}, +} + +// validPath checks whether paths are valid. +// The rules around invalid paths could differ from upstream based on how +// filesystems are managed within go-git, but they are largely the same. +// +// For upstream rules: +// https://github.com/git/git/blob/564d0252ca632e0264ed670534a51d18a689ef5d/read-cache.c#L946 +// https://github.com/git/git/blob/564d0252ca632e0264ed670534a51d18a689ef5d/path.c#L1383 +func validPath(paths ...string) error { + for _, p := range paths { + parts := strings.FieldsFunc(p, func(r rune) bool { return (r == '\\' || r == '/') }) + if _, denied := worktreeDeny[strings.ToLower(parts[0])]; denied { + return fmt.Errorf("invalid path prefix: %q", p) + } + + if runtime.GOOS == "windows" { + // Volume names are not supported, in both formats: \\ and :. + if vol := filepath.VolumeName(p); vol != "" { + return fmt.Errorf("invalid path: %q", p) + } + + if !windowsValidPath(parts[0]) { + return fmt.Errorf("invalid path: %q", p) + } + } + + for _, part := range parts { + if part == ".." { + return fmt.Errorf("invalid path %q: cannot use '..'", p) + } + } + } + return nil +} + +// windowsPathReplacer defines the chars that need to be replaced +// as part of windowsValidPath. +var windowsPathReplacer *strings.Replacer + +func init() { + windowsPathReplacer = strings.NewReplacer(" ", "", ".", "") +} + +func windowsValidPath(part string) bool { + if len(part) > 3 && strings.EqualFold(part[:4], GitDirName) { + // For historical reasons, file names that end in spaces or periods are + // automatically trimmed. Therefore, `.git . . ./` is a valid way to refer + // to `.git/`. + if windowsPathReplacer.Replace(part[4:]) == "" { + return false + } + + // For yet other historical reasons, NTFS supports so-called "Alternate Data + // Streams", i.e. metadata associated with a given file, referred to via + // `::`. There exists a default stream + // type for directories, allowing `.git/` to be accessed via + // `.git::$INDEX_ALLOCATION/`. + // + // For performance reasons, _all_ Alternate Data Streams of `.git/` are + // forbidden, not just `::$INDEX_ALLOCATION`. + if len(part) > 4 && part[4:5] == ":" { + return false + } + } + return true +} + +func (w *Worktree) validChange(ch merkletrie.Change) error { + action, err := ch.Action() + if err != nil { + return nil + } + + switch action { + case merkletrie.Delete: + return validPath(ch.From.String()) + case merkletrie.Insert: + return validPath(ch.To.String()) + case merkletrie.Modify: + return validPath(ch.From.String(), ch.To.String()) + } + + return nil +} + func (w *Worktree) checkoutChange(ch merkletrie.Change, t *object.Tree, idx *indexBuilder) error { a, err := ch.Action() if err != nil { @@ -575,6 +677,11 @@ func (w *Worktree) checkoutFile(f *object.File) (err error) { } func (w *Worktree) checkoutFileSymlink(f *object.File) (err error) { + // https://github.com/git/git/commit/10ecfa76491e4923988337b2e2243b05376b40de + if strings.EqualFold(f.Name, gitmodulesFile) { + return ErrGitModulesSymlink + } + from, err := f.Reader() if err != nil { return diff --git a/worktree_test.go b/worktree_test.go index 3e057a79d..50ff189fa 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "os" "path/filepath" @@ -2747,3 +2748,77 @@ func (s *WorktreeSuite) TestLinkedWorktree(c *C) { c.Assert(err, Equals, ErrRepositoryIncomplete) } } + +func TestValidPath(t *testing.T) { + type testcase struct { + path string + wantErr bool + } + + tests := []testcase{ + {".git", true}, + {".git/b", true}, + {".git\\b", true}, + {"git~1", true}, + {"a/../b", true}, + {"a\\..\\b", true}, + {".gitmodules", false}, + {".gitignore", false}, + {"a..b", false}, + {".", false}, + {"a/.git", false}, + {"a\\.git", false}, + {"a/.git/b", false}, + {"a\\.git\\b", false}, + } + + if runtime.GOOS == "windows" { + tests = append(tests, []testcase{ + {"\\\\a\\b", true}, + {"C:\\a\\b", true}, + {".git . . .", true}, + {".git . . ", true}, + {".git ", true}, + {".git.", true}, + {".git::$INDEX_ALLOCATION", true}, + }...) + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("%s", tc.path), func(t *testing.T) { + err := validPath(tc.path) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestWindowsValidPath(t *testing.T) { + tests := []struct { + path string + want bool + }{ + {".git", false}, + {".git . . .", false}, + {".git ", false}, + {".git ", false}, + {".git . .", false}, + {".git . .", false}, + {".git::$INDEX_ALLOCATION", false}, + {".git:", false}, + {"a", true}, + {"a\\b", true}, + {"a/b", true}, + {".gitm", true}, + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("%s", tc.path), func(t *testing.T) { + got := windowsValidPath(tc.path) + assert.Equal(t, tc.want, got) + }) + } +} From 5bd1d8f4abcfbf1345a1e5a5ec9a96121f3746dc Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 8 Dec 2023 09:04:16 +0000 Subject: [PATCH 56/56] build: Ensure checkout is the first operation The setup-go step can be sped up by caching Go dependencies. The input for that operation is the go.sum file. Previously, the checkout operation was happening after the setup-go, which meant that go.sum was never available which effectively meant the cache was disabled. Signed-off-by: Paulo Gomes --- .github/workflows/git.yml | 6 +++--- .github/workflows/test.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index 95398087c..d46cd3b0e 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -16,14 +16,14 @@ jobs: GIT_DIST_PATH: .git-dist/${{ matrix.git[0] }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Go uses: actions/setup-go@v4 with: go-version: 1.21.x - - name: Checkout code - uses: actions/checkout@v4 - - name: Install build dependencies run: sudo apt-get update && sudo apt-get install gettext libcurl4-openssl-dev diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 875dda7be..378000871 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,13 +13,13 @@ jobs: runs-on: ${{ matrix.platform }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Go uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} - - - name: Checkout code - uses: actions/checkout@v4 - name: Configure known hosts if: matrix.platform != 'ubuntu-latest'