From 02494219682689ccfae6d4ffed2734509eed02d0 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 7 Mar 2023 22:33:55 +0000 Subject: [PATCH 01/31] plumbing/hash: Add SHA256 as a supported algorithm Relates to #706. Signed-off-by: Paulo Gomes --- plumbing/hash/hash.go | 3 +++ plumbing/hash/hash_sha1.go | 15 +++++++++++++++ plumbing/hash/hash_sha256.go | 15 +++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 plumbing/hash/hash_sha1.go create mode 100644 plumbing/hash/hash_sha256.go diff --git a/plumbing/hash/hash.go b/plumbing/hash/hash.go index 80e4b5f25..82d185616 100644 --- a/plumbing/hash/hash.go +++ b/plumbing/hash/hash.go @@ -21,6 +21,7 @@ func init() { // that registers new algorithms to avoid side effects. func reset() { algos[crypto.SHA1] = sha1cd.New + algos[crypto.SHA256] = crypto.SHA256.New } // RegisterHash allows for the hash algorithm used to be overriden. @@ -34,6 +35,8 @@ func RegisterHash(h crypto.Hash, f func() hash.Hash) error { switch h { case crypto.SHA1: algos[h] = f + case crypto.SHA256: + algos[h] = f default: return fmt.Errorf("unsupported hash function: %v", h) } diff --git a/plumbing/hash/hash_sha1.go b/plumbing/hash/hash_sha1.go new file mode 100644 index 000000000..e3cb60fec --- /dev/null +++ b/plumbing/hash/hash_sha1.go @@ -0,0 +1,15 @@ +//go:build !sha256 +// +build !sha256 + +package hash + +import "crypto" + +const ( + // CryptoType defines what hash algorithm is being used. + CryptoType = crypto.SHA1 + // Size defines the amount of bytes the hash yields. + Size = 20 + // HexSize defines the strings size of the hash when represented in hexadecimal. + HexSize = 40 +) diff --git a/plumbing/hash/hash_sha256.go b/plumbing/hash/hash_sha256.go new file mode 100644 index 000000000..1c52b8975 --- /dev/null +++ b/plumbing/hash/hash_sha256.go @@ -0,0 +1,15 @@ +//go:build sha256 +// +build sha256 + +package hash + +import "crypto" + +const ( + // CryptoType defines what hash algorithm is being used. + CryptoType = crypto.SHA256 + // Size defines the amount of bytes the hash yields. + Size = 32 + // HexSize defines the strings size of the hash when represented in hexadecimal. + HexSize = 64 +) From 99e2f85843878671b028d4d01bd4668676226dd1 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 7 Mar 2023 23:16:17 +0000 Subject: [PATCH 02/31] config: Add Repository Format Extension Relates to the SHA256 implementation, defined in #706. Signed-off-by: Paulo Gomes --- config/config.go | 73 ++++++++++++++++++++++---------- options.go | 8 ++++ plumbing/format/config/format.go | 53 +++++++++++++++++++++++ repository.go | 37 ++++++++++++++++ 4 files changed, 149 insertions(+), 22 deletions(-) create mode 100644 plumbing/format/config/format.go diff --git a/config/config.go b/config/config.go index 8051bc145..83629fcef 100644 --- a/config/config.go +++ b/config/config.go @@ -59,6 +59,8 @@ type Config struct { // CommentChar is the character indicating the start of a // comment for commands like commit and tag CommentChar string + // RepositoryFormatVersion identifies the repository format and layout version. + RepositoryFormatVersion format.RepositoryFormatVersion } User struct { @@ -96,6 +98,17 @@ type Config struct { DefaultBranch string } + Extensions struct { + // ObjectFormat specifies the hash algorithm to use. The + // acceptable values are sha1 and sha256. If not specified, + // sha1 is assumed. It is an error to specify this key unless + // core.repositoryFormatVersion is 1. + // + // This setting must not be changed after repository initialization + // (e.g. clone or init). + ObjectFormat format.ObjectFormat + } + // Remotes list of repository remotes, the key of the map is the name // of the remote, should equal to RemoteConfig.Name. Remotes map[string]*RemoteConfig @@ -226,28 +239,31 @@ func (c *Config) Validate() error { } const ( - remoteSection = "remote" - submoduleSection = "submodule" - branchSection = "branch" - coreSection = "core" - packSection = "pack" - userSection = "user" - authorSection = "author" - committerSection = "committer" - initSection = "init" - urlSection = "url" - fetchKey = "fetch" - urlKey = "url" - bareKey = "bare" - worktreeKey = "worktree" - commentCharKey = "commentChar" - windowKey = "window" - mergeKey = "merge" - rebaseKey = "rebase" - nameKey = "name" - emailKey = "email" - descriptionKey = "description" - defaultBranchKey = "defaultBranch" + remoteSection = "remote" + submoduleSection = "submodule" + branchSection = "branch" + coreSection = "core" + packSection = "pack" + userSection = "user" + authorSection = "author" + committerSection = "committer" + initSection = "init" + urlSection = "url" + extensionsSection = "extensions" + fetchKey = "fetch" + urlKey = "url" + bareKey = "bare" + worktreeKey = "worktree" + commentCharKey = "commentChar" + windowKey = "window" + mergeKey = "merge" + rebaseKey = "rebase" + nameKey = "name" + emailKey = "email" + descriptionKey = "description" + defaultBranchKey = "defaultBranch" + repositoryFormatVersionKey = "repositoryformatversion" + objectFormat = "objectformat" // DefaultPackWindow holds the number of previous objects used to // generate deltas. The value 10 is the same used by git command. @@ -391,6 +407,7 @@ func (c *Config) unmarshalInit() { // Marshal returns Config encoded as a git-config file. func (c *Config) Marshal() ([]byte, error) { c.marshalCore() + c.marshalExtensions() c.marshalUser() c.marshalPack() c.marshalRemotes() @@ -410,12 +427,24 @@ func (c *Config) Marshal() ([]byte, error) { func (c *Config) marshalCore() { s := c.Raw.Section(coreSection) s.SetOption(bareKey, fmt.Sprintf("%t", c.Core.IsBare)) + if string(c.Core.RepositoryFormatVersion) != "" { + s.SetOption(repositoryFormatVersionKey, string(c.Core.RepositoryFormatVersion)) + } if c.Core.Worktree != "" { s.SetOption(worktreeKey, c.Core.Worktree) } } +func (c *Config) marshalExtensions() { + // Extensions are only supported on Version 1, therefore + // ignore them otherwise. + if c.Core.RepositoryFormatVersion == format.Version_1 { + s := c.Raw.Section(extensionsSection) + s.SetOption(objectFormat, string(c.Extensions.ObjectFormat)) + } +} + func (c *Config) marshalUser() { s := c.Raw.Section(userSection) if c.User.Name != "" { diff --git a/options.go b/options.go index 747d512bc..738c1ce9c 100644 --- a/options.go +++ b/options.go @@ -10,6 +10,7 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + formatcfg "github.com/go-git/go-git/v5/plumbing/format/config" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband" "github.com/go-git/go-git/v5/plumbing/transport" @@ -672,3 +673,10 @@ type PlainOpenOptions struct { // Validate validates the fields and sets the default values. func (o *PlainOpenOptions) Validate() error { return nil } + +type PlainInitOptions struct { + ObjectFormat formatcfg.ObjectFormat +} + +// Validate validates the fields and sets the default values. +func (o *PlainInitOptions) Validate() error { return nil } diff --git a/plumbing/format/config/format.go b/plumbing/format/config/format.go new file mode 100644 index 000000000..4873ea925 --- /dev/null +++ b/plumbing/format/config/format.go @@ -0,0 +1,53 @@ +package config + +// RepositoryFormatVersion represents the repository format version, +// as per defined at: +// +// https://git-scm.com/docs/repository-version +type RepositoryFormatVersion string + +const ( + // Version_0 is the format defined by the initial version of git, + // including but not limited to the format of the repository + // directory, the repository configuration file, and the object + // and ref storage. + // + // Specifying the complete behavior of git is beyond the scope + // of this document. + Version_0 = "0" + + // Version_1 is identical to version 0, with the following exceptions: + // + // 1. When reading the core.repositoryformatversion variable, a git + // implementation which supports version 1 MUST also read any + // configuration keys found in the extensions section of the + // configuration file. + // + // 2. If a version-1 repository specifies any extensions.* keys that + // the running git has not implemented, the operation MUST NOT proceed. + // Similarly, if the value of any known key is not understood by the + // implementation, the operation MUST NOT proceed. + // + // Note that if no extensions are specified in the config file, then + // core.repositoryformatversion SHOULD be set to 0 (setting it to 1 provides + // no benefit, and makes the repository incompatible with older + // implementations of git). + Version_1 = "1" + + // DefaultRepositoryFormatVersion holds the default repository format version. + DefaultRepositoryFormatVersion = Version_0 +) + +// ObjectFormat defines the object format. +type ObjectFormat string + +const ( + // SHA1 represents the object format used for SHA1. + SHA1 ObjectFormat = "sha1" + + // SHA256 represents the object format used for SHA256. + SHA256 ObjectFormat = "sha256" + + // DefaultObjectFormat holds the default object format. + DefaultObjectFormat = SHA1 +) diff --git a/repository.go b/repository.go index 2a06f8be3..56ae9761c 100644 --- a/repository.go +++ b/repository.go @@ -3,6 +3,7 @@ package git import ( "bytes" "context" + "crypto" "encoding/hex" "errors" "fmt" @@ -21,7 +22,9 @@ import ( "github.com/go-git/go-git/v5/internal/revision" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" + formatcfg "github.com/go-git/go-git/v5/plumbing/format/config" "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/storage" @@ -57,6 +60,7 @@ var ( ErrIsBareRepository = errors.New("worktree not available in a bare repository") ErrUnableToResolveCommit = errors.New("unable to resolve commit") ErrPackedObjectsNotSupported = errors.New("packed objects not supported") + ErrSHA256NotSupported = errors.New("go-git was not compiled with SHA256 support") ) // Repository represents a git repository @@ -228,6 +232,39 @@ func PlainInit(path string, isBare bool) (*Repository, error) { return Init(s, wt) } +func PlainInitWithOptions(path string, opts *PlainInitOptions) (*Repository, error) { + wt := osfs.New(path) + dot, _ := wt.Chroot(GitDirName) + + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) + + r, err := Init(s, wt) + if err != nil { + return nil, err + } + + cfg, err := r.Config() + if err != nil { + return nil, err + } + + if opts != nil { + if opts.ObjectFormat == formatcfg.SHA256 && hash.CryptoType != crypto.SHA256 { + return nil, ErrSHA256NotSupported + } + + cfg.Core.RepositoryFormatVersion = formatcfg.Version_1 + cfg.Extensions.ObjectFormat = opts.ObjectFormat + } + + err = r.Storer.SetConfig(cfg) + if err != nil { + return nil, err + } + + return r, err +} + // PlainOpen opens a git repository from the given path. It detects if the // repository is bare or a normal one. If the path doesn't contain a valid // repository ErrRepositoryNotExists is returned From 9822ad8573e374421a79c096d8f1dfa91366fb02 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 7 Mar 2023 23:31:29 +0000 Subject: [PATCH 03/31] *: Support variable length plumbing.Hash The variable length for plumbing.Hash is defined at build time, blocked by tag sha256. This approach was a trade-off between keeping backwards compatibility while making progress towards supporting SHA256 with a small amount of changes. Relates to the SHA256 implementation, defined in #706. Signed-off-by: Paulo Gomes --- plumbing/format/commitgraph/encoder.go | 7 +++---- plumbing/format/idxfile/decoder.go | 3 ++- plumbing/format/idxfile/encoder.go | 7 +++---- plumbing/format/idxfile/idxfile.go | 5 +++-- plumbing/format/index/decoder.go | 3 +-- plumbing/format/index/encoder.go | 3 +-- plumbing/format/packfile/encoder.go | 3 +-- plumbing/format/packfile/encoder_test.go | 9 +++++---- plumbing/format/packfile/scanner_test.go | 3 ++- plumbing/hash.go | 14 +++++++------- plumbing/protocol/packp/ulreq_decode_test.go | 5 +++-- storage/filesystem/dotgit/dotgit.go | 12 +++++++----- storage/filesystem/dotgit/writers.go | 5 +++-- 13 files changed, 41 insertions(+), 38 deletions(-) diff --git a/plumbing/format/commitgraph/encoder.go b/plumbing/format/commitgraph/encoder.go index bcf7d030a..f61025bb4 100644 --- a/plumbing/format/commitgraph/encoder.go +++ b/plumbing/format/commitgraph/encoder.go @@ -1,7 +1,6 @@ package commitgraph import ( - "crypto" "io" "github.com/go-git/go-git/v5/plumbing" @@ -17,7 +16,7 @@ type Encoder struct { // NewEncoder returns a new stream encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := hash.New(crypto.SHA1) + h := hash.New(hash.CryptoType) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } @@ -31,7 +30,7 @@ func (e *Encoder) Encode(idx Index) error { hashToIndex, fanout, extraEdgesCount := e.prepare(idx, hashes) chunkSignatures := [][]byte{oidFanoutSignature, oidLookupSignature, commitDataSignature} - chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * 20, uint64(len(hashes)) * 36} + chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * 36} if extraEdgesCount > 0 { chunkSignatures = append(chunkSignatures, extraEdgeListSignature) chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*4) @@ -183,6 +182,6 @@ func (e *Encoder) encodeExtraEdges(extraEdges []uint32) (err error) { } func (e *Encoder) encodeChecksum() error { - _, err := e.Write(e.hash.Sum(nil)[:20]) + _, err := e.Write(e.hash.Sum(nil)[:hash.Size]) return err } diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index 51a390489..9afdce301 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -6,6 +6,7 @@ import ( "errors" "io" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -19,7 +20,7 @@ var ( const ( fanout = 256 - objectIDLength = 20 + objectIDLength = hash.Size ) // Decoder reads and decodes idx files from an input stream. diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go index 6ac445ff6..75147376b 100644 --- a/plumbing/format/idxfile/encoder.go +++ b/plumbing/format/idxfile/encoder.go @@ -1,7 +1,6 @@ package idxfile import ( - "crypto" "io" "github.com/go-git/go-git/v5/plumbing/hash" @@ -16,7 +15,7 @@ type Encoder struct { // NewEncoder returns a new stream encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := hash.New(crypto.SHA1) + h := hash.New(hash.CryptoType) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } @@ -133,10 +132,10 @@ func (e *Encoder) encodeChecksums(idx *MemoryIndex) (int, error) { return 0, err } - copy(idx.IdxChecksum[:], e.hash.Sum(nil)[:20]) + copy(idx.IdxChecksum[:], e.hash.Sum(nil)[:hash.Size]) if _, err := e.Write(idx.IdxChecksum[:]); err != nil { return 0, err } - return 40, nil + return hash.HexSize, nil } diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index 64dd8dcef..9237a7434 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -8,6 +8,7 @@ import ( encbin "encoding/binary" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" ) const ( @@ -53,8 +54,8 @@ type MemoryIndex struct { Offset32 [][]byte CRC32 [][]byte Offset64 []byte - PackfileChecksum [20]byte - IdxChecksum [20]byte + PackfileChecksum [hash.Size]byte + IdxChecksum [hash.Size]byte offsetHash map[int64]plumbing.Hash offsetHashIsFull bool diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index c4da20c08..8834e2517 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -3,7 +3,6 @@ package index import ( "bufio" "bytes" - "crypto" "errors" "io" "io/ioutil" @@ -49,7 +48,7 @@ type Decoder struct { // NewDecoder returns a new decoder that reads from r. func NewDecoder(r io.Reader) *Decoder { - h := hash.New(crypto.SHA1) + h := hash.New(hash.CryptoType) return &Decoder{ r: io.TeeReader(r, h), hash: h, diff --git a/plumbing/format/index/encoder.go b/plumbing/format/index/encoder.go index a91537870..fa2d81445 100644 --- a/plumbing/format/index/encoder.go +++ b/plumbing/format/index/encoder.go @@ -2,7 +2,6 @@ package index import ( "bytes" - "crypto" "errors" "io" "sort" @@ -29,7 +28,7 @@ type Encoder struct { // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := hash.New(crypto.SHA1) + h := hash.New(hash.CryptoType) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index a8a7e967b..c9d19b332 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -2,7 +2,6 @@ package packfile import ( "compress/zlib" - "crypto" "fmt" "io" @@ -29,7 +28,7 @@ type Encoder struct { // OFSDeltaObject. To use Reference deltas, set useRefDeltas to true. func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *Encoder { h := plumbing.Hasher{ - Hash: hash.New(crypto.SHA1), + Hash: hash.New(hash.CryptoType), } mw := io.MultiWriter(w, h) ow := newOffsetWriter(mw) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index c9d49c3b5..902d8ccaa 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/idxfile" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-billy/v5/memfs" @@ -30,10 +31,10 @@ func (s *EncoderSuite) SetUpTest(c *C) { } func (s *EncoderSuite) TestCorrectPackHeader(c *C) { - hash, err := s.enc.Encode([]plumbing.Hash{}, 10) + h, err := s.enc.Encode([]plumbing.Hash{}, 10) c.Assert(err, IsNil) - hb := [20]byte(hash) + hb := [hash.Size]byte(h) // PACK + VERSION + OBJECTS + HASH expectedResult := []byte{'P', 'A', 'C', 'K', 0, 0, 0, 2, 0, 0, 0, 0} @@ -51,7 +52,7 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { _, err := s.store.SetEncodedObject(o) c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) + h, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) c.Assert(err, IsNil) // PACK + VERSION(2) + OBJECT NUMBER(1) @@ -64,7 +65,7 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { []byte{120, 156, 1, 0, 0, 255, 255, 0, 0, 0, 1}...) // + HASH - hb := [20]byte(hash) + hb := [hash.Size]byte(h) expectedResult = append(expectedResult, hb[:]...) result := s.buf.Bytes() diff --git a/plumbing/format/packfile/scanner_test.go b/plumbing/format/packfile/scanner_test.go index 892a27ca0..9dcc3594d 100644 --- a/plumbing/format/packfile/scanner_test.go +++ b/plumbing/format/packfile/scanner_test.go @@ -6,6 +6,7 @@ import ( fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" . "gopkg.in/check.v1" ) @@ -71,7 +72,7 @@ func (s *ScannerSuite) testNextObjectHeader(c *C, tag string, n, err := p.Checksum() c.Assert(err, IsNil) - c.Assert(n, HasLen, 20) + c.Assert(n, HasLen, hash.Size) } func (s *ScannerSuite) TestNextObjectHeaderWithOutReadObject(c *C) { diff --git a/plumbing/hash.go b/plumbing/hash.go index 2fab7593b..39bb73fbb 100644 --- a/plumbing/hash.go +++ b/plumbing/hash.go @@ -2,7 +2,6 @@ package plumbing import ( "bytes" - "crypto" "encoding/hex" "sort" "strconv" @@ -11,7 +10,7 @@ import ( ) // Hash SHA1 hashed content -type Hash [20]byte +type Hash [hash.Size]byte // ZeroHash is Hash with value zero var ZeroHash Hash @@ -47,7 +46,7 @@ type Hasher struct { } func NewHasher(t ObjectType, size int64) Hasher { - h := Hasher{hash.New(crypto.SHA1)} + h := Hasher{hash.New(hash.CryptoType)} h.Write(t.Bytes()) h.Write([]byte(" ")) h.Write([]byte(strconv.FormatInt(size, 10))) @@ -75,10 +74,11 @@ func (p HashSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // IsHash returns true if the given string is a valid hash. func IsHash(s string) bool { - if len(s) != 40 { + switch len(s) { + case hash.HexSize: + _, err := hex.DecodeString(s) + return err == nil + default: return false } - - _, err := hex.DecodeString(s) - return err == nil } diff --git a/plumbing/protocol/packp/ulreq_decode_test.go b/plumbing/protocol/packp/ulreq_decode_test.go index 9628f0fdd..efcc7b456 100644 --- a/plumbing/protocol/packp/ulreq_decode_test.go +++ b/plumbing/protocol/packp/ulreq_decode_test.go @@ -8,6 +8,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/pktline" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" . "gopkg.in/check.v1" @@ -119,8 +120,8 @@ type byHash []plumbing.Hash func (a byHash) Len() int { return len(a) } func (a byHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byHash) Less(i, j int) bool { - ii := [20]byte(a[i]) - jj := [20]byte(a[j]) + ii := [hash.Size]byte(a[i]) + jj := [hash.Size]byte(a[j]) return bytes.Compare(ii[:], jj[:]) < 0 } diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 6c386f799..2772b6736 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -16,6 +16,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/utils/ioutil" @@ -552,8 +553,8 @@ func (d *DotGit) hasPack(h plumbing.Hash) error { } func (d *DotGit) objectPath(h plumbing.Hash) string { - hash := h.String() - return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) + hex := h.String() + return d.fs.Join(objectsPath, hex[0:2], hex[2:hash.HexSize]) } // incomingObjectPath is intended to add support for a git pre-receive hook @@ -563,15 +564,16 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { // // More on git hooks found here : https://git-scm.com/docs/githooks // More on 'quarantine'/incoming directory here: -// https://git-scm.com/docs/git-receive-pack +// +// https://git-scm.com/docs/git-receive-pack func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { hString := h.String() if d.incomingDirName == "" { - return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + return d.fs.Join(objectsPath, hString[0:2], hString[2:hash.HexSize]) } - return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:hash.HexSize]) } // hasIncomingObjects searches for an incoming directory and keeps its name diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go index e2ede938c..849b7a176 100644 --- a/storage/filesystem/dotgit/writers.go +++ b/storage/filesystem/dotgit/writers.go @@ -9,6 +9,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/idxfile" "github.com/go-git/go-git/v5/plumbing/format/objfile" "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-billy/v5" ) @@ -277,8 +278,8 @@ func (w *ObjectWriter) Close() error { } func (w *ObjectWriter) save() error { - hash := w.Hash().String() - file := w.fs.Join(objectsPath, hash[0:2], hash[2:40]) + hex := w.Hash().String() + file := w.fs.Join(objectsPath, hex[0:2], hex[2:hash.HexSize]) return w.fs.Rename(w.f.Name(), file) } From 8e8281008ee98e9864fa9597be3e16aaecdf9891 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 7 Mar 2023 23:41:07 +0000 Subject: [PATCH 04/31] examples: Add example for SHA256 repositories Add a new example on initializing a new repository with object format SHA256. A new Makefile target was created to run the example to ensure that new changes won't break SHA256 support going forwards. Relates to the SHA256 implementation, defined in #706. Signed-off-by: Paulo Gomes --- .github/workflows/git.yml | 3 ++ Makefile | 6 ++++ _examples/sha256/main.go | 66 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 _examples/sha256/main.go diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index c945e72ff..3a40c0c58 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -42,6 +42,9 @@ jobs: - name: Test run: make test-coverage + - name: Test SHA256 + run: make test-sha256 + - name: Build go-git with CGO disabled run: go build ./... env: diff --git a/Makefile b/Makefile index 2acb8bc45..f0c1abb60 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,12 @@ test: @echo "running against `git version`"; \ $(GOTEST) -race ./... +TEMP_REPO := $(shell mktemp) +test-sha256: + $(GOCMD) run -tags sha256 _examples/sha256/main.go $(TEMP_REPO) + cd $(TEMP_REPO) && git fsck + rm -rf $(TEMP_REPO) + test-coverage: @echo "running against `git version`"; \ echo "" > $(COVERAGE_REPORT); \ diff --git a/_examples/sha256/main.go b/_examples/sha256/main.go new file mode 100644 index 000000000..2a257e863 --- /dev/null +++ b/_examples/sha256/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/go-git/go-git/v5" + . "github.com/go-git/go-git/v5/_examples" + "github.com/go-git/go-git/v5/plumbing/format/config" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// This example requires building with the sha256 tag for it to work: +// go run -tags sha256 main.go /tmp/repository + +// Basic example of how to initialise a repository using sha256 as the hashing algorithmn. +func main() { + CheckArgs("") + directory := os.Args[1] + + os.RemoveAll(directory) + + // Init a new repository using the ObjectFormat SHA256. + r, err := git.PlainInitWithOptions(directory, &git.PlainInitOptions{ObjectFormat: config.SHA256}) + CheckIfError(err) + + w, err := r.Worktree() + CheckIfError(err) + + // ... we need a file to commit so let's create a new file inside of the + // worktree of the project using the go standard library. + Info("echo \"hello world!\" > example-git-file") + filename := filepath.Join(directory, "example-git-file") + err = ioutil.WriteFile(filename, []byte("hello world!"), 0644) + CheckIfError(err) + + // Adds the new file to the staging area. + Info("git add example-git-file") + _, err = w.Add("example-git-file") + CheckIfError(err) + + // Commits the current staging area to the repository, with the new file + // just created. We should provide the object.Signature of Author of the + // commit Since version 5.0.1, we can omit the Author signature, being read + // from the git config files. + Info("git commit -m \"example go-git commit\"") + commit, err := w.Commit("example go-git commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "John Doe", + Email: "john@doe.org", + When: time.Now(), + }, + }) + + CheckIfError(err) + + // Prints the current HEAD to verify that all worked well. + Info("git show -s") + obj, err := r.CommitObject(commit) + CheckIfError(err) + + fmt.Println(obj) +} From cdd1e5562d10c6bb19f979ca32f3ae7ef646024f Mon Sep 17 00:00:00 2001 From: ZauberNerd Date: Tue, 15 Mar 2022 17:02:30 +0000 Subject: [PATCH 05/31] plumbing: resolve non-external delta references In a self-contained pack file delta references might point to base objects stored later in the file. In this case we need to replace placeholders for external refs with the actual base object and update the children references. Fixes: #484 Co-authored-by: Markus Wolf --- go.mod | 2 +- go.sum | 4 ++-- plumbing/format/packfile/parser.go | 9 +++++++++ plumbing/format/packfile/parser_test.go | 13 +++++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 85fd7b132..95cb02bb9 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/gliderlabs/ssh v0.3.5 github.com/go-git/gcfg v1.5.0 github.com/go-git/go-billy/v5 v5.4.1 - github.com/go-git/go-git-fixtures/v4 v4.3.1 + github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f github.com/google/go-cmp v0.5.9 github.com/imdario/mergo v0.3.13 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 diff --git a/go.sum b/go.sum index a7548e292..bb9bc8506 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4u github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= -github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= -github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 522c146f2..6012b0458 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -237,6 +237,15 @@ func (p *Parser) indexObjects() error { return err } + // Move children of placeholder parent into actual parent, in case this + // was a non-external delta reference. + if placeholder, ok := p.oiByHash[sha1]; ok { + ota.Children = placeholder.Children + for _, c := range ota.Children { + c.Parent = ota + } + } + ota.SHA1 = sha1 p.oiByHash[ota.SHA1] = ota } diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index 651d05f3d..b8d080f68 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -147,6 +147,19 @@ func (s *ParserSuite) TestResolveExternalRefsInThinPack(c *C) { c.Assert(err, IsNil) } +func (s *ParserSuite) TestResolveExternalRefs(c *C) { + extRefsThinPack := fixtures.ByTag("delta-before-base").One() + + scanner := packfile.NewScanner(extRefsThinPack.Packfile()) + + obs := new(testObserver) + parser, err := packfile.NewParser(scanner, obs) + c.Assert(err, IsNil) + + _, err = parser.Parse() + c.Assert(err, IsNil) +} + type observerObject struct { hash string otype plumbing.ObjectType From fc0bf7f9ccee01257879565fe36c13306408f843 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 11 Apr 2023 20:32:17 +0100 Subject: [PATCH 06/31] ci: Fix upstream git build for master branch Changes upstream broke the compatibility test, which now requires libcurl installed in order to successfully build. Additionally, the .git-dist dir was added to the .gitignore file. Fixes: https://github.com/go-git/go-git/actions/runs/4670641376/jobs/8270625669 Signed-off-by: Paulo Gomes --- .github/workflows/git.yml | 2 +- .gitignore | 3 ++- Makefile | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index c945e72ff..b95cefd7f 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v3 - name: Install build dependencies - run: sudo apt-get install gettext + run: sudo apt-get install gettext libcurl4-openssl-dev - name: Git Build run: make build-git diff --git a/.gitignore b/.gitignore index e8d25f548..361133d03 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ coverage.out *~ coverage.txt profile.out -.tmp/ \ No newline at end of file +.tmp/ +.git-dist/ diff --git a/Makefile b/Makefile index 2acb8bc45..093a4f47c 100644 --- a/Makefile +++ b/Makefile @@ -35,4 +35,4 @@ test-coverage: $(GOTEST) -coverprofile=$(COVERAGE_REPORT) -coverpkg=./... -covermode=$(COVERAGE_MODE) ./... clean: - rm -rf $(GIT_DIST_PATH) \ No newline at end of file + rm -rf $(GIT_DIST_PATH) From a71389b9d9293a6fefc3c923e765f250235a5262 Mon Sep 17 00:00:00 2001 From: Joseda Rios Date: Thu, 16 Mar 2023 15:20:01 +0100 Subject: [PATCH 07/31] internal: Fix regression in csp-like match Signed-off-by: Joseda Rios --- internal/url/url.go | 6 +- internal/url/url_test.go | 107 ++++++++++++++++++++++-------- plumbing/transport/common_test.go | 20 ++++-- 3 files changed, 98 insertions(+), 35 deletions(-) diff --git a/internal/url/url.go b/internal/url/url.go index 14cf133de..266244869 100644 --- a/internal/url/url.go +++ b/internal/url/url.go @@ -5,8 +5,10 @@ import ( ) var ( - isSchemeRegExp = regexp.MustCompile(`^[^:]+://`) - scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P[^@]+)@)?(?P[^:\s]+):(?:(?P[0-9]{1,5})(?:\/|:))?(?P[^\\].*\/[^\\].*)$`) + isSchemeRegExp = regexp.MustCompile(`^[^:]+://`) + + // Ref: https://github.com/git/git/blob/master/Documentation/urls.txt#L37 + scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P[^@]+)@)?(?P[^:\s]+):(?:(?P[0-9]{1,5}):)?(?P[^\\].*)$`) ) // MatchesScheme returns true if the given string matches a URL-like diff --git a/internal/url/url_test.go b/internal/url/url_test.go index d168db6df..29c3f3e96 100755 --- a/internal/url/url_test.go +++ b/internal/url/url_test.go @@ -13,11 +13,27 @@ type URLSuite struct{} var _ = Suite(&URLSuite{}) func (s *URLSuite) TestMatchesScpLike(c *C) { + // See https://github.com/git/git/blob/master/Documentation/urls.txt#L37 examples := []string{ + // Most-extended case "git@github.com:james/bond", - "git@github.com:007/bond", + // Most-extended case with port "git@github.com:22:james/bond", + // Most-extended case with numeric path + "git@github.com:007/bond", + // Most-extended case with port and numeric "username" "git@github.com:22:007/bond", + // Single repo path + "git@github.com:bond", + // Single repo path with port + "git@github.com:22:bond", + // Single repo path with port and numeric repo + "git@github.com:22:007", + // Repo path ending with .git and starting with _ + "git@github.com:22:_007.git", + "git@github.com:_007.git", + "git@github.com:_james.git", + "git@github.com:_james/bond.git", } for _, url := range examples { @@ -26,35 +42,68 @@ func (s *URLSuite) TestMatchesScpLike(c *C) { } func (s *URLSuite) TestFindScpLikeComponents(c *C) { - url := "git@github.com:james/bond" - user, host, port, path := FindScpLikeComponents(url) - - c.Check(user, Equals, "git") - c.Check(host, Equals, "github.com") - c.Check(port, Equals, "") - c.Check(path, Equals, "james/bond") - - url = "git@github.com:007/bond" - user, host, port, path = FindScpLikeComponents(url) - - c.Check(user, Equals, "git") - c.Check(host, Equals, "github.com") - c.Check(port, Equals, "") - c.Check(path, Equals, "007/bond") - - url = "git@github.com:22:james/bond" - user, host, port, path = FindScpLikeComponents(url) + testCases := []struct { + url, user, host, port, path string + }{ + { + // Most-extended case + url: "git@github.com:james/bond", user: "git", host: "github.com", port: "", path: "james/bond", + }, + { + // Most-extended case with port + url: "git@github.com:22:james/bond", user: "git", host: "github.com", port: "22", path: "james/bond", + }, + { + // Most-extended case with numeric path + url: "git@github.com:007/bond", user: "git", host: "github.com", port: "", path: "007/bond", + }, + { + // Most-extended case with port and numeric path + url: "git@github.com:22:007/bond", user: "git", host: "github.com", port: "22", path: "007/bond", + }, + { + // Single repo path + url: "git@github.com:bond", user: "git", host: "github.com", port: "", path: "bond", + }, + { + // Single repo path with port + url: "git@github.com:22:bond", user: "git", host: "github.com", port: "22", path: "bond", + }, + { + // Single repo path with port and numeric path + url: "git@github.com:22:007", user: "git", host: "github.com", port: "22", path: "007", + }, + { + // Repo path ending with .git and starting with _ + url: "git@github.com:22:_007.git", user: "git", host: "github.com", port: "22", path: "_007.git", + }, + { + // Repo path ending with .git and starting with _ + url: "git@github.com:_007.git", user: "git", host: "github.com", port: "", path: "_007.git", + }, + { + // Repo path ending with .git and starting with _ + url: "git@github.com:_james.git", user: "git", host: "github.com", port: "", path: "_james.git", + }, + { + // Repo path ending with .git and starting with _ + url: "git@github.com:_james/bond.git", user: "git", host: "github.com", port: "", path: "_james/bond.git", + }, + } - c.Check(user, Equals, "git") - c.Check(host, Equals, "github.com") - c.Check(port, Equals, "22") - c.Check(path, Equals, "james/bond") + for _, tc := range testCases { + user, host, port, path := FindScpLikeComponents(tc.url) - url = "git@github.com:22:007/bond" - user, host, port, path = FindScpLikeComponents(url) + logf := func(ok bool) { + if ok { + return + } + c.Logf("%q check failed", tc.url) + } - c.Check(user, Equals, "git") - c.Check(host, Equals, "github.com") - c.Check(port, Equals, "22") - c.Check(path, Equals, "007/bond") + logf(c.Check(user, Equals, tc.user)) + logf(c.Check(host, Equals, tc.host)) + logf(c.Check(port, Equals, tc.port)) + logf(c.Check(path, Equals, tc.path)) + } } diff --git a/plumbing/transport/common_test.go b/plumbing/transport/common_test.go index 0c5a01a9a..db11303a4 100644 --- a/plumbing/transport/common_test.go +++ b/plumbing/transport/common_test.go @@ -95,16 +95,28 @@ func (s *SuiteCommon) TestNewEndpointSCPLike(c *C) { c.Assert(e.String(), Equals, "ssh://git@github.com/user/repository.git") } -func (s *SuiteCommon) TestNewEndpointSCPLikeWithPort(c *C) { +func (s *SuiteCommon) TestNewEndpointSCPLikeWithNumericPath(c *C) { e, err := NewEndpoint("git@github.com:9999/user/repository.git") c.Assert(err, IsNil) c.Assert(e.Protocol, Equals, "ssh") c.Assert(e.User, Equals, "git") c.Assert(e.Password, Equals, "") c.Assert(e.Host, Equals, "github.com") - c.Assert(e.Port, Equals, 9999) - c.Assert(e.Path, Equals, "user/repository.git") - c.Assert(e.String(), Equals, "ssh://git@github.com:9999/user/repository.git") + c.Assert(e.Port, Equals, 22) + c.Assert(e.Path, Equals, "9999/user/repository.git") + c.Assert(e.String(), Equals, "ssh://git@github.com/9999/user/repository.git") +} + +func (s *SuiteCommon) TestNewEndpointSCPLikeWithPort(c *C) { + e, err := NewEndpoint("git@github.com:8080:9999/user/repository.git") + c.Assert(err, IsNil) + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 8080) + c.Assert(e.Path, Equals, "9999/user/repository.git") + c.Assert(e.String(), Equals, "ssh://git@github.com:8080/9999/user/repository.git") } func (s *SuiteCommon) TestNewEndpointFileAbs(c *C) { From 9a5b08f5c32bad31a35a53c045ebf6c8409f8b2c Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 4 Apr 2023 01:13:21 -0400 Subject: [PATCH 08/31] feat(clone): add mirror clone option Clone remote as a mirror. This fetches all remote refs, implies bare repository, and sets the appropriate configs. Fixes: https://github.com/go-git/go-git/issues/293 Update options.go Co-authored-by: Paulo Gomes Signed-off-by: Ayman Bagabas --- config/config.go | 8 ++++++++ options.go | 8 ++++++++ repository.go | 14 ++++++++++---- repository_test.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 83629fcef..60dfca4f0 100644 --- a/config/config.go +++ b/config/config.go @@ -264,6 +264,7 @@ const ( defaultBranchKey = "defaultBranch" repositoryFormatVersionKey = "repositoryformatversion" objectFormat = "objectformat" + mirrorKey = "mirror" // DefaultPackWindow holds the number of previous objects used to // generate deltas. The value 10 is the same used by git command. @@ -578,6 +579,8 @@ type RemoteConfig struct { // URLs the URLs of a remote repository. It must be non-empty. Fetch will // always use the first URL, while push will use all of them. URLs []string + // Mirror indicates that the repository is a mirror of remote. + Mirror bool // insteadOfRulesApplied have urls been modified insteadOfRulesApplied bool @@ -631,6 +634,7 @@ func (c *RemoteConfig) unmarshal(s *format.Subsection) error { c.Name = c.raw.Name c.URLs = append([]string(nil), c.raw.Options.GetAll(urlKey)...) c.Fetch = fetch + c.Mirror = c.raw.Options.Get(mirrorKey) == "true" return nil } @@ -663,6 +667,10 @@ func (c *RemoteConfig) marshal() *format.Subsection { c.raw.SetOption(fetchKey, values...) } + if c.Mirror { + c.raw.SetOption(mirrorKey, strconv.FormatBool(c.Mirror)) + } + return c.raw } diff --git a/options.go b/options.go index 738c1ce9c..6a38ad94f 100644 --- a/options.go +++ b/options.go @@ -46,6 +46,14 @@ type CloneOptions struct { ReferenceName plumbing.ReferenceName // Fetch only ReferenceName if true. SingleBranch bool + // Mirror clones the repository as a mirror. + // + // Compared to a bare clone, mirror not only maps local branches of the + // source to local branches of the target, it maps all refs (including + // remote-tracking branches, notes etc.) and sets up a refspec configuration + // such that all these refs are overwritten by a git remote update in the + // target repository. + Mirror bool // No checkout of HEAD after clone if true. NoCheckout bool // Limit fetching to the specified number of commits. diff --git a/repository.go b/repository.go index 56ae9761c..e009c5d06 100644 --- a/repository.go +++ b/repository.go @@ -444,6 +444,9 @@ func PlainCloneContext(ctx context.Context, path string, isBare bool, o *CloneOp return nil, err } + if o.Mirror { + isBare = true + } r, err := PlainInit(path, isBare) if err != nil { return nil, err @@ -851,9 +854,10 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } c := &config.RemoteConfig{ - Name: o.RemoteName, - URLs: []string{o.URL}, - Fetch: r.cloneRefSpec(o), + Name: o.RemoteName, + URLs: []string{o.URL}, + Fetch: r.cloneRefSpec(o), + Mirror: o.Mirror, } if _, err := r.CreateRemote(c); err != nil { @@ -906,7 +910,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { return err } - if ref.Name().IsBranch() { + if !o.Mirror && ref.Name().IsBranch() { branchRef := ref.Name() branchName := strings.Split(string(branchRef), "refs/heads/")[1] @@ -937,6 +941,8 @@ const ( func (r *Repository) cloneRefSpec(o *CloneOptions) []config.RefSpec { switch { + case o.Mirror: + return []config.RefSpec{"+refs/*:refs/*"} case o.ReferenceName.IsTag(): return []config.RefSpec{ config.RefSpec(fmt.Sprintf(refspecTag, o.ReferenceName.Short())), diff --git a/repository_test.go b/repository_test.go index 468ce33b0..ed3e7e69b 100644 --- a/repository_test.go +++ b/repository_test.go @@ -189,6 +189,35 @@ func (s *RepositorySuite) TestCloneContext(c *C) { c.Assert(err, Equals, context.Canceled) } +func (s *RepositorySuite) TestCloneMirror(c *C) { + r, err := Clone(memory.NewStorage(), nil, &CloneOptions{ + URL: fixtures.Basic().One().URL, + Mirror: true, + }) + + c.Assert(err, IsNil) + + refs, err := r.References() + var count int + refs.ForEach(func(r *plumbing.Reference) error { c.Log(r); count++; return nil }) + c.Assert(err, IsNil) + // 6 refs total from github.com/git-fixtures/basic.git: + // - HEAD + // - refs/heads/master + // - refs/heads/branch + // - refs/pull/1/head + // - refs/pull/2/head + // - refs/pull/2/merge + c.Assert(count, Equals, 6) + + cfg, err := r.Config() + c.Assert(err, IsNil) + + c.Assert(cfg.Core.IsBare, Equals, true) + c.Assert(cfg.Remotes[DefaultRemoteName].Validate(), IsNil) + c.Assert(cfg.Remotes[DefaultRemoteName].Mirror, Equals, true) +} + func (s *RepositorySuite) TestCloneWithTags(c *C) { url := s.GetLocalRepositoryURL( fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), From d4e36fa406a18e78a24fa9cd68d4e34d8434b6b7 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Sun, 23 Apr 2023 15:01:36 +0100 Subject: [PATCH 09/31] build: Bump dependencies. Fixes #667 - github.com/ProtonMail/go-crypto to version 0.0.0-20230417170513-8ee5748c52b5. - github.com/imdario/mergo to version 0.3.15. - golang.org/x/crypto to version 0.8.0. - golang.org/x/net to version 0.9.0. - golang.org/x/sys to version 0.7.0. - golang.org/x/text to version 0.9.0. - github.com/go-git/gcfg to version v1.5.1-0.20230307220236-3a3c6141e376 Signed-off-by: Paulo Gomes --- go.mod | 16 +++++++--------- go.sum | 41 ++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 95cb02bb9..27b74a36a 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,26 @@ module github.com/go-git/go-git/v5 go 1.13 require ( - github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 + github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5 github.com/acomagu/bufpipe v1.0.4 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/emirpasic/gods v1.18.1 github.com/gliderlabs/ssh v0.3.5 - github.com/go-git/gcfg v1.5.0 + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f github.com/google/go-cmp v0.5.9 - github.com/imdario/mergo v0.3.13 + github.com/imdario/mergo v0.3.15 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 github.com/jessevdk/go-flags v1.5.0 github.com/kevinburke/ssh_config v1.2.0 github.com/pjbgf/sha1cd v0.3.0 - github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.1.0 github.com/skeema/knownhosts v1.1.0 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.6.0 - golang.org/x/net v0.7.0 - golang.org/x/sys v0.5.0 - golang.org/x/text v0.7.0 + golang.org/x/crypto v0.8.0 + golang.org/x/net v0.9.0 + golang.org/x/sys v0.7.0 + golang.org/x/text v0.9.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c - gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index bb9bc8506..f4fb14682 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5 h1:QXMwHM/lB4ZQhdEF7JUTNgYOJR/gWoFbgQ/2Aj1h3Dk= +github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -19,8 +19,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= -github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= -github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= @@ -28,8 +28,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= @@ -73,10 +73,12 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -84,10 +86,12 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -104,25 +108,32 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -134,6 +145,6 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From ca09e555e4b4b6bdd4c25acafe1b0e42353e2c50 Mon Sep 17 00:00:00 2001 From: Force Charlie Date: Tue, 25 Apr 2023 21:51:46 +0800 Subject: [PATCH 10/31] storage: filesystem/dotgit, Improve load packed-refs --- plumbing/reference_test.go | 2 +- storage/filesystem/dotgit/dotgit.go | 80 ++++++++++++++--------------- 2 files changed, 39 insertions(+), 43 deletions(-) diff --git a/plumbing/reference_test.go b/plumbing/reference_test.go index e69076ff6..04dfef93d 100644 --- a/plumbing/reference_test.go +++ b/plumbing/reference_test.go @@ -105,7 +105,7 @@ func (s *ReferenceSuite) TestIsTag(c *C) { func benchMarkReferenceString(r *Reference, b *testing.B) { for n := 0; n < b.N; n++ { - r.String() + _ = r.String() } } diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 686832430..5bde37f6b 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -718,48 +718,56 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) { return d.packedRef(name) } -func (d *DotGit) findPackedRefsInFile(f billy.File) ([]*plumbing.Reference, error) { +func (d *DotGit) findPackedRefsInFile(f billy.File, recv refsRecv) error { s := bufio.NewScanner(f) - var refs []*plumbing.Reference for s.Scan() { ref, err := d.processLine(s.Text()) if err != nil { - return nil, err + return err } - if ref != nil { - refs = append(refs, ref) + if !recv(ref) { + // skip parse + return nil } } - - return refs, s.Err() + if err := s.Err(); err != nil { + return err + } + return nil } -func (d *DotGit) findPackedRefs() (r []*plumbing.Reference, err error) { +// refsRecv: returning true means that the reference continues to be resolved, otherwise it is stopped, which will speed up the lookup of a single reference. +type refsRecv func(*plumbing.Reference) bool + +func (d *DotGit) findPackedRefs(recv refsRecv) error { f, err := d.fs.Open(packedRefsPath) if err != nil { if os.IsNotExist(err) { - return nil, nil + return nil } - return nil, err + return err } defer ioutil.CheckClose(f, &err) - return d.findPackedRefsInFile(f) + return d.findPackedRefsInFile(f, recv) } func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) { - refs, err := d.findPackedRefs() - if err != nil { + var ref *plumbing.Reference + if err := d.findPackedRefs(func(r *plumbing.Reference) bool { + if r != nil && r.Name() == name { + ref = r + // ref found + return false + } + return true + }); err != nil { return nil, err } - - for _, ref := range refs { - if ref.Name() == name { - return ref, nil - } + if ref != nil { + return ref, nil } - return nil, plumbing.ErrReferenceNotFound } @@ -779,34 +787,22 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error { return d.rewritePackedRefsWithoutRef(name) } -func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) { - packedRefs, err := d.findPackedRefs() - if err != nil { - return err - } - - for _, ref := range packedRefs { - if !seen[ref.Name()] { - *refs = append(*refs, ref) - seen[ref.Name()] = true +func refsRecvFunc(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) refsRecv { + return func(r *plumbing.Reference) bool { + if r != nil && !seen[r.Name()] { + *refs = append(*refs, r) + seen[r.Name()] = true } + return true } - return nil } -func (d *DotGit) addRefsFromPackedRefsFile(refs *[]*plumbing.Reference, f billy.File, seen map[plumbing.ReferenceName]bool) (err error) { - packedRefs, err := d.findPackedRefsInFile(f) - if err != nil { - return err - } +func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) { + return d.findPackedRefs(refsRecvFunc(refs, seen)) +} - for _, ref := range packedRefs { - if !seen[ref.Name()] { - *refs = append(*refs, ref) - seen[ref.Name()] = true - } - } - return nil +func (d *DotGit) addRefsFromPackedRefsFile(refs *[]*plumbing.Reference, f billy.File, seen map[plumbing.ReferenceName]bool) (err error) { + return d.findPackedRefsInFile(f, refsRecvFunc(refs, seen)) } func (d *DotGit) openAndLockPackedRefs(doCreate bool) ( From 3aa7575a4d660a250edd06864d6401a302951fab Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 29 Mar 2023 23:26:53 -0400 Subject: [PATCH 11/31] fix: git grep bare repositories Perform grep on `*Repository` instead of `*Worktree`. Fixes: https://github.com/go-git/go-git/issues/68 --- options.go | 8 ++++- worktree.go | 21 ++++++++----- worktree_test.go | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 9 deletions(-) diff --git a/options.go b/options.go index 738c1ce9c..112d60004 100644 --- a/options.go +++ b/options.go @@ -642,7 +642,13 @@ var ( ) // Validate validates the fields and sets the default values. +// +// TODO: deprecate in favor of Validate(r *Repository) in v6. func (o *GrepOptions) Validate(w *Worktree) error { + return o.validate(w.r) +} + +func (o *GrepOptions) validate(r *Repository) error { if !o.CommitHash.IsZero() && o.ReferenceName != "" { return ErrHashOrReference } @@ -650,7 +656,7 @@ func (o *GrepOptions) Validate(w *Worktree) error { // If none of CommitHash and ReferenceName are provided, set commit hash of // the repository's head. if o.CommitHash.IsZero() && o.ReferenceName == "" { - ref, err := w.r.Head() + ref, err := r.Head() if err != nil { return err } diff --git a/worktree.go b/worktree.go index d28ba3241..7d2f3b47f 100644 --- a/worktree.go +++ b/worktree.go @@ -290,7 +290,7 @@ func (w *Worktree) ResetSparsely(opts *ResetOptions, dirs []string) error { return nil } - t, err := w.getTreeFromCommitHash(opts.Commit) + t, err := w.r.getTreeFromCommitHash(opts.Commit) if err != nil { return err } @@ -633,8 +633,8 @@ func (w *Worktree) addIndexFromFile(name string, h plumbing.Hash, idx *indexBuil return nil } -func (w *Worktree) getTreeFromCommitHash(commit plumbing.Hash) (*object.Tree, error) { - c, err := w.r.CommitObject(commit) +func (r *Repository) getTreeFromCommitHash(commit plumbing.Hash) (*object.Tree, error) { + c, err := r.CommitObject(commit) if err != nil { return nil, err } @@ -802,9 +802,9 @@ func (gr GrepResult) String() string { return fmt.Sprintf("%s:%s:%d:%s", gr.TreeName, gr.FileName, gr.LineNumber, gr.Content) } -// Grep performs grep on a worktree. -func (w *Worktree) Grep(opts *GrepOptions) ([]GrepResult, error) { - if err := opts.Validate(w); err != nil { +// Grep performs grep on a repository. +func (r *Repository) Grep(opts *GrepOptions) ([]GrepResult, error) { + if err := opts.validate(r); err != nil { return nil, err } @@ -814,7 +814,7 @@ func (w *Worktree) Grep(opts *GrepOptions) ([]GrepResult, error) { var treeName string if opts.ReferenceName != "" { - ref, err := w.r.Reference(opts.ReferenceName, true) + ref, err := r.Reference(opts.ReferenceName, true) if err != nil { return nil, err } @@ -827,7 +827,7 @@ func (w *Worktree) Grep(opts *GrepOptions) ([]GrepResult, error) { // Obtain a tree from the commit hash and get a tracked files iterator from // the tree. - tree, err := w.getTreeFromCommitHash(commitHash) + tree, err := r.getTreeFromCommitHash(commitHash) if err != nil { return nil, err } @@ -836,6 +836,11 @@ func (w *Worktree) Grep(opts *GrepOptions) ([]GrepResult, error) { return findMatchInFiles(fileiter, treeName, opts) } +// Grep performs grep on a worktree. +func (w *Worktree) Grep(opts *GrepOptions) ([]GrepResult, error) { + return w.r.Grep(opts) +} + // findMatchInFiles takes a FileIter, worktree name and GrepOptions, and // returns a slice of GrepResult containing the result of regex pattern matching // in content of all the files. diff --git a/worktree_test.go b/worktree_test.go index ac56a4688..1d6a05fd2 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -2211,6 +2211,87 @@ func (s *WorktreeSuite) TestGrep(c *C) { } } +func (s *WorktreeSuite) TestGrepBare(c *C) { + cases := []struct { + name string + options GrepOptions + wantResult []GrepResult + dontWantResult []GrepResult + wantError error + }{ + { + name: "basic word match", + options: GrepOptions{ + Patterns: []*regexp.Regexp{regexp.MustCompile("import")}, + CommitHash: plumbing.ZeroHash, + }, + wantResult: []GrepResult{ + { + FileName: "go/example.go", + LineNumber: 3, + Content: "import (", + TreeName: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + }, + { + FileName: "vendor/foo.go", + LineNumber: 3, + Content: "import \"fmt\"", + TreeName: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + }, + }, + }, + } + + path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() + + dir, clean := s.TemporalDir() + defer clean() + + r, err := PlainClone(dir, true, &CloneOptions{ + URL: path, + }) + c.Assert(err, IsNil) + + for _, tc := range cases { + gr, err := r.Grep(&tc.options) + if tc.wantError != nil { + c.Assert(err, Equals, tc.wantError) + } else { + c.Assert(err, IsNil) + } + + // Iterate through the results and check if the wanted result is present + // in the got result. + for _, wantResult := range tc.wantResult { + found := false + for _, gotResult := range gr { + if wantResult == gotResult { + found = true + break + } + } + if !found { + c.Errorf("unexpected grep results for %q, expected result to contain: %v", tc.name, wantResult) + } + } + + // Iterate through the results and check if the not wanted result is + // present in the got result. + for _, dontWantResult := range tc.dontWantResult { + found := false + for _, gotResult := range gr { + if dontWantResult == gotResult { + found = true + break + } + } + if found { + c.Errorf("unexpected grep results for %q, expected result to NOT contain: %v", tc.name, dontWantResult) + } + } + } +} + func (s *WorktreeSuite) TestResetLingeringDirectories(c *C) { dir, clean := s.TemporalDir() defer clean() From cf51e2febf37332f11ae63feca768d9672e10a36 Mon Sep 17 00:00:00 2001 From: John Pastore Date: Tue, 21 Mar 2023 09:15:32 -0400 Subject: [PATCH 12/31] Worktree: Status, add check to see if file already checked in [Fixes 718] Checks if an ignored file was previously checked in. If it was, then the file is not ignored matching native git behavior. --- worktree_status.go | 4 +++- worktree_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/worktree_status.go b/worktree_status.go index f3091cff6..a26c9e51f 100644 --- a/worktree_status.go +++ b/worktree_status.go @@ -169,7 +169,9 @@ func (w *Worktree) excludeIgnoredChanges(changes merkletrie.Changes) merkletrie. if len(path) != 0 { isDir := (len(ch.To) > 0 && ch.To.IsDir()) || (len(ch.From) > 0 && ch.From.IsDir()) if m.Match(path, isDir) { - continue + if len(ch.From) == 0 { + continue + } } } res = append(res, ch) diff --git a/worktree_test.go b/worktree_test.go index ac56a4688..294b7ed23 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -871,6 +871,51 @@ func (s *WorktreeSuite) TestStatusEmpty(c *C) { c.Assert(status, NotNil) } +func (s *WorktreeSuite) TestStatusCheckedInBeforeIgnored(c *C) { + fs := memfs.New() + storage := memory.NewStorage() + + r, err := Init(storage, fs) + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + err = util.WriteFile(fs, "fileToIgnore", []byte("Initial data"), 0755) + c.Assert(err, IsNil) + _, err = w.Add("fileToIgnore") + c.Assert(err, IsNil) + _, err = w.Commit("Added file that will be ignored later", &CommitOptions{}) + c.Assert(err, IsNil) + + err = util.WriteFile(fs, ".gitignore", []byte("fileToIgnore\nsecondIgnoredFile"), 0755) + c.Assert(err, IsNil) + _, err = w.Add(".gitignore") + c.Assert(err, IsNil) + _, err = w.Commit("Added .gitignore", &CommitOptions{}) + c.Assert(err, IsNil) + status, err := w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, true) + c.Assert(status, NotNil) + + err = util.WriteFile(fs, "secondIgnoredFile", []byte("Should be completly ignored"), 0755) + c.Assert(err, IsNil) + status = nil + status, err = w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, true) + c.Assert(status, NotNil) + + err = util.WriteFile(fs, "fileToIgnore", []byte("Updated data"), 0755) + c.Assert(err, IsNil) + status = nil + status, err = w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, false) + c.Assert(status, NotNil) +} + func (s *WorktreeSuite) TestStatusEmptyDirty(c *C) { fs := memfs.New() err := util.WriteFile(fs, "foo", []byte("foo"), 0755) From a4b11abc55bf88fbd07a00a5985a34750bee1d72 Mon Sep 17 00:00:00 2001 From: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> Date: Wed, 3 May 2023 14:01:24 +1000 Subject: [PATCH 13/31] git: fix cloning with branch name Signed-off-by: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> --- repository.go | 16 ++++++++++++++-- repository_test.go | 10 +++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/repository.go b/repository.go index e009c5d06..be5f140d7 100644 --- a/repository.go +++ b/repository.go @@ -1012,9 +1012,21 @@ func (r *Repository) fetchAndUpdateReferences( return nil, err } - resolvedRef, err := storer.ResolveReference(remoteRefs, ref) + var resolvedRef *plumbing.Reference + // return error from checking the raw ref passed in + var rawRefError error + for _, rule := range append([]string{"%s"}, plumbing.RefRevParseRules...) { + resolvedRef, err = storer.ResolveReference(remoteRefs, plumbing.ReferenceName(fmt.Sprintf(rule, ref))) + + if err == nil { + break + } else if rawRefError == nil { + rawRefError = err + } + } + if err != nil { - return nil, err + return nil, rawRefError } refsUpdated, err := r.updateReferences(remote.c.Fetch, resolvedRef) diff --git a/repository_test.go b/repository_test.go index ed3e7e69b..0080a8384 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1020,6 +1020,14 @@ func (s *RepositorySuite) TestCloneConfig(c *C) { } func (s *RepositorySuite) TestCloneSingleBranchAndNonHEAD(c *C) { + s.testCloneSingleBranchAndNonHEADReference(c, "refs/heads/branch") +} + +func (s *RepositorySuite) TestCloneSingleBranchAndNonHEADAndNonFull(c *C) { + s.testCloneSingleBranchAndNonHEADReference(c, "branch") +} + +func (s *RepositorySuite) testCloneSingleBranchAndNonHEADReference(c *C, ref string) { r, _ := Init(memory.NewStorage(), nil) head, err := r.Head() @@ -1028,7 +1036,7 @@ func (s *RepositorySuite) TestCloneSingleBranchAndNonHEAD(c *C) { err = r.clone(context.Background(), &CloneOptions{ URL: s.GetBasicLocalRepositoryURL(), - ReferenceName: plumbing.ReferenceName("refs/heads/branch"), + ReferenceName: plumbing.ReferenceName(ref), SingleBranch: true, }) From 19b39e150071832541f2cdcb669446a8609f57e7 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Wed, 3 May 2023 08:39:07 +0100 Subject: [PATCH 14/31] git: Add support to ls-remote with peeled references. Fixes #749 A new PeelingOption field was introduced into ListOptions. The new options include the default (and backwards compatible) IgnorePeeled. Plus another two variations which either only returns peeled references (OnlyPeeled), or append peeled references to the list (AppendPeeled). The ls-remote example was updated to align with upstream, in which peeled references are appended to the results by default. A new ErrEmptyUrls error is now returned when List or ListContext do not receive a URL to work with, to improve overall execution flow. Signed-off-by: Paulo Gomes --- _examples/ls-remote/main.go | 14 ++++++++++++-- options.go | 20 ++++++++++++++++++++ remote.go | 29 +++++++++++++++++++++++------ remote_test.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/_examples/ls-remote/main.go b/_examples/ls-remote/main.go index af038d6e2..e49e8c9e4 100644 --- a/_examples/ls-remote/main.go +++ b/_examples/ls-remote/main.go @@ -2,25 +2,35 @@ package main import ( "log" + "os" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/storage/memory" + + . "github.com/go-git/go-git/v5/_examples" ) // Retrieve remote tags without cloning repository func main() { + CheckArgs("") + url := os.Args[1] + + Info("git ls-remote --tags %s", url) // Create the remote with repository URL rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ Name: "origin", - URLs: []string{"https://github.com/Zenika/MARCEL"}, + URLs: []string{url}, }) log.Print("Fetching tags...") // We can then use every Remote functions to retrieve wanted information - refs, err := rem.List(&git.ListOptions{}) + refs, err := rem.List(&git.ListOptions{ + // Returns all references, including peeled references. + PeelingOption: git.AppendPeeled, + }) if err != nil { log.Fatal(err) } diff --git a/options.go b/options.go index e79cf9dae..04c83de61 100644 --- a/options.go +++ b/options.go @@ -624,8 +624,28 @@ type ListOptions struct { InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte + + // PeelingOption defines how peeled objects are handled during a + // remote list. + PeelingOption PeelingOption } +// PeelingOption represents the different ways to handle peeled references. +// +// Peeled references represent the underlying object of an annotated +// (or signed) tag. Refer to upstream documentation for more info: +// https://github.com/git/git/blob/master/Documentation/technical/reftable.txt +type PeelingOption uint8 + +const ( + // IgnorePeeled ignores all peeled reference names. This is the default behavior. + IgnorePeeled PeelingOption = 0 + // OnlyPeeled returns only peeled reference names. + OnlyPeeled PeelingOption = 1 + // AppendPeeled appends peeled reference names to the reference list. + AppendPeeled PeelingOption = 2 +) + // CleanOptions describes how a clean should be performed. type CleanOptions struct { Dir bool diff --git a/remote.go b/remote.go index db78ae718..ff72bdf36 100644 --- a/remote.go +++ b/remote.go @@ -33,6 +33,7 @@ var ( ErrDeleteRefNotSupported = errors.New("server does not support delete-refs") ErrForceNeeded = errors.New("some refs were not updated") ErrExactSHA1NotSupported = errors.New("server does not support exact SHA1 refspec") + ErrEmptyUrls = errors.New("URLs cannot be empty") ) type NoMatchingRefSpecError struct { @@ -54,6 +55,9 @@ const ( // repo containing this remote, when not using the multi-ack // protocol. Setting this to 0 means there is no limit. maxHavesToVisitPerRef = 100 + + // peeledSuffix is the suffix used to build peeled reference names. + peeledSuffix = "^{}" ) // Remote represents a connection to a remote repository. @@ -1259,6 +1263,10 @@ func (r *Remote) List(o *ListOptions) (rfs []*plumbing.Reference, err error) { } func (r *Remote) list(ctx context.Context, o *ListOptions) (rfs []*plumbing.Reference, err error) { + if r.c == nil || len(r.c.URLs) == 0 { + return nil, ErrEmptyUrls + } + s, err := newUploadPackSession(r.c.URLs[0], o.Auth, o.InsecureSkipTLS, o.CABundle) if err != nil { return nil, err @@ -1282,13 +1290,22 @@ func (r *Remote) list(ctx context.Context, o *ListOptions) (rfs []*plumbing.Refe } var resultRefs []*plumbing.Reference - err = refs.ForEach(func(ref *plumbing.Reference) error { - resultRefs = append(resultRefs, ref) - return nil - }) - if err != nil { - return nil, err + if o.PeelingOption == AppendPeeled || o.PeelingOption == IgnorePeeled { + err = refs.ForEach(func(ref *plumbing.Reference) error { + resultRefs = append(resultRefs, ref) + return nil + }) + if err != nil { + return nil, err + } } + + if o.PeelingOption == AppendPeeled || o.PeelingOption == OnlyPeeled { + for k, v := range ar.Peeled { + resultRefs = append(resultRefs, plumbing.NewReferenceFromStrings(k+"^{}", v.String())) + } + } + return resultRefs, nil } diff --git a/remote_test.go b/remote_test.go index 751c89a1b..164e4e509 100644 --- a/remote_test.go +++ b/remote_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "time" "github.com/go-git/go-git/v5/config" @@ -865,6 +866,7 @@ func (s *RemoteSuite) TestPushForceWithLease_success(c *C) { c.Assert(sto.SetReference(newCommit), IsNil) ref, err := sto.Reference("refs/heads/branch") + c.Assert(err, IsNil) c.Log(ref.String()) url := dstFs.Root() @@ -1210,6 +1212,41 @@ func (s *RemoteSuite) TestList(c *C) { } } +func (s *RemoteSuite) TestListPeeling(c *C) { + remote := NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{"https://github.com/git-fixtures/tags.git"}, + }) + + for _, tc := range []struct { + peelingOption PeelingOption + expectPeeled bool + expectNonPeeled bool + }{ + {peelingOption: AppendPeeled, expectPeeled: true, expectNonPeeled: true}, + {peelingOption: IgnorePeeled, expectPeeled: false, expectNonPeeled: true}, + {peelingOption: OnlyPeeled, expectPeeled: true, expectNonPeeled: false}, + } { + refs, err := remote.List(&ListOptions{ + PeelingOption: tc.peelingOption, + }) + c.Assert(err, IsNil) + c.Assert(len(refs) > 0, Equals, true) + + foundPeeled, foundNonPeeled := false, false + for _, ref := range refs { + if strings.HasSuffix(ref.Name().String(), peeledSuffix) { + foundPeeled = true + } else { + foundNonPeeled = true + } + } + + c.Assert(foundPeeled, Equals, tc.expectPeeled) + c.Assert(foundNonPeeled, Equals, tc.expectNonPeeled) + } +} + func (s *RemoteSuite) TestListTimeout(c *C) { remote := NewRemote(memory.NewStorage(), &config.RemoteConfig{ Name: DefaultRemoteName, From 7d183c99d041ecf5f98d6b4c7eb25449e21a5153 Mon Sep 17 00:00:00 2001 From: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> Date: Thu, 4 May 2023 07:33:19 +1000 Subject: [PATCH 15/31] storage: filesystem, Populate index before use. Fixes #148 Signed-off-by: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> --- storage/filesystem/object.go | 10 ++++++++++ storage/filesystem/object_test.go | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 21667fa5a..846a7b860 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -537,14 +537,21 @@ func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, pl return plumbing.ZeroHash, plumbing.ZeroHash, -1 } +// HashesWithPrefix returns all objects with a hash that starts with a prefix by searching for +// them in the packfile and the git object directories. func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error) { hashes, err := s.dir.ObjectsWithPrefix(prefix) if err != nil { return nil, err } + seen := hashListAsMap(hashes) + // TODO: This could be faster with some idxfile changes, // or diving into the packfile. + if err := s.requireIndex(); err != nil { + return nil, err + } for _, index := range s.index { ei, err := index.Entries() if err != nil { @@ -558,6 +565,9 @@ func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error) return nil, err } if bytes.HasPrefix(e.Hash[:], prefix) { + if _, ok := seen[e.Hash]; ok { + continue + } hashes = append(hashes, e.Hash) } } diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 19a791464..b5f137f29 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -406,6 +406,21 @@ func (s *FsSuite) TestHashesWithPrefix(c *C) { c.Assert(hashes[0].String(), Equals, "f3dfe29d268303fc6e1bbce268605fc99573406e") } +func (s *FsSuite) TestHashesWithPrefixFromPackfile(c *C) { + // Same setup as TestGetFromPackfile + fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) + + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + // Only pass the first 8 bytes + hashes, err := o.HashesWithPrefix(expected[:8]) + c.Assert(err, IsNil) + c.Assert(hashes, HasLen, 1) + c.Assert(hashes[0], Equals, expected) + }) +} + func BenchmarkPackfileIter(b *testing.B) { defer fixtures.Clean() From c2a93140c4d2a7df5666c7e436d8f1cb337a579d Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Thu, 13 Apr 2023 12:49:53 +0530 Subject: [PATCH 16/31] plumbing/transport: add ProxyOptions to specify proxy details Signed-off-by: Sanskar Jaiswal --- options.go | 11 ++++++++++- plumbing/transport/common.go | 31 +++++++++++++++++++++++++++++++ remote.go | 17 +++++++++-------- repository.go | 1 + 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/options.go b/options.go index 04c83de61..84744c745 100644 --- a/options.go +++ b/options.go @@ -73,6 +73,8 @@ type CloneOptions struct { InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte + // ProxyOptions provides info required for connecting to a proxy. + ProxyOptions transport.ProxyOptions } // Validate validates the fields and sets the default values. @@ -124,6 +126,8 @@ type PullOptions struct { InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte + // ProxyOptions provides info required for connecting to a proxy. + ProxyOptions transport.ProxyOptions } // Validate validates the fields and sets the default values. @@ -180,6 +184,8 @@ type FetchOptions struct { InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte + // ProxyOptions provides info required for connecting to a proxy. + ProxyOptions transport.ProxyOptions } // Validate validates the fields and sets the default values. @@ -243,6 +249,8 @@ type PushOptions struct { Options map[string]string // Atomic sets option to be an atomic push Atomic bool + // ProxyOptions provides info required for connecting to a proxy. + ProxyOptions transport.ProxyOptions } // ForceWithLease sets fields on the lease @@ -624,10 +632,11 @@ type ListOptions struct { InsecureSkipTLS bool // CABundle specify additional ca bundle with system cert pool CABundle []byte - // PeelingOption defines how peeled objects are handled during a // remote list. PeelingOption PeelingOption + // ProxyOptions provides info required for connecting to a proxy. + ProxyOptions transport.ProxyOptions } // PeelingOption represents the different ways to handle peeled references. diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index a2a78f028..89bd3d528 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -116,6 +116,37 @@ type Endpoint struct { InsecureSkipTLS bool // CaBundle specify additional ca bundle with system cert pool CaBundle []byte + // Proxy provides info required for connecting to a proxy. + Proxy ProxyOptions +} + +type ProxyOptions struct { + URL string + Username string + Password string +} + +func (o *ProxyOptions) Validate() error { + if o.URL != "" { + _, err := url.Parse(o.URL) + return err + } + return nil +} + +func (o *ProxyOptions) FullURL() (*url.URL, error) { + proxyURL, err := url.Parse(o.URL) + if err != nil { + return nil, err + } + if o.Username != "" { + if o.Password != "" { + proxyURL.User = url.UserPassword(o.Username, o.Password) + } else { + proxyURL.User = url.User(o.Username) + } + } + return proxyURL, nil } var defaultPorts = map[string]int{ diff --git a/remote.go b/remote.go index ff72bdf36..df2649112 100644 --- a/remote.go +++ b/remote.go @@ -113,7 +113,7 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) (err error) { o.RemoteURL = r.c.URLs[0] } - s, err := newSendPackSession(o.RemoteURL, o.Auth, o.InsecureSkipTLS, o.CABundle) + s, err := newSendPackSession(o.RemoteURL, o.Auth, o.InsecureSkipTLS, o.CABundle, o.ProxyOptions) if err != nil { return err } @@ -414,7 +414,7 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (sto storer.Referen o.RemoteURL = r.c.URLs[0] } - s, err := newUploadPackSession(o.RemoteURL, o.Auth, o.InsecureSkipTLS, o.CABundle) + s, err := newUploadPackSession(o.RemoteURL, o.Auth, o.InsecureSkipTLS, o.CABundle, o.ProxyOptions) if err != nil { return nil, err } @@ -511,8 +511,8 @@ func depthChanged(before []plumbing.Hash, s storage.Storer) (bool, error) { return false, nil } -func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, cabundle []byte) (transport.UploadPackSession, error) { - c, ep, err := newClient(url, auth, insecure, cabundle) +func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, cabundle []byte, proxyOpts transport.ProxyOptions) (transport.UploadPackSession, error) { + c, ep, err := newClient(url, insecure, cabundle, proxyOpts) if err != nil { return nil, err } @@ -520,8 +520,8 @@ func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, return c.NewUploadPackSession(ep, auth) } -func newSendPackSession(url string, auth transport.AuthMethod, insecure bool, cabundle []byte) (transport.ReceivePackSession, error) { - c, ep, err := newClient(url, auth, insecure, cabundle) +func newSendPackSession(url string, auth transport.AuthMethod, insecure bool, cabundle []byte, proxyOpts transport.ProxyOptions) (transport.ReceivePackSession, error) { + c, ep, err := newClient(url, insecure, cabundle, proxyOpts) if err != nil { return nil, err } @@ -529,13 +529,14 @@ func newSendPackSession(url string, auth transport.AuthMethod, insecure bool, ca return c.NewReceivePackSession(ep, auth) } -func newClient(url string, auth transport.AuthMethod, insecure bool, cabundle []byte) (transport.Transport, *transport.Endpoint, error) { +func newClient(url string, insecure bool, cabundle []byte, proxyOpts transport.ProxyOptions) (transport.Transport, *transport.Endpoint, error) { ep, err := transport.NewEndpoint(url) if err != nil { return nil, nil, err } ep.InsecureSkipTLS = insecure ep.CaBundle = cabundle + ep.Proxy = proxyOpts c, err := client.NewClient(ep) if err != nil { @@ -1267,7 +1268,7 @@ func (r *Remote) list(ctx context.Context, o *ListOptions) (rfs []*plumbing.Refe return nil, ErrEmptyUrls } - s, err := newUploadPackSession(r.c.URLs[0], o.Auth, o.InsecureSkipTLS, o.CABundle) + s, err := newUploadPackSession(r.c.URLs[0], o.Auth, o.InsecureSkipTLS, o.CABundle, o.ProxyOptions) if err != nil { return nil, err } diff --git a/repository.go b/repository.go index be5f140d7..287b597bb 100644 --- a/repository.go +++ b/repository.go @@ -873,6 +873,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { RemoteName: o.RemoteName, InsecureSkipTLS: o.InsecureSkipTLS, CABundle: o.CABundle, + ProxyOptions: o.ProxyOptions, }, o.ReferenceName) if err != nil { return err From 223727feb195642234a600040b12a2d3597d0989 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Thu, 13 Apr 2023 16:29:13 +0530 Subject: [PATCH 17/31] plumbing: transport/ssh, add support for custom proxy URLs Signed-off-by: Sanskar Jaiswal --- plumbing/transport/ssh/common.go | 30 ++++- .../transport/ssh/internal/test/proxy_test.go | 113 ++++++++++++++++++ .../transport/ssh/internal/test/test_utils.go | 83 +++++++++++++ plumbing/transport/ssh/proxy_test.go | 72 +++++++++-- plumbing/transport/ssh/upload_pack_test.go | 3 +- 5 files changed, 287 insertions(+), 14 deletions(-) create mode 100644 plumbing/transport/ssh/internal/test/proxy_test.go create mode 100644 plumbing/transport/ssh/internal/test/test_utils.go diff --git a/plumbing/transport/ssh/common.go b/plumbing/transport/ssh/common.go index e06958a3b..6617d9b71 100644 --- a/plumbing/transport/ssh/common.go +++ b/plumbing/transport/ssh/common.go @@ -4,6 +4,7 @@ package ssh import ( "context" "fmt" + "net" "reflect" "strconv" "strings" @@ -139,7 +140,7 @@ func (c *command) connect() error { overrideConfig(c.config, config) - c.client, err = dial("tcp", hostWithPort, config) + c.client, err = dial("tcp", hostWithPort, c.endpoint.Proxy, config) if err != nil { return err } @@ -154,7 +155,7 @@ func (c *command) connect() error { return nil } -func dial(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { +func dial(network, addr string, proxyOpts transport.ProxyOptions, config *ssh.ClientConfig) (*ssh.Client, error) { var ( ctx = context.Background() cancel context.CancelFunc @@ -166,10 +167,33 @@ func dial(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { } defer cancel() - conn, err := proxy.Dial(ctx, network, addr) + var conn net.Conn + var err error + + if proxyOpts.URL != "" { + proxyUrl, err := proxyOpts.FullURL() + if err != nil { + return nil, err + } + dialer, err := proxy.FromURL(proxyUrl, proxy.Direct) + if err != nil { + return nil, err + } + + // Try to use a ContextDialer, but fall back to a Dialer if that goes south. + ctxDialer, ok := dialer.(proxy.ContextDialer) + if !ok { + return nil, fmt.Errorf("expected ssh proxy dialer to be of type %s; got %s", + reflect.TypeOf(ctxDialer), reflect.TypeOf(dialer)) + } + conn, err = ctxDialer.DialContext(ctx, "tcp", addr) + } else { + conn, err = proxy.Dial(ctx, network, addr) + } if err != nil { return nil, err } + c, chans, reqs, err := ssh.NewClientConn(conn, addr, config) if err != nil { return nil, err diff --git a/plumbing/transport/ssh/internal/test/proxy_test.go b/plumbing/transport/ssh/internal/test/proxy_test.go new file mode 100644 index 000000000..8baac2b57 --- /dev/null +++ b/plumbing/transport/ssh/internal/test/proxy_test.go @@ -0,0 +1,113 @@ +package test + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/armon/go-socks5" + "github.com/gliderlabs/ssh" + "github.com/go-git/go-git/v5/plumbing/transport" + ggssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + + fixtures "github.com/go-git/go-git-fixtures/v4" + stdssh "golang.org/x/crypto/ssh" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type ProxyEnvSuite struct { + fixtures.Suite + port int + base string +} + +var _ = Suite(&ProxyEnvSuite{}) + +var socksProxiedRequests int32 + +// This test tests proxy support via an env var, i.e. `ALL_PROXY`. +// Its located in a separate package because golang caches the value +// of proxy env vars leading to misleading/unexpected test results. +func (s *ProxyEnvSuite) TestCommand(c *C) { + socksListener, err := net.Listen("tcp", "localhost:0") + c.Assert(err, IsNil) + + socksServer, err := socks5.New(&socks5.Config{ + Rules: TestProxyRule{}, + }) + c.Assert(err, IsNil) + go func() { + socksServer.Serve(socksListener) + }() + socksProxyAddr := fmt.Sprintf("socks5://localhost:%d", socksListener.Addr().(*net.TCPAddr).Port) + os.Setenv("ALL_PROXY", socksProxyAddr) + defer os.Unsetenv("ALL_PROXY") + + sshListener, err := net.Listen("tcp", "localhost:0") + c.Assert(err, IsNil) + sshServer := &ssh.Server{Handler: HandlerSSH} + go func() { + log.Fatal(sshServer.Serve(sshListener)) + }() + + s.port = sshListener.Addr().(*net.TCPAddr).Port + s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) + c.Assert(err, IsNil) + + ggssh.DefaultAuthBuilder = func(user string) (ggssh.AuthMethod, error) { + return &ggssh.Password{User: user}, nil + } + + ep := s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + c.Assert(err, IsNil) + + client := ggssh.NewClient(&stdssh.ClientConfig{ + HostKeyCallback: stdssh.InsecureIgnoreHostKey(), + }) + r, err := client.NewUploadPackSession(ep, nil) + c.Assert(err, IsNil) + defer func() { c.Assert(r.Close(), IsNil) }() + + info, err := r.AdvertisedReferences() + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed := atomic.LoadInt32(&socksProxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} + +func (s *ProxyEnvSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *transport.Endpoint { + fs := f.DotGit() + + err := fixtures.EnsureIsBare(fs) + c.Assert(err, IsNil) + + path := filepath.Join(s.base, name) + err = os.Rename(fs.Root(), path) + c.Assert(err, IsNil) + + return s.newEndpoint(c, name) +} + +func (s *ProxyEnvSuite) newEndpoint(c *C, name string) *transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf( + "ssh://git@localhost:%d/%s/%s", s.port, filepath.ToSlash(s.base), name, + )) + + c.Assert(err, IsNil) + return ep +} + +type TestProxyRule struct{} + +func (dr TestProxyRule) Allow(ctx context.Context, req *socks5.Request) (context.Context, bool) { + atomic.AddInt32(&socksProxiedRequests, 1) + return ctx, true +} diff --git a/plumbing/transport/ssh/internal/test/test_utils.go b/plumbing/transport/ssh/internal/test/test_utils.go new file mode 100644 index 000000000..c3797b1d2 --- /dev/null +++ b/plumbing/transport/ssh/internal/test/test_utils.go @@ -0,0 +1,83 @@ +package test + +import ( + "fmt" + "io" + "os/exec" + "runtime" + "strings" + "sync" + + "github.com/gliderlabs/ssh" +) + +func HandlerSSH(s ssh.Session) { + cmd, stdin, stderr, stdout, err := buildCommand(s.Command()) + if err != nil { + fmt.Println(err) + return + } + + if err := cmd.Start(); err != nil { + fmt.Println(err) + return + } + + go func() { + defer stdin.Close() + io.Copy(stdin, s) + }() + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + io.Copy(s.Stderr(), stderr) + }() + + go func() { + defer wg.Done() + io.Copy(s, stdout) + }() + + wg.Wait() + + if err := cmd.Wait(); err != nil { + return + } + +} + +func buildCommand(c []string) (cmd *exec.Cmd, stdin io.WriteCloser, stderr, stdout io.ReadCloser, err error) { + if len(c) != 2 { + err = fmt.Errorf("invalid command") + return + } + + // fix for Windows environments + var path string + if runtime.GOOS == "windows" { + path = strings.Replace(c[1], "/C:/", "C:/", 1) + } else { + path = c[1] + } + + cmd = exec.Command(c[0], path) + stdout, err = cmd.StdoutPipe() + if err != nil { + return + } + + stdin, err = cmd.StdinPipe() + if err != nil { + return + } + + stderr, err = cmd.StderrPipe() + if err != nil { + return + } + + return +} diff --git a/plumbing/transport/ssh/proxy_test.go b/plumbing/transport/ssh/proxy_test.go index 3caf1ff1d..2fab8512b 100644 --- a/plumbing/transport/ssh/proxy_test.go +++ b/plumbing/transport/ssh/proxy_test.go @@ -1,36 +1,88 @@ package ssh import ( + "context" "fmt" + "io/ioutil" "log" "net" "os" + "sync/atomic" "github.com/armon/go-socks5" + "github.com/gliderlabs/ssh" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/ssh/internal/test" + + fixtures "github.com/go-git/go-git-fixtures/v4" + stdssh "golang.org/x/crypto/ssh" . "gopkg.in/check.v1" ) type ProxySuite struct { - UploadPackSuite + u UploadPackSuite + fixtures.Suite } var _ = Suite(&ProxySuite{}) -func (s *ProxySuite) SetUpSuite(c *C) { - s.UploadPackSuite.SetUpSuite(c) +var socksProxiedRequests int32 - l, err := net.Listen("tcp", "localhost:0") +func (s *ProxySuite) TestCommand(c *C) { + socksListener, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) - server, err := socks5.New(&socks5.Config{}) + socksServer, err := socks5.New(&socks5.Config{ + AuthMethods: []socks5.Authenticator{socks5.UserPassAuthenticator{ + Credentials: socks5.StaticCredentials{ + "user": "pass", + }, + }}, + Rules: TestProxyRule{}, + }) c.Assert(err, IsNil) + go func() { + socksServer.Serve(socksListener) + }() + socksProxyAddr := fmt.Sprintf("socks5://localhost:%d", socksListener.Addr().(*net.TCPAddr).Port) - port := l.Addr().(*net.TCPAddr).Port - - err = os.Setenv("ALL_PROXY", fmt.Sprintf("socks5://localhost:%d", port)) + sshListener, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) - + sshServer := &ssh.Server{Handler: test.HandlerSSH} go func() { - log.Fatal(server.Serve(l)) + log.Fatal(sshServer.Serve(sshListener)) }() + + s.u.port = sshListener.Addr().(*net.TCPAddr).Port + s.u.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.u.port)) + c.Assert(err, IsNil) + + DefaultAuthBuilder = func(user string) (AuthMethod, error) { + return &Password{User: user}, nil + } + + ep := s.u.prepareRepository(c, fixtures.Basic().One(), "basic.git") + c.Assert(err, IsNil) + ep.Proxy = transport.ProxyOptions{ + URL: socksProxyAddr, + Username: "user", + Password: "pass", + } + + runner := runner{ + config: &stdssh.ClientConfig{ + HostKeyCallback: stdssh.InsecureIgnoreHostKey(), + }, + } + _, err = runner.Command(transport.UploadPackServiceName, ep, nil) + c.Assert(err, IsNil) + proxyUsed := atomic.LoadInt32(&socksProxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} + +type TestProxyRule struct{} + +func (dr TestProxyRule) Allow(ctx context.Context, req *socks5.Request) (context.Context, bool) { + atomic.AddInt32(&socksProxiedRequests, 1) + return ctx, true } diff --git a/plumbing/transport/ssh/upload_pack_test.go b/plumbing/transport/ssh/upload_pack_test.go index f172feeda..fafff485d 100644 --- a/plumbing/transport/ssh/upload_pack_test.go +++ b/plumbing/transport/ssh/upload_pack_test.go @@ -14,6 +14,7 @@ import ( "sync" "github.com/go-git/go-git/v5/plumbing/transport" + testutils "github.com/go-git/go-git/v5/plumbing/transport/ssh/internal/test" "github.com/go-git/go-git/v5/plumbing/transport/test" "github.com/gliderlabs/ssh" @@ -57,7 +58,7 @@ func (s *UploadPackSuite) SetUpSuite(c *C) { s.UploadPackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") s.UploadPackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") - server := &ssh.Server{Handler: handlerSSH} + server := &ssh.Server{Handler: testutils.HandlerSSH} for _, opt := range s.opts { opt(server) } From 399b1ec2d598b7950816727b8d92e8580553372c Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Tue, 18 Apr 2023 16:31:58 +0530 Subject: [PATCH 18/31] plumbing: transport/http, refactor transport to cache underlying transport objects Refactor the in-built http transport to cache the underlying http transport objects mapped to its specific options for each Git transport object. This lets us reuse the transport for a specific set of configurations as recommended. (ref: https://pkg.go.dev/net/http#Transport) If there are no transport specific options provided, the default transport is used. Signed-off-by: Sanskar Jaiswal --- go.mod | 1 + go.sum | 2 + plumbing/transport/client/client.go | 32 ------ plumbing/transport/http/common.go | 147 ++++++++++++++++++++++-- plumbing/transport/http/common_test.go | 54 +++++++++ plumbing/transport/http/receive_pack.go | 2 +- plumbing/transport/http/transport.go | 38 ++++++ plumbing/transport/http/upload_pack.go | 2 +- 8 files changed, 233 insertions(+), 45 deletions(-) create mode 100644 plumbing/transport/http/transport.go diff --git a/go.mod b/go.mod index 27b74a36a..9bcad627a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/go-cmp v0.5.9 github.com/imdario/mergo v0.3.15 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 diff --git a/go.sum b/go.sum index f4fb14682..ccc6597f0 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8 github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= diff --git a/plumbing/transport/client/client.go b/plumbing/transport/client/client.go index 20c3d0560..1948c2301 100644 --- a/plumbing/transport/client/client.go +++ b/plumbing/transport/client/client.go @@ -3,10 +3,7 @@ package client import ( - "crypto/tls" - "crypto/x509" "fmt" - gohttp "net/http" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/file" @@ -24,14 +21,6 @@ var Protocols = map[string]transport.Transport{ "file": file.DefaultClient, } -var insecureClient = http.NewClient(&gohttp.Client{ - Transport: &gohttp.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, -}) - // InstallProtocol adds or modifies an existing protocol. func InstallProtocol(scheme string, c transport.Transport) { if c == nil { @@ -50,27 +39,6 @@ func NewClient(endpoint *transport.Endpoint) (transport.Transport, error) { } func getTransport(endpoint *transport.Endpoint) (transport.Transport, error) { - if endpoint.Protocol == "https" { - if endpoint.InsecureSkipTLS { - return insecureClient, nil - } - - if len(endpoint.CaBundle) != 0 { - rootCAs, _ := x509.SystemCertPool() - if rootCAs == nil { - rootCAs = x509.NewCertPool() - } - rootCAs.AppendCertsFromPEM(endpoint.CaBundle) - return http.NewClient(&gohttp.Client{ - Transport: &gohttp.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: rootCAs, - }, - }, - }), nil - } - } - f, ok := Protocols[endpoint.Protocol] if !ok { return nil, fmt.Errorf("unsupported scheme %q", endpoint.Protocol) diff --git a/plumbing/transport/http/common.go b/plumbing/transport/http/common.go index d57c0feef..530034105 100644 --- a/plumbing/transport/http/common.go +++ b/plumbing/transport/http/common.go @@ -4,16 +4,21 @@ package http import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "fmt" "net" "net/http" + "reflect" "strconv" "strings" + "sync" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/protocol/packp" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/golang/groupcache/lru" ) // it requires a bytes.Buffer, because we need to know the length @@ -74,40 +79,83 @@ func advertisedReferences(ctx context.Context, s *session, serviceName string) ( } type client struct { - c *http.Client + c *http.Client + transports *lru.Cache + m sync.RWMutex } -// DefaultClient is the default HTTP client, which uses `http.DefaultClient`. -var DefaultClient = NewClient(nil) +// ClientOptions holds user configurable options for the client. +type ClientOptions struct { + // CacheMaxEntries is the max no. of entries that the transport objects + // cache will hold at any given point of time. It must be a positive integer. + // Calling `client.addTransport()` after the cache has reached the specified + // size, will result in the least recently used transport getting deleted + // before the provided transport is added to the cache. + CacheMaxEntries int +} + +var ( + // defaultTransportCacheSize is the default capacity of the transport objects cache. + // Its value is 0 because transport caching is turned off by default and is an + // opt-in feature. + defaultTransportCacheSize = 0 + + // DefaultClient is the default HTTP client, which uses a net/http client configured + // with http.DefaultTransport. + DefaultClient = NewClient(nil) +) // NewClient creates a new client with a custom net/http client. // See `InstallProtocol` to install and override default http client. -// Unless a properly initialized client is given, it will fall back into -// `http.DefaultClient`. +// If the net/http client is nil or empty, it will use a net/http client configured +// with http.DefaultTransport. // // Note that for HTTP client cannot distinguish between private repositories and // unexistent repositories on GitHub. So it returns `ErrAuthorizationRequired` // for both. func NewClient(c *http.Client) transport.Transport { if c == nil { - return &client{http.DefaultClient} + c = &http.Client{ + Transport: http.DefaultTransport, + } } + return NewClientWithOptions(c, &ClientOptions{ + CacheMaxEntries: defaultTransportCacheSize, + }) +} - return &client{ +// NewClientWithOptions returns a new client configured with the provided net/http client +// and other custom options specific to the client. +// If the net/http client is nil or empty, it will use a net/http client configured +// with http.DefaultTransport. +func NewClientWithOptions(c *http.Client, opts *ClientOptions) transport.Transport { + if c == nil { + c = &http.Client{ + Transport: http.DefaultTransport, + } + } + cl := &client{ c: c, } + + if opts != nil { + if opts.CacheMaxEntries > 0 { + cl.transports = lru.New(opts.CacheMaxEntries) + } + } + return cl } func (c *client) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.UploadPackSession, error) { - return newUploadPackSession(c.c, ep, auth) + return newUploadPackSession(c, ep, auth) } func (c *client) NewReceivePackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.ReceivePackSession, error) { - return newReceivePackSession(c.c, ep, auth) + return newReceivePackSession(c, ep, auth) } type session struct { @@ -117,10 +165,87 @@ type session struct { advRefs *packp.AdvRefs } -func newSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (*session, error) { +func transportWithInsecureTLS(transport *http.Transport) { + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = true +} + +func transportWithCABundle(transport *http.Transport, caBundle []byte) error { + rootCAs, err := x509.SystemCertPool() + if err != nil { + return err + } + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + rootCAs.AppendCertsFromPEM(caBundle) + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = rootCAs + return nil +} + +func configureTransport(transport *http.Transport, ep *transport.Endpoint) error { + if len(ep.CaBundle) > 0 { + if err := transportWithCABundle(transport, ep.CaBundle); err != nil { + return err + } + } + if ep.InsecureSkipTLS { + transportWithInsecureTLS(transport) + } + return nil +} + +func newSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (*session, error) { + var httpClient *http.Client + + // We need to configure the http transport if there are transport specific + // options present in the endpoint. + if len(ep.CaBundle) > 0 || ep.InsecureSkipTLS { + var transport *http.Transport + // if the client wasn't configured to have a cache for transports then just configure + // the transport and use it directly, otherwise try to use the cache. + if c.transports == nil { + tr, ok := c.c.Transport.(*http.Transport) + if !ok { + return nil, fmt.Errorf("expected underlying client transport to be of type: %s; got: %s", + reflect.TypeOf(transport), reflect.TypeOf(c.c.Transport)) + } + + transport = tr.Clone() + configureTransport(transport, ep) + } else { + transportOpts := transportOptions{ + caBundle: string(ep.CaBundle), + insecureSkipTLS: ep.InsecureSkipTLS, + } + var found bool + transport, found = c.fetchTransport(transportOpts) + + if !found { + transport = c.c.Transport.(*http.Transport).Clone() + configureTransport(transport, ep) + c.addTransport(transportOpts, transport) + } + } + + httpClient = &http.Client{ + Transport: transport, + CheckRedirect: c.c.CheckRedirect, + Jar: c.c.Jar, + Timeout: c.c.Timeout, + } + } else { + httpClient = c.c + } + s := &session{ auth: basicAuthFromEndpoint(ep), - client: c, + client: httpClient, endpoint: ep, } if auth != nil { diff --git a/plumbing/transport/http/common_test.go b/plumbing/transport/http/common_test.go index 4122e6279..41188e677 100644 --- a/plumbing/transport/http/common_test.go +++ b/plumbing/transport/http/common_test.go @@ -91,6 +91,60 @@ func (s *ClientSuite) TestNewHTTPError40x(c *C) { "unexpected client error.*") } +func (s *ClientSuite) Test_newSession(c *C) { + cl := NewClientWithOptions(nil, &ClientOptions{ + CacheMaxEntries: 2, + }).(*client) + + insecureEP := s.Endpoint + insecureEP.InsecureSkipTLS = true + session, err := newSession(cl, insecureEP, nil) + c.Assert(err, IsNil) + + sessionTransport := session.client.Transport.(*http.Transport) + c.Assert(sessionTransport.TLSClientConfig.InsecureSkipVerify, Equals, true) + t, ok := cl.fetchTransport(transportOptions{ + insecureSkipTLS: true, + }) + // transport should be cached. + c.Assert(ok, Equals, true) + // cached transport should be the one that's used. + c.Assert(sessionTransport, Equals, t) + + caEndpoint := insecureEP + caEndpoint.CaBundle = []byte("this is the way") + session, err = newSession(cl, caEndpoint, nil) + c.Assert(err, IsNil) + + sessionTransport = session.client.Transport.(*http.Transport) + c.Assert(sessionTransport.TLSClientConfig.InsecureSkipVerify, Equals, true) + c.Assert(sessionTransport.TLSClientConfig.RootCAs, NotNil) + t, ok = cl.fetchTransport(transportOptions{ + insecureSkipTLS: true, + caBundle: "this is the way", + }) + // transport should be cached. + c.Assert(ok, Equals, true) + // cached transport should be the one that's used. + c.Assert(sessionTransport, Equals, t) + + session, err = newSession(cl, caEndpoint, nil) + c.Assert(err, IsNil) + sessionTransport = session.client.Transport.(*http.Transport) + // transport that's going to be used should be cached already. + c.Assert(sessionTransport, Equals, t) + // no new transport got cached. + c.Assert(cl.transports.Len(), Equals, 2) + + // if the cache does not exist, the transport should still be correctly configured. + cl.transports = nil + session, err = newSession(cl, insecureEP, nil) + c.Assert(err, IsNil) + + sessionTransport = session.client.Transport.(*http.Transport) + c.Assert(sessionTransport.TLSClientConfig.InsecureSkipVerify, Equals, true) +} + func (s *ClientSuite) testNewHTTPError(c *C, code int, msg string) { req, _ := http.NewRequest("GET", "foo", nil) res := &http.Response{ diff --git a/plumbing/transport/http/receive_pack.go b/plumbing/transport/http/receive_pack.go index 4d14ff21e..4387ecffe 100644 --- a/plumbing/transport/http/receive_pack.go +++ b/plumbing/transport/http/receive_pack.go @@ -19,7 +19,7 @@ type rpSession struct { *session } -func newReceivePackSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { +func newReceivePackSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { s, err := newSession(c, ep, auth) return &rpSession{s}, err } diff --git a/plumbing/transport/http/transport.go b/plumbing/transport/http/transport.go new file mode 100644 index 000000000..cd6787ad3 --- /dev/null +++ b/plumbing/transport/http/transport.go @@ -0,0 +1,38 @@ +package http + +import ( + "net/http" +) + +// transportOptions contains transport specific configuration. +type transportOptions struct { + insecureSkipTLS bool + // []byte is not comparable. + caBundle string +} + +func (c *client) addTransport(opts transportOptions, transport *http.Transport) { + c.m.Lock() + c.transports.Add(opts, transport) + c.m.Unlock() +} + +func (c *client) removeTransport(opts transportOptions) { + c.m.Lock() + c.transports.Remove(opts) + c.m.Unlock() +} + +func (c *client) fetchTransport(opts transportOptions) (*http.Transport, bool) { + c.m.RLock() + t, ok := c.transports.Get(opts) + c.m.RUnlock() + if !ok { + return nil, false + } + transport, ok := t.(*http.Transport) + if !ok { + return nil, false + } + return transport, true +} diff --git a/plumbing/transport/http/upload_pack.go b/plumbing/transport/http/upload_pack.go index e735b3d7c..4f851459f 100644 --- a/plumbing/transport/http/upload_pack.go +++ b/plumbing/transport/http/upload_pack.go @@ -19,7 +19,7 @@ type upSession struct { *session } -func newUploadPackSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { +func newUploadPackSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { s, err := newSession(c, ep, auth) return &upSession{s}, err } From a830187d90a6bc36f9466c075ed49076f591efa9 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Fri, 28 Apr 2023 17:17:00 +0530 Subject: [PATCH 19/31] plumbing: transport/http, add support for custom proxy URLs Add support for custom HTTP and HTTPS proxies for each session. The tests require server certificate and a matching private key to be able to run a TLS server and test HTTPS proxy functionality. The cert and the key are stored in `plumbing/transport/http/testdata/certs` and were generated using the following command: `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout server.key -out server.crt` Note: All details were left empty, except for the FQDN for which example.com was used. Signed-off-by: Sanskar Jaiswal --- go.mod | 1 + go.sum | 5 + plumbing/transport/http/common.go | 22 +++- .../http/internal/test/proxy_test.go | 72 +++++++++++ .../http/internal/test/test_utils.go | 43 +++++++ plumbing/transport/http/proxy_test.go | 119 ++++++++++++++++++ .../transport/http/testdata/certs/server.crt | 22 ++++ .../transport/http/testdata/certs/server.key | 28 +++++ plumbing/transport/http/transport.go | 2 + 9 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 plumbing/transport/http/internal/test/proxy_test.go create mode 100644 plumbing/transport/http/internal/test/test_utils.go create mode 100644 plumbing/transport/http/proxy_test.go create mode 100644 plumbing/transport/http/testdata/certs/server.crt create mode 100644 plumbing/transport/http/testdata/certs/server.key diff --git a/go.mod b/go.mod index 9bcad627a..f062b1a8d 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5 github.com/acomagu/bufpipe v1.0.4 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 github.com/emirpasic/gods v1.18.1 github.com/gliderlabs/ssh v0.3.5 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 diff --git a/go.sum b/go.sum index ccc6597f0..db7d92ba3 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= +github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= @@ -55,6 +59,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/plumbing/transport/http/common.go b/plumbing/transport/http/common.go index 530034105..f9b7a0e59 100644 --- a/plumbing/transport/http/common.go +++ b/plumbing/transport/http/common.go @@ -9,6 +9,7 @@ import ( "fmt" "net" "net/http" + "net/url" "reflect" "strconv" "strings" @@ -188,6 +189,10 @@ func transportWithCABundle(transport *http.Transport, caBundle []byte) error { return nil } +func transportWithProxy(transport *http.Transport, proxyURL *url.URL) { + transport.Proxy = http.ProxyURL(proxyURL) +} + func configureTransport(transport *http.Transport, ep *transport.Endpoint) error { if len(ep.CaBundle) > 0 { if err := transportWithCABundle(transport, ep.CaBundle); err != nil { @@ -197,6 +202,14 @@ func configureTransport(transport *http.Transport, ep *transport.Endpoint) error if ep.InsecureSkipTLS { transportWithInsecureTLS(transport) } + + if ep.Proxy.URL != "" { + proxyURL, err := ep.Proxy.FullURL() + if err != nil { + return err + } + transportWithProxy(transport, proxyURL) + } return nil } @@ -205,7 +218,7 @@ func newSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (* // We need to configure the http transport if there are transport specific // options present in the endpoint. - if len(ep.CaBundle) > 0 || ep.InsecureSkipTLS { + if len(ep.CaBundle) > 0 || ep.InsecureSkipTLS || ep.Proxy.URL != "" { var transport *http.Transport // if the client wasn't configured to have a cache for transports then just configure // the transport and use it directly, otherwise try to use the cache. @@ -223,6 +236,13 @@ func newSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (* caBundle: string(ep.CaBundle), insecureSkipTLS: ep.InsecureSkipTLS, } + if ep.Proxy.URL != "" { + proxyURL, err := ep.Proxy.FullURL() + if err != nil { + return nil, err + } + transportOpts.proxyURL = *proxyURL + } var found bool transport, found = c.fetchTransport(transportOpts) diff --git a/plumbing/transport/http/internal/test/proxy_test.go b/plumbing/transport/http/internal/test/proxy_test.go new file mode 100644 index 000000000..6ae2943b0 --- /dev/null +++ b/plumbing/transport/http/internal/test/proxy_test.go @@ -0,0 +1,72 @@ +package test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + nethttp "net/http" + "os" + "sync/atomic" + "testing" + + "github.com/elazarl/goproxy" + + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + + . "gopkg.in/check.v1" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type ProxySuite struct{} + +var _ = Suite(&ProxySuite{}) + +var proxiedRequests int32 + +// This test tests proxy support via an env var, i.e. `HTTPS_PROXY`. +// Its located in a separate package because golang caches the value +// of proxy env vars leading to misleading/unexpected test results. +func (s *ProxySuite) TestAdvertisedReferences(c *C) { + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + SetupHTTPSProxy(proxy, &proxiedRequests) + httpsListener, err := net.Listen("tcp", ":0") + c.Assert(err, IsNil) + defer httpsListener.Close() + httpProxyAddr := fmt.Sprintf("localhost:%d", httpsListener.Addr().(*net.TCPAddr).Port) + + proxyServer := nethttp.Server{ + Addr: httpProxyAddr, + Handler: proxy, + // Due to how golang manages http/2 when provided with custom TLS config, + // servers and clients running in the same process leads to issues. + // Ref: https://github.com/golang/go/issues/21336 + TLSConfig: &tls.Config{ + NextProtos: []string{"http/1.1"}, + }, + } + go proxyServer.ServeTLS(httpsListener, "../../testdata/certs/server.crt", "../../testdata/certs/server.key") + defer proxyServer.Close() + os.Setenv("HTTPS_PROXY", fmt.Sprintf("https://user:pass@%s", httpProxyAddr)) + defer os.Unsetenv("HTTPS_PROXY") + + endpoint, err := transport.NewEndpoint("https://github.com/git-fixtures/basic.git") + c.Assert(err, IsNil) + endpoint.InsecureSkipTLS = true + + client := http.DefaultClient + session, err := client.NewUploadPackSession(endpoint, nil) + c.Assert(err, IsNil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + info, err := session.AdvertisedReferencesContext(ctx) + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed := atomic.LoadInt32(&proxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} diff --git a/plumbing/transport/http/internal/test/test_utils.go b/plumbing/transport/http/internal/test/test_utils.go new file mode 100644 index 000000000..6665fb3c6 --- /dev/null +++ b/plumbing/transport/http/internal/test/test_utils.go @@ -0,0 +1,43 @@ +package test + +import ( + "encoding/base64" + "strings" + "sync/atomic" + + "github.com/elazarl/goproxy" +) + +func SetupHTTPSProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) { + var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + if strings.Contains(host, "github.com") { + user, pass, _ := ParseBasicAuth(ctx.Req.Header.Get("Proxy-Authorization")) + if user != "user" || pass != "pass" { + return goproxy.RejectConnect, host + } + atomic.AddInt32(proxiedRequests, 1) + return goproxy.OkConnect, host + } + // Reject if it isn't our request. + return goproxy.RejectConnect, host + } + proxy.OnRequest().HandleConnect(proxyHandler) +} + +// adapted from https://github.com/golang/go/blob/2ef70d9d0f98832c8103a7968b195e560a8bb262/src/net/http/request.go#L959 +func ParseBasicAuth(auth string) (username, password string, ok bool) { + const prefix = "Basic " + if len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) { + return "", "", false + } + c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) + if err != nil { + return "", "", false + } + cs := string(c) + username, password, ok = strings.Cut(cs, ":") + if !ok { + return "", "", false + } + return username, password, true +} diff --git a/plumbing/transport/http/proxy_test.go b/plumbing/transport/http/proxy_test.go new file mode 100644 index 000000000..f3024da92 --- /dev/null +++ b/plumbing/transport/http/proxy_test.go @@ -0,0 +1,119 @@ +package http + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "strings" + "sync/atomic" + + "github.com/elazarl/goproxy" + fixtures "github.com/go-git/go-git-fixtures/v4" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http/internal/test" + + . "gopkg.in/check.v1" +) + +type ProxySuite struct { + u UploadPackSuite + fixtures.Suite +} + +var _ = Suite(&ProxySuite{}) + +var proxiedRequests int32 + +func (s *ProxySuite) TestAdvertisedReferences(c *C) { + s.u.SetUpTest(c) + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + setupHTTPProxy(proxy, &proxiedRequests) + httpListener, err := net.Listen("tcp", ":0") + c.Assert(err, IsNil) + defer httpListener.Close() + + httpProxyAddr := fmt.Sprintf("http://localhost:%d", httpListener.Addr().(*net.TCPAddr).Port) + proxyServer := http.Server{ + Addr: httpProxyAddr, + Handler: proxy, + } + go proxyServer.Serve(httpListener) + defer proxyServer.Close() + + endpoint := s.u.prepareRepository(c, fixtures.Basic().One(), "basic.git") + endpoint.Proxy = transport.ProxyOptions{ + URL: httpProxyAddr, + Username: "user", + Password: "pass", + } + + s.u.Client = NewClient(nil) + session, err := s.u.Client.NewUploadPackSession(endpoint, nil) + c.Assert(err, IsNil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + info, err := session.AdvertisedReferencesContext(ctx) + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed := atomic.LoadInt32(&proxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) + + atomic.StoreInt32(&proxiedRequests, 0) + test.SetupHTTPSProxy(proxy, &proxiedRequests) + httpsListener, err := net.Listen("tcp", ":0") + c.Assert(err, IsNil) + defer httpsListener.Close() + httpsProxyAddr := fmt.Sprintf("https://localhost:%d", httpsListener.Addr().(*net.TCPAddr).Port) + + tlsProxyServer := http.Server{ + Addr: httpsProxyAddr, + Handler: proxy, + // Due to how golang manages http/2 when provided with custom TLS config, + // servers and clients running in the same process leads to issues. + // Ref: https://github.com/golang/go/issues/21336 + TLSConfig: &tls.Config{ + NextProtos: []string{"http/1.1"}, + }, + } + go tlsProxyServer.ServeTLS(httpsListener, "testdata/certs/server.crt", "testdata/certs/server.key") + defer tlsProxyServer.Close() + + endpoint, err = transport.NewEndpoint("https://github.com/git-fixtures/basic.git") + c.Assert(err, IsNil) + endpoint.Proxy = transport.ProxyOptions{ + URL: httpsProxyAddr, + Username: "user", + Password: "pass", + } + endpoint.InsecureSkipTLS = true + + session, err = s.u.Client.NewUploadPackSession(endpoint, nil) + c.Assert(err, IsNil) + + info, err = session.AdvertisedReferencesContext(ctx) + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed = atomic.LoadInt32(&proxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} + +func setupHTTPProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) { + // The request is being forwarded to the local test git server in this handler. + var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + if strings.Contains(req.Host, "localhost") { + user, pass, _ := test.ParseBasicAuth(req.Header.Get("Proxy-Authorization")) + if user != "user" || pass != "pass" { + return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusUnauthorized, "") + } + atomic.AddInt32(proxiedRequests, 1) + return req, nil + } + // Reject if it isn't our request. + return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "") + } + proxy.OnRequest().Do(proxyHandler) +} diff --git a/plumbing/transport/http/testdata/certs/server.crt b/plumbing/transport/http/testdata/certs/server.crt new file mode 100644 index 000000000..9bdec2ce9 --- /dev/null +++ b/plumbing/transport/http/testdata/certs/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUWcuzUyG3EfGsXVUH0BAmnCJyNHswDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMDMwNzA3MTgwNloXDTI0MDMwNjA3MTgwNlowWTELMAkGA1UEBhMCQVUxEzAR +BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5 +IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAvyKX6vJXt1u+WBfBNJByFDAb7msdsk6SiPFlX5uyilaWmlRxvLo1 +GZMjjuQbs4wU6BAoZcgiELnsC9GSyxgrhk7NIW3ud/QD7s8ZxETxFLb0ur6tJj7/ +ETEcU/AKSl1FpeJbGHqGipYp5+0GU0zPDxRYqC2N3+fcGZPQbBwxb1f+MrBjWutb +3eNYTLdPH3W7RUqbunC1KZRJ8XOcU5XZ4qEaMkZYdz1QItxwPnpPuSZs53ga3TDF +zclpQcT8OH2JNwSI6bwlwFJ0Es06manw7XHmgd8anhix9FdsQYaTOW4kqh1iKQ/P +jPG50bdTUEqlOsaa+9av/qf+90npzt3xqQIDAQABo1MwUTAdBgNVHQ4EFgQUqTqb +q+jiJVgwftQS+YLcQWnvTuAwHwYDVR0jBBgwFoAUqTqbq+jiJVgwftQS+YLcQWnv +TuAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAVUaFSikxyCy1 +4P/ZZgeuR7vEJ5vWBxKPw/jFNZUFWy2Ag32w1BhrDwoYoc1Awg76QF2TqBQAhFNm +ek9aE+L83P/R2UhE9+LHnzwdMXt9HYOI1grONk2z3lMI1y4FCJBxHfGyC/XMoNgZ +qP7UdLgLGTIMN3O1Fww416Hn8BHzxN4o5ZEHJZ6QPMuN8OLk9oVu3yQIq/QcmSDH +GT2RiwT5IJWMUKK1UrV+y3/T9FwW2qqu+LX+coxjk7HgDWy3y66V9ahLBt8kONcr +qK0zutoQ5WPSmvnD2Nr0LVLGXEd7hbQNO7bgjO2YOBtnagUQJt72i/OmvZv8Mfnp +Bu6Qgl5hDw== +-----END CERTIFICATE----- diff --git a/plumbing/transport/http/testdata/certs/server.key b/plumbing/transport/http/testdata/certs/server.key new file mode 100644 index 000000000..9a0cd8f92 --- /dev/null +++ b/plumbing/transport/http/testdata/certs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC/Ipfq8le3W75Y +F8E0kHIUMBvuax2yTpKI8WVfm7KKVpaaVHG8ujUZkyOO5BuzjBToEChlyCIQuewL +0ZLLGCuGTs0hbe539APuzxnERPEUtvS6vq0mPv8RMRxT8ApKXUWl4lsYeoaKlinn +7QZTTM8PFFioLY3f59wZk9BsHDFvV/4ysGNa61vd41hMt08fdbtFSpu6cLUplEnx +c5xTldnioRoyRlh3PVAi3HA+ek+5JmzneBrdMMXNyWlBxPw4fYk3BIjpvCXAUnQS +zTqZqfDtceaB3xqeGLH0V2xBhpM5biSqHWIpD8+M8bnRt1NQSqU6xpr71q/+p/73 +SenO3fGpAgMBAAECggEAQUjenQhzv5Rdmpdajcq8vHqGP9Rki0/dK1tQpex3elsD +C+nGA5GSq46feaIeeCBjz7QdKE7Im+/1WUAXJLm3vCNUW5PB/UTixwIEKg7mTY4E +X3jbiZHA661boKv/x9C+BmAff2fyZonN/ILwQymcG+l2MtOEfzMh8baUXSjwFbhg +B08u4iXjee0y9I0CGMYWfasHLOIuhACCFKtqnvdQp8B82g8eSPhme5IjfPP8KZVr +00n6z8m00HVk6/yYJ8pVZ82j3T+wH6IqvlvaC320sbto8YXV6i8GWHaJumzU4/9s +IRm4459E+NmNcLNY/TCu89zsfrgNirN+qFfvJIOTxQKBgQDtME8s4UP0MhGuJ2lD +1HD64fAxMC6Xp/QSzY91Yn79UNssUUV+IwjuUnLIz3U8DBs/QETLm7CkNtI7h5m5 +dBdeBBzCRGxhe8WqRfvceu4s0zr08ZkDaKLjFsBSnBsXZhKAAuRqBjnGAoAiKgVa +WpEAug00ThhQjipSY9tO9NSBawKBgQDOSz+8m2HJFktEdSctKIB9DesqlAg7YCyy +dHzywP0/r7wEvsCN7xPgCT5g8JBkRaFvLLKgw7gMKAUx8V2iwizEoDCAs/pbTWji +uZwPC8lWtbkpBMQIaP4Wap+GyFQJKv1/qZduwpkwkj+ok+m3WwIW55VFGyLn3XGG +VcLZm83aOwKBgQDXXI/nXjqHVZb8HEjWD+Ttx4yB/Q+xIAzbrc3edap8c5guKzUA +DOulCTOz5bq65PsweTh970V6NVS6PKt12lUFRpKeSeZmtS2LJ7RCQ1RTWxAjK+MV +V0LfEt9ZouhuXH3bwcSICFMY2VhirOjjW2xhzo0Cuw4UxqDi4kxU6rSxNQKBgQCI +sn5KmV/jot0/QK40E0mJFEcHkM4foiwcGGqPZWiq4eUh89CefJTb+OQX0nCrsSQ3 +ChRXyTlU/NPsczcL2cVWiZt6PUihZZsh2cJaigHhbkuCrcDEneX4rrCE3IwrAwy1 +oohRAawG7nI2X8UYFbs9uDlGcKPhpvBKBtw13DM87wKBgE8fOiFoTph//6piU7dV +pN33UfhPcAFwsIzxAH6Ljo6BYx2hfPRCxI2g0wchk6ydbDecLgMwVgugdJZ6+tRf +P+YV3wEwPcWOvWby3+EmJh0cXUTl6ZMA+eR4pvCi6kf2xJf9dRmEeNNhOuzn9Y0J +cT9yhBFG4iejKP0iTwET1JKY +-----END PRIVATE KEY----- diff --git a/plumbing/transport/http/transport.go b/plumbing/transport/http/transport.go index cd6787ad3..052f3c8e2 100644 --- a/plumbing/transport/http/transport.go +++ b/plumbing/transport/http/transport.go @@ -2,6 +2,7 @@ package http import ( "net/http" + "net/url" ) // transportOptions contains transport specific configuration. @@ -9,6 +10,7 @@ type transportOptions struct { insecureSkipTLS bool // []byte is not comparable. caBundle string + proxyURL url.URL } func (c *client) addTransport(opts transportOptions, transport *http.Transport) { From ecc9fe45b6261492c0cfdcdb1ff4e642252d6694 Mon Sep 17 00:00:00 2001 From: "matej.risek" Date: Thu, 4 May 2023 09:04:23 +0200 Subject: [PATCH 20/31] git: Add Depth to SubmoduleUpdateOptions --- options.go | 3 +++ submodule.go | 3 ++- submodule_test.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/options.go b/options.go index 738c1ce9c..d1200d8a0 100644 --- a/options.go +++ b/options.go @@ -284,6 +284,9 @@ type SubmoduleUpdateOptions struct { RecurseSubmodules SubmoduleRescursivity // Auth credentials, if required, to use with the remote repository. Auth transport.AuthMethod + // Depth limit fetching to the specified number of commits from the tip of + // each remote branch history. + Depth int } var ( diff --git a/submodule.go b/submodule.go index a202a9b60..b0c416961 100644 --- a/submodule.go +++ b/submodule.go @@ -243,7 +243,7 @@ func (s *Submodule) fetchAndCheckout( ctx context.Context, r *Repository, o *SubmoduleUpdateOptions, hash plumbing.Hash, ) error { if !o.NoFetch { - err := r.FetchContext(ctx, &FetchOptions{Auth: o.Auth}) + err := r.FetchContext(ctx, &FetchOptions{Auth: o.Auth, Depth: o.Depth}) if err != nil && err != NoErrAlreadyUpToDate { return err } @@ -265,6 +265,7 @@ func (s *Submodule) fetchAndCheckout( err := r.FetchContext(ctx, &FetchOptions{ Auth: o.Auth, RefSpecs: []config.RefSpec{refSpec}, + Depth: o.Depth, }) if err != nil && err != NoErrAlreadyUpToDate && err != ErrExactSHA1NotSupported { return err diff --git a/submodule_test.go b/submodule_test.go index 4bae544d1..a8a0681a1 100644 --- a/submodule_test.go +++ b/submodule_test.go @@ -233,3 +233,32 @@ func (s *SubmoduleSuite) TestSubmodulesUpdateContext(c *C) { err = sm.UpdateContext(ctx, &SubmoduleUpdateOptions{Init: true}) c.Assert(err, NotNil) } + +func (s *SubmoduleSuite) TestSubmodulesFetchDepth(c *C) { + if testing.Short() { + c.Skip("skipping test in short mode.") + } + + sm, err := s.Worktree.Submodule("basic") + c.Assert(err, IsNil) + + err = sm.Update(&SubmoduleUpdateOptions{ + Init: true, + Depth: 1, + }) + c.Assert(err, IsNil) + + r, err := sm.Repository() + c.Assert(err, IsNil) + + lr, err := r.Log(&LogOptions{}) + c.Assert(err, IsNil) + + commitCount := 0 + for _, err := lr.Next(); err == nil; _, err = lr.Next() { + commitCount++ + } + c.Assert(err, IsNil) + + c.Assert(commitCount, Equals, 1) +} From b8dd39abcc9dd35808ec625a90a68c0ace4973e4 Mon Sep 17 00:00:00 2001 From: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> Date: Thu, 11 May 2023 11:17:04 +1000 Subject: [PATCH 21/31] git: Testing, Fix tests not cleaning temp folders Signed-off-by: Arieh Schneier <15041913+AriehSchneier@users.noreply.github.com> --- common_test.go | 24 ++++++++++++++++-------- repository_test.go | 18 +++++++++--------- submodule_test.go | 16 ++++++---------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/common_test.go b/common_test.go index c0c4009e2..3311a1793 100644 --- a/common_test.go +++ b/common_test.go @@ -37,7 +37,7 @@ func (s *BaseSuite) TearDownSuite(c *C) { s.Suite.TearDownSuite(c) } -func (s *BaseSuite) buildBasicRepository(c *C) { +func (s *BaseSuite) buildBasicRepository(_ *C) { f := fixtures.Basic().One() s.Repository = s.NewRepository(f) } @@ -105,13 +105,16 @@ func (s *BaseSuite) NewRepositoryFromPackfile(f *fixtures.Fixture) *Repository { storer := memory.NewStorage() p := f.Packfile() - defer p.Close() + defer func() { _ = p.Close() }() if err := packfile.UpdateObjectStorage(storer, p); err != nil { panic(err) } - storer.SetReference(plumbing.NewHashReference(plumbing.HEAD, plumbing.NewHash(f.Head))) + err := storer.SetReference(plumbing.NewHashReference(plumbing.HEAD, plumbing.NewHash(f.Head))) + if err != nil { + panic(err) + } r, err := Open(storer, memfs.New()) if err != nil { @@ -133,14 +136,17 @@ func (s *BaseSuite) GetLocalRepositoryURL(f *fixtures.Fixture) string { func (s *BaseSuite) TemporalDir() (path string, clean func()) { fs := osfs.New(os.TempDir()) - path, err := util.TempDir(fs, "", "") + relPath, err := util.TempDir(fs, "", "") if err != nil { panic(err) } - return fs.Join(fs.Root(), path), func() { - util.RemoveAll(fs, path) + path = fs.Join(fs.Root(), relPath) + clean = func() { + _ = util.RemoveAll(fs, relPath) } + + return } func (s *BaseSuite) TemporalFilesystem() (fs billy.Filesystem, clean func()) { @@ -155,9 +161,11 @@ func (s *BaseSuite) TemporalFilesystem() (fs billy.Filesystem, clean func()) { panic(err) } - return fs, func() { - util.RemoveAll(fs, path) + clean = func() { + _ = util.RemoveAll(fs, path) } + + return } type SuiteCommon struct{} diff --git a/repository_test.go b/repository_test.go index 0080a8384..45af396be 100644 --- a/repository_test.go +++ b/repository_test.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -60,17 +59,18 @@ func (s *RepositorySuite) TestInitNonStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot("storage") - storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) + st := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) wt, _ := fs.Chroot("worktree") - r, err := Init(storage, wt) + r, err := Init(st, wt) c.Assert(err, IsNil) c.Assert(r, NotNil) f, err := fs.Open(fs.Join("worktree", ".git")) c.Assert(err, IsNil) + defer func() { _ = f.Close() }() - all, err := ioutil.ReadAll(f) + all, err := io.ReadAll(f) c.Assert(err, IsNil) c.Assert(string(all), Equals, fmt.Sprintf("gitdir: %s\n", filepath.Join("..", "storage"))) @@ -85,9 +85,9 @@ func (s *RepositorySuite) TestInitStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot(".git") - storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) + st := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - r, err := Init(storage, fs) + r, err := Init(st, fs) c.Assert(err, IsNil) c.Assert(r, NotNil) @@ -3009,14 +3009,14 @@ func BenchmarkObjects(b *testing.B) { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - storer := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + st := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) worktree, err := fs.Chroot(filepath.Dir(fs.Root())) if err != nil { b.Fatal(err) } - repo, err := Open(storer, worktree) + repo, err := Open(st, worktree) if err != nil { b.Fatal(err) } @@ -3046,7 +3046,7 @@ func BenchmarkObjects(b *testing.B) { func BenchmarkPlainClone(b *testing.B) { for i := 0; i < b.N; i++ { - t, err := ioutil.TempDir("", "") + t, err := os.MkdirTemp("", "") if err != nil { b.Fatal(err) } diff --git a/submodule_test.go b/submodule_test.go index a8a0681a1..92943e11d 100644 --- a/submodule_test.go +++ b/submodule_test.go @@ -2,7 +2,6 @@ package git import ( "context" - "os" "path/filepath" "testing" @@ -15,7 +14,7 @@ import ( type SubmoduleSuite struct { BaseSuite Worktree *Worktree - path string + clean func() } var _ = Suite(&SubmoduleSuite{}) @@ -23,8 +22,8 @@ var _ = Suite(&SubmoduleSuite{}) func (s *SubmoduleSuite) SetUpTest(c *C) { path := fixtures.ByTag("submodule").One().Worktree().Root() - dir, clean := s.TemporalDir() - defer clean() + var dir string + dir, s.clean = s.TemporalDir() r, err := PlainClone(filepath.Join(dir, "worktree"), false, &CloneOptions{ URL: path, @@ -35,13 +34,10 @@ func (s *SubmoduleSuite) SetUpTest(c *C) { s.Repository = r s.Worktree, err = r.Worktree() c.Assert(err, IsNil) - - s.path = dir } -func (s *SubmoduleSuite) TearDownTest(c *C) { - err := os.RemoveAll(s.path) - c.Assert(err, IsNil) +func (s *SubmoduleSuite) TearDownTest(_ *C) { + s.clean() } func (s *SubmoduleSuite) TestInit(c *C) { @@ -198,7 +194,7 @@ func (s *SubmoduleSuite) TestSubmodulesInit(c *C) { func (s *SubmoduleSuite) TestGitSubmodulesSymlink(c *C) { f, err := s.Worktree.Filesystem.Create("badfile") c.Assert(err, IsNil) - defer f.Close() + defer func() { _ = f.Close() }() err = s.Worktree.Filesystem.Remove(gitmodulesFile) c.Assert(err, IsNil) From 9dfaf6acd8e3aa1c85bcce7d45f8e8c6b908319c Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Thu, 11 May 2023 21:34:07 +0100 Subject: [PATCH 22/31] *: Remove redudant err nil checks Signed-off-by: Paulo Gomes --- plumbing/format/idxfile/writer.go | 5 +---- plumbing/format/packfile/encoder.go | 6 +----- plumbing/object/rename.go | 5 +---- remote.go | 6 +----- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index daa160502..e5560130f 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -84,11 +84,8 @@ func (w *Writer) OnFooter(h plumbing.Hash) error { w.checksum = h w.finished = true _, err := w.createIndex() - if err != nil { - return err - } - return nil + return err } // creatIndex returns a filled MemoryIndex with the information filled by diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index c9d19b332..804f5a876 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -131,11 +131,7 @@ func (e *Encoder) entry(o *ObjectToPack) (err error) { defer ioutil.CheckClose(or, &err) _, err = io.Copy(e.zw, or) - if err != nil { - return err - } - - return nil + return err } func (e *Encoder) writeBaseIfDelta(o *ObjectToPack) error { diff --git a/plumbing/object/rename.go b/plumbing/object/rename.go index 0394613ff..ad2b902c2 100644 --- a/plumbing/object/rename.go +++ b/plumbing/object/rename.go @@ -741,10 +741,7 @@ func (i *similarityIndex) add(key int, cnt uint64) error { // It's the same key, so increment the counter. var err error i.hashes[j], err = newKeyCountPair(key, v.count()+cnt) - if err != nil { - return err - } - return nil + return err } else if j+1 >= len(i.hashes) { j = 0 } else { diff --git a/remote.go b/remote.go index df2649112..dd12f8ab6 100644 --- a/remote.go +++ b/remote.go @@ -1250,11 +1250,7 @@ func (r *Remote) buildFetchedTags(refs memory.ReferenceStorage) (updated bool, e // operation is complete, an error is returned. The context only affects to the // transport operations. func (r *Remote) ListContext(ctx context.Context, o *ListOptions) (rfs []*plumbing.Reference, err error) { - refs, err := r.list(ctx, o) - if err != nil { - return refs, err - } - return refs, nil + return r.list(ctx, o) } func (r *Remote) List(o *ListOptions) (rfs []*plumbing.Reference, err error) { From 096b3cc16b547f3c0d6e4f92046945bcfac0fa14 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Thu, 11 May 2023 21:59:37 +0100 Subject: [PATCH 23/31] *: Remove use of deprecated io/util Signed-off-by: Paulo Gomes --- _examples/commit/main.go | 3 +-- _examples/common_test.go | 3 +-- _examples/sha256/main.go | 3 +-- _examples/tag-create-push/main.go | 3 +-- config/config.go | 3 +-- example_test.go | 7 +++---- plumbing/format/gitattributes/attributes.go | 3 +-- plumbing/format/gitignore/dir.go | 4 ++-- plumbing/format/idxfile/decoder_test.go | 3 +-- plumbing/format/idxfile/encoder_test.go | 4 ++-- plumbing/format/idxfile/writer_test.go | 6 +++--- plumbing/format/index/decoder.go | 4 ++-- plumbing/format/objfile/reader_test.go | 3 +-- plumbing/format/packfile/delta_test.go | 10 +++++----- plumbing/format/packfile/encoder_test.go | 5 ++--- plumbing/format/packfile/parser.go | 3 +-- plumbing/format/packfile/scanner.go | 5 ++--- plumbing/memory_test.go | 5 ++--- plumbing/object/blob_test.go | 7 +++---- plumbing/object/commit_test.go | 5 ++--- plumbing/object/object_test.go | 3 +-- plumbing/object/tag_test.go | 3 +-- .../protocol/packp/sideband/demux_test.go | 3 +-- plumbing/protocol/packp/updreq_decode.go | 3 +-- plumbing/protocol/packp/updreq_decode_test.go | 15 +++++++------- plumbing/protocol/packp/updreq_encode_test.go | 5 ++--- plumbing/protocol/packp/uppackresp_test.go | 20 +++++++++---------- plumbing/transport/file/common_test.go | 3 +-- plumbing/transport/git/common_test.go | 3 +-- plumbing/transport/http/common_test.go | 3 +-- plumbing/transport/http/upload_pack_test.go | 4 ++-- plumbing/transport/internal/common/common.go | 3 +-- plumbing/transport/ssh/auth_method.go | 3 +-- .../transport/ssh/internal/test/proxy_test.go | 3 +-- plumbing/transport/ssh/proxy_test.go | 3 +-- plumbing/transport/ssh/upload_pack_test.go | 3 +-- plumbing/transport/test/receive_pack.go | 6 ++---- plumbing/transport/test/upload_pack.go | 6 ++---- remote_test.go | 3 +-- repository.go | 6 +++--- storage/filesystem/dotgit/dotgit.go | 3 +-- storage/filesystem/dotgit/dotgit_test.go | 7 +++---- storage/filesystem/object_test.go | 3 +-- storage/test/storage_suite.go | 5 ++--- utils/ioutil/common_test.go | 8 ++++---- worktree.go | 5 ++--- worktree_test.go | 11 +++++----- 47 files changed, 96 insertions(+), 136 deletions(-) diff --git a/_examples/commit/main.go b/_examples/commit/main.go index 4529c845a..3f3c88048 100644 --- a/_examples/commit/main.go +++ b/_examples/commit/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io/ioutil" "os" "path/filepath" "time" @@ -29,7 +28,7 @@ func main() { // worktree of the project using the go standard library. Info("echo \"hello world!\" > example-git-file") filename := filepath.Join(directory, "example-git-file") - err = ioutil.WriteFile(filename, []byte("hello world!"), 0644) + err = os.WriteFile(filename, []byte("hello world!"), 0644) CheckIfError(err) // Adds the new file to the staging area. diff --git a/_examples/common_test.go b/_examples/common_test.go index 9945c875a..6630f15ee 100644 --- a/_examples/common_test.go +++ b/_examples/common_test.go @@ -3,7 +3,6 @@ package examples import ( "flag" "go/build" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -65,7 +64,7 @@ func TestExamples(t *testing.T) { } func tempFolder() string { - path, err := ioutil.TempDir("", "") + path, err := os.MkdirTemp("", "") CheckIfError(err) tempFolders = append(tempFolders, path) diff --git a/_examples/sha256/main.go b/_examples/sha256/main.go index 2a257e863..e1772d274 100644 --- a/_examples/sha256/main.go +++ b/_examples/sha256/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io/ioutil" "os" "path/filepath" "time" @@ -34,7 +33,7 @@ func main() { // worktree of the project using the go standard library. Info("echo \"hello world!\" > example-git-file") filename := filepath.Join(directory, "example-git-file") - err = ioutil.WriteFile(filename, []byte("hello world!"), 0644) + err = os.WriteFile(filename, []byte("hello world!"), 0644) CheckIfError(err) // Adds the new file to the staging area. diff --git a/_examples/tag-create-push/main.go b/_examples/tag-create-push/main.go index c443641e2..b820c76b7 100644 --- a/_examples/tag-create-push/main.go +++ b/_examples/tag-create-push/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io/ioutil" "log" "os" @@ -67,7 +66,7 @@ func cloneRepo(url, dir, publicKeyPath string) (*git.Repository, error) { func publicKey(filePath string) (*ssh.PublicKeys, error) { var publicKey *ssh.PublicKeys - sshKey, _ := ioutil.ReadFile(filePath) + sshKey, _ := os.ReadFile(filePath) publicKey, err := ssh.NewPublicKeys("git", []byte(sshKey), "") if err != nil { return nil, err diff --git a/config/config.go b/config/config.go index 60dfca4f0..82af12d28 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sort" @@ -144,7 +143,7 @@ func NewConfig() *Config { // ReadConfig reads a config file from a io.Reader. func ReadConfig(r io.Reader) (*Config, error) { - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/example_test.go b/example_test.go index bba2961c0..27ea4a2d5 100644 --- a/example_test.go +++ b/example_test.go @@ -3,7 +3,6 @@ package git_test import ( "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -44,7 +43,7 @@ func ExampleClone() { func ExamplePlainClone() { // Tempdir to clone the repository - dir, err := ioutil.TempDir("", "clone-example") + dir, err := os.MkdirTemp("", "clone-example") if err != nil { log.Fatal(err) } @@ -72,7 +71,7 @@ func ExamplePlainClone() { func ExamplePlainClone_usernamePassword() { // Tempdir to clone the repository - dir, err := ioutil.TempDir("", "clone-example") + dir, err := os.MkdirTemp("", "clone-example") if err != nil { log.Fatal(err) } @@ -95,7 +94,7 @@ func ExamplePlainClone_usernamePassword() { func ExamplePlainClone_accessToken() { // Tempdir to clone the repository - dir, err := ioutil.TempDir("", "clone-example") + dir, err := os.MkdirTemp("", "clone-example") if err != nil { log.Fatal(err) } diff --git a/plumbing/format/gitattributes/attributes.go b/plumbing/format/gitattributes/attributes.go index 329e66731..d36ec1b53 100644 --- a/plumbing/format/gitattributes/attributes.go +++ b/plumbing/format/gitattributes/attributes.go @@ -3,7 +3,6 @@ package gitattributes import ( "errors" "io" - "io/ioutil" "strings" ) @@ -89,7 +88,7 @@ func (a attribute) String() string { // ReadAttributes reads patterns and attributes from the gitattributes format. func ReadAttributes(r io.Reader, domain []string, allowMacro bool) (attributes []MatchAttribute, err error) { - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/plumbing/format/gitignore/dir.go b/plumbing/format/gitignore/dir.go index 15bc9c779..bb786557c 100644 --- a/plumbing/format/gitignore/dir.go +++ b/plumbing/format/gitignore/dir.go @@ -3,7 +3,7 @@ package gitignore import ( "bufio" "bytes" - "io/ioutil" + "io" "os" "strings" @@ -86,7 +86,7 @@ func loadPatterns(fs billy.Filesystem, path string) (ps []Pattern, err error) { defer gioutil.CheckClose(f, &err) - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return } diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go index 94059ccb3..2c4a801a7 100644 --- a/plumbing/format/idxfile/decoder_test.go +++ b/plumbing/format/idxfile/decoder_test.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "fmt" "io" - "io/ioutil" "testing" "github.com/go-git/go-git/v5/plumbing" @@ -119,7 +118,7 @@ ch2xUA== func BenchmarkDecode(b *testing.B) { f := fixtures.Basic().One() - fixture, err := ioutil.ReadAll(f.Idx()) + fixture, err := io.ReadAll(f.Idx()) if err != nil { b.Errorf("unexpected error reading idx file: %s", err) } diff --git a/plumbing/format/idxfile/encoder_test.go b/plumbing/format/idxfile/encoder_test.go index 32b60f9b2..b8ece8398 100644 --- a/plumbing/format/idxfile/encoder_test.go +++ b/plumbing/format/idxfile/encoder_test.go @@ -2,7 +2,7 @@ package idxfile_test import ( "bytes" - "io/ioutil" + "io" . "github.com/go-git/go-git/v5/plumbing/format/idxfile" @@ -12,7 +12,7 @@ import ( func (s *IdxfileSuite) TestDecodeEncode(c *C) { fixtures.ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - expected, err := ioutil.ReadAll(f.Idx()) + expected, err := io.ReadAll(f.Idx()) c.Assert(err, IsNil) idx := new(MemoryIndex) diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go index fba3e4272..eaa8605f7 100644 --- a/plumbing/format/idxfile/writer_test.go +++ b/plumbing/format/idxfile/writer_test.go @@ -3,7 +3,7 @@ package idxfile_test import ( "bytes" "encoding/base64" - "io/ioutil" + "io" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/idxfile" @@ -34,7 +34,7 @@ func (s *WriterSuite) TestWriter(c *C) { c.Assert(err, IsNil) idxFile := f.Idx() - expected, err := ioutil.ReadAll(idxFile) + expected, err := io.ReadAll(idxFile) c.Assert(err, IsNil) idxFile.Close() @@ -65,7 +65,7 @@ func (s *WriterSuite) TestWriterLarge(c *C) { // load fixture index f := bytes.NewBufferString(fixtureLarge4GB) - expected, err := ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, f)) + expected, err := io.ReadAll(base64.NewDecoder(base64.StdEncoding, f)) c.Assert(err, IsNil) buf := new(bytes.Buffer) diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index 8834e2517..6778cf74e 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "io" - "io/ioutil" + "strconv" "time" @@ -201,7 +201,7 @@ func (d *Decoder) padEntry(idx *Index, e *Entry, read int) error { entrySize := read + len(e.Name) padLen := 8 - entrySize%8 - _, err := io.CopyN(ioutil.Discard, d.r, int64(padLen)) + _, err := io.CopyN(io.Discard, d.r, int64(padLen)) return err } diff --git a/plumbing/format/objfile/reader_test.go b/plumbing/format/objfile/reader_test.go index d697d5464..5526f7f4e 100644 --- a/plumbing/format/objfile/reader_test.go +++ b/plumbing/format/objfile/reader_test.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "fmt" "io" - "io/ioutil" "github.com/go-git/go-git/v5/plumbing" @@ -36,7 +35,7 @@ func testReader(c *C, source io.Reader, hash plumbing.Hash, t plumbing.ObjectTyp c.Assert(typ, Equals, t) c.Assert(content, HasLen, int(size)) - rc, err := ioutil.ReadAll(r) + rc, err := io.ReadAll(r) c.Assert(err, IsNil) c.Assert(rc, DeepEquals, content, Commentf("%scontent=%s, expected=%s", base64.StdEncoding.EncodeToString(rc), base64.StdEncoding.EncodeToString(content))) diff --git a/plumbing/format/packfile/delta_test.go b/plumbing/format/packfile/delta_test.go index 137e4859b..e8f5ea68f 100644 --- a/plumbing/format/packfile/delta_test.go +++ b/plumbing/format/packfile/delta_test.go @@ -2,7 +2,7 @@ package packfile import ( "bytes" - "io/ioutil" + "io" "math/rand" "github.com/go-git/go-git/v5/plumbing" @@ -109,14 +109,14 @@ func (s *DeltaSuite) TestAddDeltaReader(c *C) { targetBuf := genBytes(t.target) delta := DiffDelta(baseBuf, targetBuf) - deltaRC := ioutil.NopCloser(bytes.NewReader(delta)) + deltaRC := io.NopCloser(bytes.NewReader(delta)) c.Log("Executing test case:", t.description) resultRC, err := ReaderFromDelta(baseObj, deltaRC) c.Assert(err, IsNil) - result, err := ioutil.ReadAll(resultRC) + result, err := io.ReadAll(resultRC) c.Assert(err, IsNil) err = resultRC.Close() @@ -164,12 +164,12 @@ func (s *DeltaSuite) TestMaxCopySizeDeltaReader(c *C) { targetBuf = append(targetBuf, byte(1)) delta := DiffDelta(baseBuf, targetBuf) - deltaRC := ioutil.NopCloser(bytes.NewReader(delta)) + deltaRC := io.NopCloser(bytes.NewReader(delta)) resultRC, err := ReaderFromDelta(baseObj, deltaRC) c.Assert(err, IsNil) - result, err := ioutil.ReadAll(resultRC) + result, err := io.ReadAll(resultRC) c.Assert(err, IsNil) err = resultRC.Close() diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index 902d8ccaa..6719f376a 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -3,7 +3,6 @@ package packfile import ( "bytes" "io" - stdioutil "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/idxfile" @@ -278,13 +277,13 @@ func objectsEqual(c *C, o1, o2 plumbing.EncodedObject) { r1, err := o1.Reader() c.Assert(err, IsNil) - b1, err := stdioutil.ReadAll(r1) + b1, err := io.ReadAll(r1) c.Assert(err, IsNil) r2, err := o2.Reader() c.Assert(err, IsNil) - b2, err := stdioutil.ReadAll(r2) + b2, err := io.ReadAll(r2) c.Assert(err, IsNil) c.Assert(bytes.Compare(b1, b2), Equals, 0) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 6012b0458..edbc0e796 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "io" - stdioutil "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" @@ -297,7 +296,7 @@ func (p *Parser) resolveDeltas() error { if !obj.IsDelta() && len(obj.Children) > 0 { for _, child := range obj.Children { - if err := p.resolveObject(stdioutil.Discard, child, content); err != nil { + if err := p.resolveObject(io.Discard, child, content); err != nil { return err } p.resolveExternalRef(child) diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 9ebb84a24..730343ee3 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -7,7 +7,6 @@ import ( "hash" "hash/crc32" "io" - stdioutil "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/binary" @@ -242,7 +241,7 @@ func (s *Scanner) discardObjectIfNeeded() error { } h := s.pendingObject - n, _, err := s.NextObject(stdioutil.Discard) + n, _, err := s.NextObject(io.Discard) if err != nil { return err } @@ -381,7 +380,7 @@ func (s *Scanner) Checksum() (plumbing.Hash, error) { // Close reads the reader until io.EOF func (s *Scanner) Close() error { buf := sync.GetByteSlice() - _, err := io.CopyBuffer(stdioutil.Discard, s.r, *buf) + _, err := io.CopyBuffer(io.Discard, s.r, *buf) sync.PutByteSlice(buf) return err diff --git a/plumbing/memory_test.go b/plumbing/memory_test.go index 2a141f491..f76b4f40f 100644 --- a/plumbing/memory_test.go +++ b/plumbing/memory_test.go @@ -2,7 +2,6 @@ package plumbing import ( "io" - "io/ioutil" . "gopkg.in/check.v1" ) @@ -52,7 +51,7 @@ func (s *MemoryObjectSuite) TestReader(c *C) { c.Assert(err, IsNil) defer func() { c.Assert(reader.Close(), IsNil) }() - b, err := ioutil.ReadAll(reader) + b, err := io.ReadAll(reader) c.Assert(err, IsNil) c.Assert(b, DeepEquals, []byte("foo")) } @@ -75,7 +74,7 @@ func (s *MemoryObjectSuite) TestSeekableReader(c *C) { _, err = rs.Seek(pageSize, io.SeekStart) c.Assert(err, IsNil) - b, err := ioutil.ReadAll(rs) + b, err := io.ReadAll(rs) c.Assert(err, IsNil) c.Assert(b, DeepEquals, []byte(payload)) diff --git a/plumbing/object/blob_test.go b/plumbing/object/blob_test.go index 44613433a..9481dbe44 100644 --- a/plumbing/object/blob_test.go +++ b/plumbing/object/blob_test.go @@ -3,7 +3,6 @@ package object import ( "bytes" "io" - "io/ioutil" "github.com/go-git/go-git/v5/plumbing" @@ -37,7 +36,7 @@ func (s *BlobsSuite) TestBlobHash(c *C) { c.Assert(err, IsNil) defer func() { c.Assert(reader.Close(), IsNil) }() - data, err := ioutil.ReadAll(reader) + data, err := io.ReadAll(reader) c.Assert(err, IsNil) c.Assert(string(data), Equals, "FOO") } @@ -96,14 +95,14 @@ func (s *BlobsSuite) TestBlobIter(c *C) { r1, err := b.Reader() c.Assert(err, IsNil) - b1, err := ioutil.ReadAll(r1) + b1, err := io.ReadAll(r1) c.Assert(err, IsNil) c.Assert(r1.Close(), IsNil) r2, err := blobs[i].Reader() c.Assert(err, IsNil) - b2, err := ioutil.ReadAll(r2) + b2, err := io.ReadAll(r2) c.Assert(err, IsNil) c.Assert(r2.Close(), IsNil) diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index 468a751fa..a939bc6a4 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "io" - "io/ioutil" "strings" "time" @@ -449,7 +448,7 @@ YIefGtzXfldDxg4= ` e, err := commit.Verify(armoredKeyRing) - c.Assert(err, IsNil) + c.Assert(err, IsNil) _, ok := e.Identities["go-git test key"] c.Assert(ok, Equals, true) @@ -492,7 +491,7 @@ func (s *SuiteCommit) TestEncodeWithoutSignature(c *C) { c.Assert(err, IsNil) er, err := encoded.Reader() c.Assert(err, IsNil) - payload, err := ioutil.ReadAll(er) + payload, err := io.ReadAll(er) c.Assert(err, IsNil) c.Assert(string(payload), Equals, ""+ diff --git a/plumbing/object/object_test.go b/plumbing/object/object_test.go index 6c95eef9c..c4fdb4c7a 100644 --- a/plumbing/object/object_test.go +++ b/plumbing/object/object_test.go @@ -2,7 +2,6 @@ package object import ( "io" - "io/ioutil" "testing" "time" @@ -103,7 +102,7 @@ func (s *ObjectsSuite) TestParseTree(c *C) { reader, err := f.Reader() c.Assert(err, IsNil) defer func() { c.Assert(reader.Close(), IsNil) }() - content, _ := ioutil.ReadAll(reader) + content, _ := io.ReadAll(reader) c.Assert(content, HasLen, 2780) } } diff --git a/plumbing/object/tag_test.go b/plumbing/object/tag_test.go index 15b943e07..d374c6c54 100644 --- a/plumbing/object/tag_test.go +++ b/plumbing/object/tag_test.go @@ -3,7 +3,6 @@ package object import ( "fmt" "io" - "io/ioutil" "strings" "time" @@ -466,7 +465,7 @@ func (s *TagSuite) TestEncodeWithoutSignature(c *C) { c.Assert(err, IsNil) er, err := encoded.Reader() c.Assert(err, IsNil) - payload, err := ioutil.ReadAll(er) + payload, err := io.ReadAll(er) c.Assert(err, IsNil) c.Assert(string(payload), Equals, ""+ diff --git a/plumbing/protocol/packp/sideband/demux_test.go b/plumbing/protocol/packp/sideband/demux_test.go index 6cda70381..8f233538c 100644 --- a/plumbing/protocol/packp/sideband/demux_test.go +++ b/plumbing/protocol/packp/sideband/demux_test.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "io" - "io/ioutil" "testing" "github.com/go-git/go-git/v5/plumbing/format/pktline" @@ -101,7 +100,7 @@ func (s *SidebandSuite) TestDecodeWithProgress(c *C) { c.Assert(n, Equals, 26) c.Assert(content, DeepEquals, expected) - progress, err := ioutil.ReadAll(output) + progress, err := io.ReadAll(output) c.Assert(err, IsNil) c.Assert(progress, DeepEquals, []byte{'F', 'O', 'O', '\n'}) } diff --git a/plumbing/protocol/packp/updreq_decode.go b/plumbing/protocol/packp/updreq_decode.go index 2c9843a56..076de545f 100644 --- a/plumbing/protocol/packp/updreq_decode.go +++ b/plumbing/protocol/packp/updreq_decode.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/pktline" @@ -81,7 +80,7 @@ func (req *ReferenceUpdateRequest) Decode(r io.Reader) error { var ok bool rc, ok = r.(io.ReadCloser) if !ok { - rc = ioutil.NopCloser(r) + rc = io.NopCloser(r) } d := &updReqDecoder{r: rc, s: pktline.NewScanner(r)} diff --git a/plumbing/protocol/packp/updreq_decode_test.go b/plumbing/protocol/packp/updreq_decode_test.go index 26301123b..bdcbdf503 100644 --- a/plumbing/protocol/packp/updreq_decode_test.go +++ b/plumbing/protocol/packp/updreq_decode_test.go @@ -3,7 +3,6 @@ package packp import ( "bytes" "io" - "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/pktline" @@ -157,7 +156,7 @@ func (s *UpdReqDecodeSuite) TestOneUpdateCommand(c *C) { expected.Commands = []*Command{ {Name: name, Old: hash1, New: hash2}, } - expected.Packfile = ioutil.NopCloser(bytes.NewReader([]byte{})) + expected.Packfile = io.NopCloser(bytes.NewReader([]byte{})) payloads := []string{ "1ecf0ef2c2dffb796033e5a02219af86ec6584e5 2ecf0ef2c2dffb796033e5a02219af86ec6584e5 myref\x00", @@ -177,7 +176,7 @@ func (s *UpdReqDecodeSuite) TestMultipleCommands(c *C) { {Name: plumbing.ReferenceName("myref2"), Old: plumbing.ZeroHash, New: hash2}, {Name: plumbing.ReferenceName("myref3"), Old: hash1, New: plumbing.ZeroHash}, } - expected.Packfile = ioutil.NopCloser(bytes.NewReader([]byte{})) + expected.Packfile = io.NopCloser(bytes.NewReader([]byte{})) payloads := []string{ "1ecf0ef2c2dffb796033e5a02219af86ec6584e5 2ecf0ef2c2dffb796033e5a02219af86ec6584e5 myref1\x00", @@ -200,7 +199,7 @@ func (s *UpdReqDecodeSuite) TestMultipleCommandsAndCapabilities(c *C) { {Name: plumbing.ReferenceName("myref3"), Old: hash1, New: plumbing.ZeroHash}, } expected.Capabilities.Add("shallow") - expected.Packfile = ioutil.NopCloser(bytes.NewReader([]byte{})) + expected.Packfile = io.NopCloser(bytes.NewReader([]byte{})) payloads := []string{ "1ecf0ef2c2dffb796033e5a02219af86ec6584e5 2ecf0ef2c2dffb796033e5a02219af86ec6584e5 myref1\x00shallow", @@ -224,7 +223,7 @@ func (s *UpdReqDecodeSuite) TestMultipleCommandsAndCapabilitiesShallow(c *C) { } expected.Capabilities.Add("shallow") expected.Shallow = &hash1 - expected.Packfile = ioutil.NopCloser(bytes.NewReader([]byte{})) + expected.Packfile = io.NopCloser(bytes.NewReader([]byte{})) payloads := []string{ "shallow 1ecf0ef2c2dffb796033e5a02219af86ec6584e5", @@ -247,7 +246,7 @@ func (s *UpdReqDecodeSuite) TestWithPackfile(c *C) { {Name: name, Old: hash1, New: hash2}, } packfileContent := []byte("PACKabc") - expected.Packfile = ioutil.NopCloser(bytes.NewReader(packfileContent)) + expected.Packfile = io.NopCloser(bytes.NewReader(packfileContent)) payloads := []string{ "1ecf0ef2c2dffb796033e5a02219af86ec6584e5 2ecf0ef2c2dffb796033e5a02219af86ec6584e5 myref\x00", @@ -298,10 +297,10 @@ func (s *UpdReqDecodeSuite) testDecodeOkExpected(c *C, expected *ReferenceUpdate } func (s *UpdReqDecodeSuite) compareReaders(c *C, a io.ReadCloser, b io.ReadCloser) { - pba, err := ioutil.ReadAll(a) + pba, err := io.ReadAll(a) c.Assert(err, IsNil) c.Assert(a.Close(), IsNil) - pbb, err := ioutil.ReadAll(b) + pbb, err := io.ReadAll(b) c.Assert(err, IsNil) c.Assert(b.Close(), IsNil) c.Assert(pba, DeepEquals, pbb) diff --git a/plumbing/protocol/packp/updreq_encode_test.go b/plumbing/protocol/packp/updreq_encode_test.go index 4370b794f..97868bd64 100644 --- a/plumbing/protocol/packp/updreq_encode_test.go +++ b/plumbing/protocol/packp/updreq_encode_test.go @@ -2,13 +2,12 @@ package packp import ( "bytes" + "io" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/pktline" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" - "io/ioutil" - . "gopkg.in/check.v1" ) @@ -128,7 +127,7 @@ func (s *UpdReqEncodeSuite) TestWithPackfile(c *C) { packfileContent := []byte("PACKabc") packfileReader := bytes.NewReader(packfileContent) - packfileReadCloser := ioutil.NopCloser(packfileReader) + packfileReadCloser := io.NopCloser(packfileReader) r := NewReferenceUpdateRequest() r.Commands = []*Command{ diff --git a/plumbing/protocol/packp/uppackresp_test.go b/plumbing/protocol/packp/uppackresp_test.go index 3f87804a5..8fbf92467 100644 --- a/plumbing/protocol/packp/uppackresp_test.go +++ b/plumbing/protocol/packp/uppackresp_test.go @@ -2,7 +2,7 @@ package packp import ( "bytes" - "io/ioutil" + "io" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" @@ -21,10 +21,10 @@ func (s *UploadPackResponseSuite) TestDecodeNAK(c *C) { res := NewUploadPackResponse(req) defer res.Close() - err := res.Decode(ioutil.NopCloser(bytes.NewBufferString(raw))) + err := res.Decode(io.NopCloser(bytes.NewBufferString(raw))) c.Assert(err, IsNil) - pack, err := ioutil.ReadAll(res) + pack, err := io.ReadAll(res) c.Assert(err, IsNil) c.Assert(pack, DeepEquals, []byte("PACK")) } @@ -38,10 +38,10 @@ func (s *UploadPackResponseSuite) TestDecodeDepth(c *C) { res := NewUploadPackResponse(req) defer res.Close() - err := res.Decode(ioutil.NopCloser(bytes.NewBufferString(raw))) + err := res.Decode(io.NopCloser(bytes.NewBufferString(raw))) c.Assert(err, IsNil) - pack, err := ioutil.ReadAll(res) + pack, err := io.ReadAll(res) c.Assert(err, IsNil) c.Assert(pack, DeepEquals, []byte("PACK")) } @@ -55,7 +55,7 @@ func (s *UploadPackResponseSuite) TestDecodeMalformed(c *C) { res := NewUploadPackResponse(req) defer res.Close() - err := res.Decode(ioutil.NopCloser(bytes.NewBufferString(raw))) + err := res.Decode(io.NopCloser(bytes.NewBufferString(raw))) c.Assert(err, NotNil) } @@ -70,7 +70,7 @@ func (s *UploadPackResponseSuite) TestDecodeMultiACK(c *C) { res := NewUploadPackResponse(req) defer res.Close() - err := res.Decode(ioutil.NopCloser(bytes.NewBuffer(nil))) + err := res.Decode(io.NopCloser(bytes.NewBuffer(nil))) c.Assert(err, IsNil) } @@ -87,7 +87,7 @@ func (s *UploadPackResponseSuite) TestReadNoDecode(c *C) { } func (s *UploadPackResponseSuite) TestEncodeNAK(c *C) { - pf := ioutil.NopCloser(bytes.NewBuffer([]byte("[PACK]"))) + pf := io.NopCloser(bytes.NewBuffer([]byte("[PACK]"))) req := NewUploadPackRequest() res := NewUploadPackResponseWithPackfile(req, pf) defer func() { c.Assert(res.Close(), IsNil) }() @@ -100,7 +100,7 @@ func (s *UploadPackResponseSuite) TestEncodeNAK(c *C) { } func (s *UploadPackResponseSuite) TestEncodeDepth(c *C) { - pf := ioutil.NopCloser(bytes.NewBuffer([]byte("PACK"))) + pf := io.NopCloser(bytes.NewBuffer([]byte("PACK"))) req := NewUploadPackRequest() req.Depth = DepthCommits(1) @@ -115,7 +115,7 @@ func (s *UploadPackResponseSuite) TestEncodeDepth(c *C) { } func (s *UploadPackResponseSuite) TestEncodeMultiACK(c *C) { - pf := ioutil.NopCloser(bytes.NewBuffer([]byte("[PACK]"))) + pf := io.NopCloser(bytes.NewBuffer([]byte("[PACK]"))) req := NewUploadPackRequest() res := NewUploadPackResponseWithPackfile(req, pf) diff --git a/plumbing/transport/file/common_test.go b/plumbing/transport/file/common_test.go index 4d6612b70..7e033a80b 100644 --- a/plumbing/transport/file/common_test.go +++ b/plumbing/transport/file/common_test.go @@ -1,7 +1,6 @@ package file import ( - "io/ioutil" "os" "os/exec" "path/filepath" @@ -25,7 +24,7 @@ func (s *CommonSuite) SetUpSuite(c *C) { } var err error - s.tmpDir, err = ioutil.TempDir("", "") + s.tmpDir, err = os.MkdirTemp("", "") c.Assert(err, IsNil) s.ReceivePackBin = filepath.Join(s.tmpDir, "git-receive-pack") s.UploadPackBin = filepath.Join(s.tmpDir, "git-upload-pack") diff --git a/plumbing/transport/git/common_test.go b/plumbing/transport/git/common_test.go index 3391aafd6..73899198f 100644 --- a/plumbing/transport/git/common_test.go +++ b/plumbing/transport/git/common_test.go @@ -2,7 +2,6 @@ package git import ( "fmt" - "io/ioutil" "net" "os" "os/exec" @@ -37,7 +36,7 @@ func (s *BaseSuite) SetUpTest(c *C) { s.port, err = freePort() c.Assert(err, IsNil) - s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-protocol-%d", s.port)) + s.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-protocol-%d", s.port)) c.Assert(err, IsNil) } diff --git a/plumbing/transport/http/common_test.go b/plumbing/transport/http/common_test.go index 41188e677..151722876 100644 --- a/plumbing/transport/http/common_test.go +++ b/plumbing/transport/http/common_test.go @@ -3,7 +3,6 @@ package http import ( "crypto/tls" "fmt" - "io/ioutil" "log" "net" "net/http" @@ -222,7 +221,7 @@ func (s *BaseSuite) SetUpTest(c *C) { l, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) - base, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-http-%d", s.port)) + base, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-http-%d", s.port)) c.Assert(err, IsNil) s.port = l.Addr().(*net.TCPAddr).Port diff --git a/plumbing/transport/http/upload_pack_test.go b/plumbing/transport/http/upload_pack_test.go index c088ecccd..abb7adf37 100644 --- a/plumbing/transport/http/upload_pack_test.go +++ b/plumbing/transport/http/upload_pack_test.go @@ -3,7 +3,7 @@ package http import ( "context" "fmt" - "io/ioutil" + "io" "net/url" "os" "path/filepath" @@ -49,7 +49,7 @@ func (s *UploadPackSuite) TestuploadPackRequestToReader(c *C) { sr, err := uploadPackRequestToReader(r) c.Assert(err, IsNil) - b, _ := ioutil.ReadAll(sr) + b, _ := io.ReadAll(sr) c.Assert(string(b), Equals, "0032want 2b41ef280fdb67a9b250678686a0c3e03b0a9989\n"+ "0032want d82f291cde9987322c8a0c81a325e1ba6159684c\n0000"+ diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index b2c2fee38..99e0850f9 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "io" - stdioutil "io/ioutil" "strings" "time" @@ -156,7 +155,7 @@ func (c *client) listenFirstError(r io.Reader) chan string { close(errLine) } - _, _ = io.Copy(stdioutil.Discard, r) + _, _ = io.Copy(io.Discard, r) }() return errLine diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index e89ce4ba3..ac4e3583c 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -3,7 +3,6 @@ package ssh import ( "errors" "fmt" - "io/ioutil" "os" "os/user" "path/filepath" @@ -134,7 +133,7 @@ func NewPublicKeys(user string, pemBytes []byte, password string) (*PublicKeys, // encoded private key. An encryption password should be given if the pemBytes // contains a password encrypted PEM block otherwise password should be empty. func NewPublicKeysFromFile(user, pemFile, password string) (*PublicKeys, error) { - bytes, err := ioutil.ReadFile(pemFile) + bytes, err := os.ReadFile(pemFile) if err != nil { return nil, err } diff --git a/plumbing/transport/ssh/internal/test/proxy_test.go b/plumbing/transport/ssh/internal/test/proxy_test.go index 8baac2b57..8e775f89a 100644 --- a/plumbing/transport/ssh/internal/test/proxy_test.go +++ b/plumbing/transport/ssh/internal/test/proxy_test.go @@ -3,7 +3,6 @@ package test import ( "context" "fmt" - "io/ioutil" "log" "net" "os" @@ -59,7 +58,7 @@ func (s *ProxyEnvSuite) TestCommand(c *C) { }() s.port = sshListener.Addr().(*net.TCPAddr).Port - s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) + s.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) c.Assert(err, IsNil) ggssh.DefaultAuthBuilder = func(user string) (ggssh.AuthMethod, error) { diff --git a/plumbing/transport/ssh/proxy_test.go b/plumbing/transport/ssh/proxy_test.go index 2fab8512b..2ba98e823 100644 --- a/plumbing/transport/ssh/proxy_test.go +++ b/plumbing/transport/ssh/proxy_test.go @@ -3,7 +3,6 @@ package ssh import ( "context" "fmt" - "io/ioutil" "log" "net" "os" @@ -54,7 +53,7 @@ func (s *ProxySuite) TestCommand(c *C) { }() s.u.port = sshListener.Addr().(*net.TCPAddr).Port - s.u.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.u.port)) + s.u.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.u.port)) c.Assert(err, IsNil) DefaultAuthBuilder = func(user string) (AuthMethod, error) { diff --git a/plumbing/transport/ssh/upload_pack_test.go b/plumbing/transport/ssh/upload_pack_test.go index fafff485d..67af566e6 100644 --- a/plumbing/transport/ssh/upload_pack_test.go +++ b/plumbing/transport/ssh/upload_pack_test.go @@ -3,7 +3,6 @@ package ssh import ( "fmt" "io" - "io/ioutil" "log" "net" "os" @@ -43,7 +42,7 @@ func (s *UploadPackSuite) SetUpSuite(c *C) { c.Assert(err, IsNil) s.port = l.Addr().(*net.TCPAddr).Port - s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) + s.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) c.Assert(err, IsNil) DefaultAuthBuilder = func(user string) (AuthMethod, error) { diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index 018d38e30..9414fbaff 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -1,13 +1,11 @@ // Package test implements common test suite for different transport // implementations. -// package test import ( "bytes" "context" "io" - "io/ioutil" "os" "path/filepath" @@ -235,7 +233,7 @@ func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep *transport.Endpoint, if rootPath != "" && err == nil && stat.IsDir() { objectPath := filepath.Join(rootPath, "objects/pack") - files, err := ioutil.ReadDir(objectPath) + files, err := os.ReadDir(objectPath) c.Assert(err, IsNil) for _, file := range files { @@ -371,5 +369,5 @@ func (s *ReceivePackSuite) emptyPackfile() io.ReadCloser { panic(err) } - return ioutil.NopCloser(&buf) + return io.NopCloser(&buf) } diff --git a/plumbing/transport/test/upload_pack.go b/plumbing/transport/test/upload_pack.go index 3ee029d40..f7842ebb7 100644 --- a/plumbing/transport/test/upload_pack.go +++ b/plumbing/transport/test/upload_pack.go @@ -1,13 +1,11 @@ // Package test implements common test suite for different transport // implementations. -// package test import ( "bytes" "context" "io" - "io/ioutil" "time" "github.com/go-git/go-git/v5/plumbing" @@ -154,7 +152,7 @@ func (s *UploadPackSuite) TestUploadPackWithContextOnRead(c *C) { cancel() - _, err = io.Copy(ioutil.Discard, reader) + _, err = io.Copy(io.Discard, reader) c.Assert(err, NotNil) err = reader.Close() @@ -255,7 +253,7 @@ func (s *UploadPackSuite) TestFetchError(c *C) { } func (s *UploadPackSuite) checkObjectNumber(c *C, r io.Reader, n int) { - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) c.Assert(err, IsNil) buf := bytes.NewBuffer(b) storage := memory.NewStorage() diff --git a/remote_test.go b/remote_test.go index 164e4e509..1bcdb0f75 100644 --- a/remote_test.go +++ b/remote_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -1387,7 +1386,7 @@ func (s *RemoteSuite) TestPushRequireRemoteRefs(c *C) { } func (s *RemoteSuite) TestCanPushShasToReference(c *C) { - d, err := ioutil.TempDir("", "TestCanPushShasToReference") + d, err := os.MkdirTemp("", "TestCanPushShasToReference") c.Assert(err, IsNil) if err != nil { return diff --git a/repository.go b/repository.go index 287b597bb..89c48db82 100644 --- a/repository.go +++ b/repository.go @@ -7,7 +7,7 @@ import ( "encoding/hex" "errors" "fmt" - stdioutil "io/ioutil" + "io" "os" "path" "path/filepath" @@ -367,7 +367,7 @@ func dotGitFileToOSFilesystem(path string, fs billy.Filesystem) (bfs billy.Files } defer ioutil.CheckClose(f, &err) - b, err := stdioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return nil, err } @@ -396,7 +396,7 @@ func dotGitCommonDirectory(fs billy.Filesystem) (commonDir billy.Filesystem, err return nil, err } - b, err := stdioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return nil, err } diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 5bde37f6b..19d702632 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - stdioutil "io/ioutil" "os" "path/filepath" "sort" @@ -647,7 +646,7 @@ func (d *DotGit) ObjectDelete(h plumbing.Hash) error { } func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { - b, err := stdioutil.ReadAll(rd) + b, err := io.ReadAll(rd) if err != nil { return nil, err } diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 63c9eb015..6a339d1e5 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -4,7 +4,6 @@ import ( "bufio" "encoding/hex" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -374,7 +373,7 @@ func (s *SuiteDotGit) TestConfigWriteAndConfig(c *C) { f, err = dir.Config() c.Assert(err, IsNil) - cnt, err := ioutil.ReadAll(f) + cnt, err := io.ReadAll(f) c.Assert(err, IsNil) c.Assert(string(cnt), Equals, "foo") @@ -404,7 +403,7 @@ func (s *SuiteDotGit) TestIndexWriteAndIndex(c *C) { f, err = dir.Index() c.Assert(err, IsNil) - cnt, err := ioutil.ReadAll(f) + cnt, err := io.ReadAll(f) c.Assert(err, IsNil) c.Assert(string(cnt), Equals, "foo") @@ -434,7 +433,7 @@ func (s *SuiteDotGit) TestShallowWriteAndShallow(c *C) { f, err = dir.Shallow() c.Assert(err, IsNil) - cnt, err := ioutil.ReadAll(f) + cnt, err := io.ReadAll(f) c.Assert(err, IsNil) c.Assert(string(cnt), Equals, "foo") diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index b5f137f29..251077a61 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "os" "path/filepath" "testing" @@ -510,7 +509,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { b.Fatal(err) } - if _, err := ioutil.ReadAll(r); err != nil { + if _, err := io.ReadAll(r); err != nil { b.Fatal(err) } diff --git a/storage/test/storage_suite.go b/storage/test/storage_suite.go index 2c00e75fe..ee67fc791 100644 --- a/storage/test/storage_suite.go +++ b/storage/test/storage_suite.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" @@ -502,12 +501,12 @@ func objectEquals(a plumbing.EncodedObject, b plumbing.EncodedObject) error { return fmt.Errorf("can't get reader on b: %q", err) } - ca, err := ioutil.ReadAll(ra) + ca, err := io.ReadAll(ra) if err != nil { return fmt.Errorf("error reading a: %q", err) } - cb, err := ioutil.ReadAll(rb) + cb, err := io.ReadAll(rb) if err != nil { return fmt.Errorf("error reading b: %q", err) } diff --git a/utils/ioutil/common_test.go b/utils/ioutil/common_test.go index 27bfa62ed..e3c9d69fa 100644 --- a/utils/ioutil/common_test.go +++ b/utils/ioutil/common_test.go @@ -3,7 +3,7 @@ package ioutil import ( "bytes" "context" - "io/ioutil" + "io" "strings" "testing" @@ -38,7 +38,7 @@ func (s *CommonSuite) TestNonEmptyReader_NonEmpty(c *C) { c.Assert(err, IsNil) c.Assert(r, NotNil) - read, err := ioutil.ReadAll(r) + read, err := io.ReadAll(r) c.Assert(err, IsNil) c.Assert(string(read), Equals, "1") } @@ -48,7 +48,7 @@ func (s *CommonSuite) TestNewReadCloser(c *C) { closer := &closer{} r := NewReadCloser(buf, closer) - read, err := ioutil.ReadAll(r) + read, err := io.ReadAll(r) c.Assert(err, IsNil) c.Assert(string(read), Equals, "1") @@ -160,7 +160,7 @@ func ExampleCheckClose() { // CheckClose is commonly used with named return values f := func() (err error) { // Get a io.ReadCloser - r := ioutil.NopCloser(strings.NewReader("foo")) + r := io.NopCloser(strings.NewReader("foo")) // defer CheckClose call with an io.Closer and pointer to error defer CheckClose(r, &err) diff --git a/worktree.go b/worktree.go index 7d2f3b47f..595dceaae 100644 --- a/worktree.go +++ b/worktree.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - stdioutil "io/ioutil" "os" "path/filepath" "strings" @@ -569,7 +568,7 @@ func (w *Worktree) checkoutFileSymlink(f *object.File) (err error) { defer ioutil.CheckClose(from, &err) - bytes, err := stdioutil.ReadAll(from) + bytes, err := io.ReadAll(from) if err != nil { return } @@ -718,7 +717,7 @@ func (w *Worktree) readGitmodulesFile() (*config.Modules, error) { } defer f.Close() - input, err := stdioutil.ReadAll(f) + input, err := io.ReadAll(f) if err != nil { return nil, err } diff --git a/worktree_test.go b/worktree_test.go index 4f55844a9..ea55ac6f3 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "io/ioutil" "os" "path/filepath" "regexp" @@ -81,7 +80,7 @@ func (s *WorktreeSuite) TestPullFastForward(c *C) { w, err := server.Worktree() c.Assert(err, IsNil) - err = ioutil.WriteFile(filepath.Join(path, "foo"), []byte("foo"), 0755) + err = os.WriteFile(filepath.Join(path, "foo"), []byte("foo"), 0755) c.Assert(err, IsNil) hash, err := w.Commit("foo", &CommitOptions{Author: defaultSignature()}) c.Assert(err, IsNil) @@ -118,14 +117,14 @@ func (s *WorktreeSuite) TestPullNonFastForward(c *C) { w, err := server.Worktree() c.Assert(err, IsNil) - err = ioutil.WriteFile(filepath.Join(path, "foo"), []byte("foo"), 0755) + err = os.WriteFile(filepath.Join(path, "foo"), []byte("foo"), 0755) c.Assert(err, IsNil) _, err = w.Commit("foo", &CommitOptions{Author: defaultSignature()}) c.Assert(err, IsNil) w, err = r.Worktree() c.Assert(err, IsNil) - err = ioutil.WriteFile(filepath.Join(path, "bar"), []byte("bar"), 0755) + err = os.WriteFile(filepath.Join(path, "bar"), []byte("bar"), 0755) c.Assert(err, IsNil) _, err = w.Commit("bar", &CommitOptions{Author: defaultSignature()}) c.Assert(err, IsNil) @@ -287,7 +286,7 @@ func (s *WorktreeSuite) TestPullAlreadyUptodate(c *C) { w, err := r.Worktree() c.Assert(err, IsNil) - err = ioutil.WriteFile(filepath.Join(path, "bar"), []byte("bar"), 0755) + err = os.WriteFile(filepath.Join(path, "bar"), []byte("bar"), 0755) c.Assert(err, IsNil) _, err = w.Commit("bar", &CommitOptions{Author: defaultSignature()}) c.Assert(err, IsNil) @@ -315,7 +314,7 @@ func (s *WorktreeSuite) TestCheckout(c *C) { ch, err := fs.Open("CHANGELOG") c.Assert(err, IsNil) - content, err := ioutil.ReadAll(ch) + content, err := io.ReadAll(ch) c.Assert(err, IsNil) c.Assert(string(content), Equals, "Initial changelog\n") From cb72b58b7576e28627e58338c92e61d3d41a72d7 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Thu, 11 May 2023 22:01:31 +0100 Subject: [PATCH 24/31] *: Replace fmt.Sprintf with net.JoinHostPort Signed-off-by: Paulo Gomes --- plumbing/transport/git/common.go | 5 +++-- plumbing/transport/ssh/common.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plumbing/transport/git/common.go b/plumbing/transport/git/common.go index c18d600c2..92fc0becc 100644 --- a/plumbing/transport/git/common.go +++ b/plumbing/transport/git/common.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net" + "strconv" "github.com/go-git/go-git/v5/plumbing/format/pktline" "github.com/go-git/go-git/v5/plumbing/transport" @@ -69,7 +70,7 @@ func (c *command) getHostWithPort() string { port = DefaultPort } - return fmt.Sprintf("%s:%d", host, port) + return net.JoinHostPort(host, strconv.Itoa(port)) } // StderrPipe git protocol doesn't have any dedicated error channel @@ -92,7 +93,7 @@ func (c *command) StdoutPipe() (io.Reader, error) { func endpointToCommand(cmd string, ep *transport.Endpoint) string { host := ep.Host if ep.Port != DefaultPort { - host = fmt.Sprintf("%s:%d", ep.Host, ep.Port) + host = net.JoinHostPort(ep.Host, strconv.Itoa(ep.Port)) } return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path, 0, host, 0) diff --git a/plumbing/transport/ssh/common.go b/plumbing/transport/ssh/common.go index 6617d9b71..15316038b 100644 --- a/plumbing/transport/ssh/common.go +++ b/plumbing/transport/ssh/common.go @@ -212,7 +212,7 @@ func (c *command) getHostWithPort() string { port = DefaultPort } - return fmt.Sprintf("%s:%d", host, port) + return net.JoinHostPort(host, strconv.Itoa(port)) } func (c *command) doGetHostWithPortFromSSHConfig() (addr string, found bool) { @@ -240,7 +240,7 @@ func (c *command) doGetHostWithPortFromSSHConfig() (addr string, found bool) { } } - addr = fmt.Sprintf("%s:%d", host, port) + addr = net.JoinHostPort(host, strconv.Itoa(port)) return } From bddd89e697ebdaad2fe341b3e8a3233691f5544a Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Thu, 11 May 2023 22:14:26 +0100 Subject: [PATCH 25/31] *: Add missing error checks Some areas of the code base were missing error checks, without them it may be harder to troubleshoot unexpected behaviours. Signed-off-by: Paulo Gomes --- plumbing/format/idxfile/writer.go | 24 +++++++++++++++++------- plumbing/protocol/packp/advrefs.go | 14 +++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index e5560130f..c4c21e167 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -136,15 +136,23 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { offset := o.Offset if offset > math.MaxInt32 { - offset = w.addOffset64(offset) + var err error + offset, err = w.addOffset64(offset) + if err != nil { + return nil, err + } } buf.Truncate(0) - binary.WriteUint32(buf, uint32(offset)) + if err := binary.WriteUint32(buf, uint32(offset)); err != nil { + return nil, err + } idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) buf.Truncate(0) - binary.WriteUint32(buf, o.CRC32) + if err := binary.WriteUint32(buf, o.CRC32); err != nil { + return nil, err + } idx.CRC32[bucket] = append(idx.CRC32[bucket], buf.Bytes()...) } @@ -158,15 +166,17 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { return idx, nil } -func (w *Writer) addOffset64(pos uint64) uint64 { +func (w *Writer) addOffset64(pos uint64) (uint64, error) { buf := new(bytes.Buffer) - binary.WriteUint64(buf, pos) - w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) + if err := binary.WriteUint64(buf, pos); err != nil { + return 0, err + } + w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) index := uint64(w.offset64 | (1 << 31)) w.offset64++ - return index + return index, nil } func (o objects) Len() int { diff --git a/plumbing/protocol/packp/advrefs.go b/plumbing/protocol/packp/advrefs.go index 1bd724cad..f93ad3047 100644 --- a/plumbing/protocol/packp/advrefs.go +++ b/plumbing/protocol/packp/advrefs.go @@ -57,7 +57,7 @@ func (a *AdvRefs) AddReference(r *plumbing.Reference) error { switch r.Type() { case plumbing.SymbolicReference: v := fmt.Sprintf("%s:%s", r.Name().String(), r.Target().String()) - a.Capabilities.Add(capability.SymRef, v) + return a.Capabilities.Add(capability.SymRef, v) case plumbing.HashReference: a.References[r.Name().String()] = r.Hash() default: @@ -96,12 +96,12 @@ func (a *AdvRefs) addRefs(s storer.ReferenceStorer) error { // // Git versions prior to 1.8.4.3 has an special procedure to get // the reference where is pointing to HEAD: -// - Check if a reference called master exists. If exists and it -// has the same hash as HEAD hash, we can say that HEAD is pointing to master -// - If master does not exists or does not have the same hash as HEAD, -// order references and check in that order if that reference has the same -// hash than HEAD. If yes, set HEAD pointing to that branch hash -// - If no reference is found, throw an error +// - Check if a reference called master exists. If exists and it +// has the same hash as HEAD hash, we can say that HEAD is pointing to master +// - If master does not exists or does not have the same hash as HEAD, +// order references and check in that order if that reference has the same +// hash than HEAD. If yes, set HEAD pointing to that branch hash +// - If no reference is found, throw an error func (a *AdvRefs) resolveHead(s storer.ReferenceStorer) error { if a.Head == nil { return nil From f47bb2d633347913d5575a42e25872c38569a4a3 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Tue, 2 May 2023 21:05:24 +1000 Subject: [PATCH 26/31] git: remote, add support for a configurable timeout. The previous hard-coded 10 second value is too short for listing large repositories like https://gitlab.com/gitlab-org/gitlab Return an error on nonsensical subzero timeout values --- options.go | 2 ++ remote.go | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/options.go b/options.go index 3d75e0335..d607b3078 100644 --- a/options.go +++ b/options.go @@ -640,6 +640,8 @@ type ListOptions struct { PeelingOption PeelingOption // ProxyOptions provides info required for connecting to a proxy. ProxyOptions transport.ProxyOptions + // Timeout specifies the timeout in seconds for list operations + Timeout int } // PeelingOption represents the different ways to handle peeled references. diff --git a/remote.go b/remote.go index df2649112..7d44c5b91 100644 --- a/remote.go +++ b/remote.go @@ -1258,7 +1258,15 @@ func (r *Remote) ListContext(ctx context.Context, o *ListOptions) (rfs []*plumbi } func (r *Remote) List(o *ListOptions) (rfs []*plumbing.Reference, err error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + timeout := o.Timeout + // Default to the old hardcoded 10s value if a timeout is not explicitly set. + if timeout == 0 { + timeout = 10 + } + if timeout < 0 { + return nil, fmt.Errorf("invalid timeout: %d", timeout) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() return r.ListContext(ctx, o) } From 1aa8e8940336aa80eccdd8dd9b46b0e6547e7127 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Sun, 21 May 2023 02:24:18 -0400 Subject: [PATCH 27/31] git: Allow Initial Branch to be configurable --- plumbing/reference.go | 1 + repository.go | 18 +++++++++++++++- repository_test.go | 48 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/plumbing/reference.go b/plumbing/reference.go index eef11e827..aeb4227bf 100644 --- a/plumbing/reference.go +++ b/plumbing/reference.go @@ -126,6 +126,7 @@ func (r ReferenceName) Short() string { const ( HEAD ReferenceName = "HEAD" Master ReferenceName = "refs/heads/master" + Main ReferenceName = "refs/heads/main" ) // Reference is a representation of git reference diff --git a/repository.go b/repository.go index 287b597bb..ffd4f0116 100644 --- a/repository.go +++ b/repository.go @@ -71,14 +71,30 @@ type Repository struct { wt billy.Filesystem } +type InitOptions struct { + // The default branch (e.g. "refs/heads/master") + DefaultBranch plumbing.ReferenceName +} + // Init creates an empty git repository, based on the given Storer and worktree. // The worktree Filesystem is optional, if nil a bare repository is created. If // the given storer is not empty ErrRepositoryAlreadyExists is returned func Init(s storage.Storer, worktree billy.Filesystem) (*Repository, error) { + options := InitOptions{ + DefaultBranch: plumbing.Master, + } + return InitWithOptions(s, worktree, options) +} + +func InitWithOptions(s storage.Storer, worktree billy.Filesystem, options InitOptions) (*Repository, error) { if err := initStorer(s); err != nil { return nil, err } + if options.DefaultBranch == "" { + options.DefaultBranch = plumbing.Master + } + r := newRepository(s, worktree) _, err := r.Reference(plumbing.HEAD, false) switch err { @@ -89,7 +105,7 @@ func Init(s storage.Storer, worktree billy.Filesystem) (*Repository, error) { return nil, err } - h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.Master) + h := plumbing.NewSymbolicReference(plumbing.HEAD, options.DefaultBranch) if err := s.SetReference(h); err != nil { return nil, err } diff --git a/repository_test.go b/repository_test.go index 0080a8384..f9926727a 100644 --- a/repository_test.go +++ b/repository_test.go @@ -52,6 +52,54 @@ func (s *RepositorySuite) TestInit(c *C) { cfg, err := r.Config() c.Assert(err, IsNil) c.Assert(cfg.Core.IsBare, Equals, false) + + // check the HEAD to see what the default branch is + createCommit(c, r) + ref, err := r.Head() + c.Assert(err, IsNil) + c.Assert(ref.Name().String(), Equals, plumbing.Master.String()) +} + +func (s *RepositorySuite) TestInitWithOptions(c *C) { + r, err := InitWithOptions(memory.NewStorage(), memfs.New(), InitOptions{ + DefaultBranch: "refs/heads/foo", + }) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + createCommit(c, r) + + ref, err := r.Head() + c.Assert(err, IsNil) + c.Assert(ref.Name().String(), Equals, "refs/heads/foo") + +} + +func createCommit(c *C, r *Repository) { + // Create a commit so there is a HEAD to check + wt, err := r.Worktree() + c.Assert(err, IsNil) + + rm, err := wt.Filesystem.Create("foo.txt") + c.Assert(err, IsNil) + + _, err = rm.Write([]byte("foo text")) + c.Assert(err, IsNil) + + _, err = wt.Add("foo.txt") + c.Assert(err, IsNil) + + author := object.Signature{ + Name: "go-git", + Email: "go-git@fake.local", + When: time.Now(), + } + _, err = wt.Commit("test commit message", &CommitOptions{ + All: true, + Author: &author, + Committer: &author, + }) + c.Assert(err, IsNil) + } func (s *RepositorySuite) TestInitNonStandardDotGit(c *C) { From a8bb14673e8d56ce012c790a88b9a372267186c8 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Sun, 21 May 2023 08:51:51 +0100 Subject: [PATCH 28/31] *: Bump Go version to 1.18 on go.mod Signed-off-by: Paulo Gomes --- go.mod | 12 +++++++++++- go.sum | 12 ------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index f062b1a8d..604a3309e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/go-git/go-git/v5 -go 1.13 +// go-git supports the last 3 stable Go versions. +go 1.18 require ( github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5 @@ -28,3 +29,12 @@ require ( golang.org/x/text v0.9.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) + +require ( + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/cloudflare/circl v1.1.0 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.2.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect +) diff --git a/go.sum b/go.sum index db7d92ba3..9c02e8eee 100644 --- a/go.sum +++ b/go.sum @@ -51,7 +51,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= @@ -73,25 +72,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= @@ -113,7 +107,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -122,16 +115,13 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= @@ -139,7 +129,6 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -154,4 +143,3 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From b2576e56b9ad4f6e5bb3400e74d1f74c501f7d40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 May 2023 15:13:02 +0000 Subject: [PATCH 29/31] build(deps): bump github.com/cloudflare/circl from 1.1.0 to 1.3.3 Bumps [github.com/cloudflare/circl](https://github.com/cloudflare/circl) from 1.1.0 to 1.3.3. - [Release notes](https://github.com/cloudflare/circl/releases) - [Commits](https://github.com/cloudflare/circl/compare/v1.1.0...v1.3.3) --- updated-dependencies: - dependency-name: github.com/cloudflare/circl dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 604a3309e..8989f3691 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 9c02e8eee..2401aa38d 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,9 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From ad19238d2cabadf84f4cec64d3f1e7d05966c3cc Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Sun, 21 May 2023 08:57:01 +0100 Subject: [PATCH 30/31] *: Add CodeQL workflow Signed-off-by: Paulo Gomes --- .github/workflows/codeql.yml | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..fbb867cc7 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,44 @@ +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: '00 5 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + + steps: + - name: Checkout code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # v2.3.3 + with: + languages: ${{ matrix.language }} + # xref: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # xref: https://codeql.github.com/codeql-query-help/go/ + queries: security-and-quality + + - name: Manual Build + run: go build ./... + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # v2.3.3 + with: + category: "/language:${{matrix.language}}" From 7cd1ae26f8f2108570753e947100c1f38f89fd8d Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 23 May 2023 09:10:10 +0100 Subject: [PATCH 31/31] *: Bump dependencies - github.com/ProtonMail/go-crypto to 0.0.0-20230518184743-7afd39499903. - github.com/skeema/knownhosts to 1.1.1. - golang.org/x/crypto to 0.9.0. - golang.org/x/net to 0.10.0. - golang.org/x/sys to 0.8.0. Signed-off-by: Paulo Gomes --- go.mod | 10 +++++----- go.sum | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 8989f3691..ee58848c8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ module github.com/go-git/go-git/v5 go 1.18 require ( - github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5 + github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 github.com/acomagu/bufpipe v1.0.4 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 @@ -21,11 +21,11 @@ require ( github.com/kevinburke/ssh_config v1.2.0 github.com/pjbgf/sha1cd v0.3.0 github.com/sergi/go-diff v1.1.0 - github.com/skeema/knownhosts v1.1.0 + github.com/skeema/knownhosts v1.1.1 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.8.0 - golang.org/x/net v0.9.0 - golang.org/x/sys v0.7.0 + golang.org/x/crypto v0.9.0 + golang.org/x/net v0.10.0 + golang.org/x/sys v0.8.0 golang.org/x/text v0.9.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) diff --git a/go.sum b/go.sum index 2401aa38d..fcfc9f5b0 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5 h1:QXMwHM/lB4ZQhdEF7JUTNgYOJR/gWoFbgQ/2Aj1h3Dk= -github.com/ProtonMail/go-crypto v0.0.0-20230417170513-8ee5748c52b5/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek= +github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -63,8 +63,8 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYe github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= -github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= +github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= +github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -78,8 +78,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -89,8 +89,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -111,14 +111,14 @@ golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=