From 2b69e653690125e05907f37a6edc3248b20d9839 Mon Sep 17 00:00:00 2001 From: Masaya Suzuki Date: Thu, 21 Jun 2018 18:08:20 -0700 Subject: [PATCH 1/4] Add background and TODOs --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index aea5948..837d01c 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,57 @@ A Git protocol parser written in Go. This is more like an experimental project to better understand the protocol. This is not an official Google product (i.e. a 20% project). + +## Background + +Git protocol is defined in +[Documentation/technical/pack-protocol.txt](https://git.kernel.org/pub/scm/git/git.git/tree/Documentation/technical/pack-protocol.txt). +This is not a complete definition. Also, a transport specific spec +[Documentation/technical/http--protocol.txt](https://git.kernel.org/pub/scm/git/git.git/tree/Documentation/technical/http-protocol.txt) +is not complete. This project was started so that these upstream protocol spec +becomes more accurate. To verify the written syntax is accurate, this project +includes a Git protocol parser written in Go, and have end-to-end test suites. + +This makes it easy to write a test case for Git client. Currently the test cases +are run against the canonical Git implementation, but this can be extended to +run against JGit, etc.. Also it makes it easy to test an attack case. With this +library, one can write an attack case like +[git-bomb](https://github.com/Katee/git-bomb) against Git protocol by producing +a request that is usually not produced by a sane Git client. Protocol properties +can also be checked. For example, it's possible to write a test to check valid +request/response's prefixes are not a valid request/response. This property +makes sure a client won't process an incomplete response thinking it's complete. + +## TODOs + +* Signed push is not implemented and tested. + + There was a bug in upstream Git around singed push, and in order to get + this work, git needs to be built from the source. As of 2018-06-21, the fix + is in pu. Add bin-wrappers to the PATH to use the latest binary in the + end-to-end tests. + +* Protocol semantics is not defined. + + The syntax is relatively complete. The semantics is not even mentioned. One + idea is to define the semantics by treating the request/response as an + operation to modify Git repositories. This perspective makes it possible to + define a formal protocol semantics in a same way as programming language + formal semantics. + + Defining a simple git-push semantics seems easy. Defining a pack + negotiation semantics for shallow cloned repositories seems difficult. + +* Upstream pack-protocol.txt is not updated. + + The initial purpose, create a complete pack-protocol.txt, is not yet done. + We can start from a minor fix (e.g. capability separator in some places is + space not NUL). Also relationship between Git protocol and network + transports (HTTPS, SSH, Git wire) are good to be mentioned. + +* Bidi-transports are not tested and defined. + + Git's bidi-transports, SSH and Git-wire protocol, are not tested with this + project and the protocol syntax is also not defined. The majority of the + syntax is same, but there's a slight difference. Go has an SSH library, so + it's easy to run a test SSH server. From 25da5ed83f3d4629c3802601f9e208580b7b560f Mon Sep 17 00:00:00 2001 From: Masaya Suzuki Date: Thu, 21 Jun 2018 18:09:14 -0700 Subject: [PATCH 2/4] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 837d01c..39c0542 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This is not an official Google product (i.e. a 20% project). Git protocol is defined in [Documentation/technical/pack-protocol.txt](https://git.kernel.org/pub/scm/git/git.git/tree/Documentation/technical/pack-protocol.txt). This is not a complete definition. Also, a transport specific spec -[Documentation/technical/http--protocol.txt](https://git.kernel.org/pub/scm/git/git.git/tree/Documentation/technical/http-protocol.txt) +[Documentation/technical/http-protocol.txt](https://git.kernel.org/pub/scm/git/git.git/tree/Documentation/technical/http-protocol.txt) is not complete. This project was started so that these upstream protocol spec becomes more accurate. To verify the written syntax is accurate, this project includes a Git protocol parser written in Go, and have end-to-end test suites. From 8d2b3b1c37f6f39243e393dffd17e9d733ac4c9e Mon Sep 17 00:00:00 2001 From: Masaya Suzuki Date: Sat, 30 Jun 2018 10:29:20 -0700 Subject: [PATCH 3/4] Implement GPG push --- PROTOCOL.md | 7 +- README.md | 7 -- v1receivepackreq.go | 198 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 197 insertions(+), 15 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index d3236cd..31f8ad8 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -166,7 +166,7 @@ COMMAND ::= OID_STR SP OID_STR SP REF_NAME PUSH_CERT ::= BytesPacket("push-cert" NUL SP? CAPABILITIY_LIST? LF) BytesPacket("certificate version 0.1" LF) BytesPacket("pusher" SP ANY_STR LF) - BytesPacket("pushee" SP ANY_STR LF) + (BytesPacket("pushee" SP ANY_STR LF))? BytesPacket("nonce" SP ANY_STR LF) BytesPacket("push-option" SP ANY_STR LF)* BytesPacket(LF) @@ -211,3 +211,8 @@ will get rid of this. When a client pushes a push certificate, it sends the push options twice. We have no idea what's going on. + +### Is push cert's pushee optional? + +This can be a bug. Looking at builtin/send-pack.c, args.url is not set for this +path always. diff --git a/README.md b/README.md index 39c0542..785c33a 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,6 @@ makes sure a client won't process an incomplete response thinking it's complete. ## TODOs -* Signed push is not implemented and tested. - - There was a bug in upstream Git around singed push, and in order to get - this work, git needs to be built from the source. As of 2018-06-21, the fix - is in pu. Add bin-wrappers to the PATH to use the latest binary in the - end-to-end tests. - * Protocol semantics is not defined. The syntax is relatively complete. The semantics is not even mentioned. One diff --git a/v1receivepackreq.go b/v1receivepackreq.go index ca1b7d0..e885f16 100644 --- a/v1receivepackreq.go +++ b/v1receivepackreq.go @@ -32,7 +32,7 @@ const ( protocolV1ReceivePackRequestStateScanCert protocolV1ReceivePackRequestStateScanCertVersion protocolV1ReceivePackRequestStateScanCertPusher - protocolV1ReceivePackRequestStateScanCertPushee + protocolV1ReceivePackRequestStateScanCertPusheeOrNonce protocolV1ReceivePackRequestStateScanCertNonce protocolV1ReceivePackRequestStateScanOptionalCertPushOptions protocolV1ReceivePackRequestStateScanCertCommand @@ -75,11 +75,11 @@ func (c *ProtocolV1ReceivePackRequestChunk) EncodeToPktLine() []byte { if c.ClientShallow != "" { return BytesPacket([]byte(fmt.Sprintf("shallow %s\n", c.ClientShallow))).EncodeToPktLine() } - if len(c.Capabilities) != 0 { - return BytesPacket([]byte(fmt.Sprintf("%s %s %s\x00%s\n", c.OldObjectID, c.NewObjectID, c.RefName, strings.Join(c.Capabilities, " ")))).EncodeToPktLine() - } if c.OldObjectID != "" && c.NewObjectID != "" && c.RefName != "" { - return BytesPacket([]byte(fmt.Sprintf("%s %s %s", c.OldObjectID, c.NewObjectID, c.RefName))).EncodeToPktLine() + if len(c.Capabilities) != 0 { + return BytesPacket([]byte(fmt.Sprintf("%s %s %s\x00%s\n", c.OldObjectID, c.NewObjectID, c.RefName, strings.Join(c.Capabilities, " ")))).EncodeToPktLine() + } + return BytesPacket([]byte(fmt.Sprintf("%s %s %s\n", c.OldObjectID, c.NewObjectID, c.RefName))).EncodeToPktLine() } if c.EndOfCommands { return FlushPacket{}.EncodeToPktLine() @@ -90,6 +90,33 @@ func (c *ProtocolV1ReceivePackRequestChunk) EncodeToPktLine() []byte { if c.EndOfPushOptions { return FlushPacket{}.EncodeToPktLine() } + if c.StartOfPushCert { + return BytesPacket([]byte(fmt.Sprintf("push-cert\x00%s\n", strings.Join(c.Capabilities, " ")))).EncodeToPktLine() + } + if c.PushCertHeader { + return BytesPacket([]byte("certificate version 0.1\n")).EncodeToPktLine() + } + if c.Pusher != "" { + return BytesPacket([]byte(fmt.Sprintf("pusher %s\n", c.Pusher))).EncodeToPktLine() + } + if c.Pushee != "" { + return BytesPacket([]byte(fmt.Sprintf("pushee %s\n", c.Pushee))).EncodeToPktLine() + } + if c.Nonce != "" { + return BytesPacket([]byte(fmt.Sprintf("nonce %s\n", c.Nonce))).EncodeToPktLine() + } + if c.CertPushOption != "" { + return BytesPacket([]byte(fmt.Sprintf("push-option %s\n", c.CertPushOption))).EncodeToPktLine() + } + if c.EndOfCertPushOptions { + return BytesPacket([]byte("\n")).EncodeToPktLine() + } + if len(c.GPGSignaturePart) != 0 { + return BytesPacket(c.GPGSignaturePart).EncodeToPktLine() + } + if c.EndOfPushCert { + return BytesPacket([]byte("push-cert-end\n")).EncodeToPktLine() + } // TODO if len(c.PackStream) != 0 { return c.PackStream @@ -154,7 +181,7 @@ transition: } return true } - if bytes.HasPrefix(bp, []byte("push-cert ")) { + if bytes.HasPrefix(bp, []byte("push-cert\x00")) { r.state = protocolV1ReceivePackRequestStateScanCert goto transition } @@ -214,13 +241,170 @@ transition: return false } case protocolV1ReceivePackRequestStateScanCert: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + zss := bytes.SplitN(bp, []byte{0}, 2) + if len(zss) != 2 { + r.err = SyntaxError("cannot split into two: " + string(bp)) + return false + } + caps := []string{} + if capStr := strings.TrimPrefix(strings.TrimSuffix(string(zss[1]), "\n"), " "); capStr != "" { + // This is to avoid strings.Split("", " ") => []string{""}. + caps = strings.Split(capStr, " ") + } + r.state = protocolV1ReceivePackRequestStateScanCertVersion + r.curr = &ProtocolV1ReceivePackRequestChunk{ + Capabilities: caps, + StartOfPushCert: true, + } + return true case protocolV1ReceivePackRequestStateScanCertVersion: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + if string(bp) != "certificate version 0.1\n" { + r.err = SyntaxError(fmt.Sprintf("unexpected certificate version: %#q", string(bp))) + return false + } + r.state = protocolV1ReceivePackRequestStateScanCertPusher + r.curr = &ProtocolV1ReceivePackRequestChunk{ + PushCertHeader: true, + } + return true case protocolV1ReceivePackRequestStateScanCertPusher: - case protocolV1ReceivePackRequestStateScanCertPushee: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + ss := strings.SplitN(strings.TrimSuffix(string(bp), "\n"), " ", 2) + if len(ss) != 2 { + r.err = SyntaxError("cannot split into two: " + string(bp)) + return false + } + if ss[0] != "pusher" { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", string(bp))) + return false + } + r.state = protocolV1ReceivePackRequestStateScanCertPusheeOrNonce + r.curr = &ProtocolV1ReceivePackRequestChunk{ + Pusher: ss[1], + } + return true + case protocolV1ReceivePackRequestStateScanCertPusheeOrNonce: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + ss := strings.SplitN(strings.TrimSuffix(string(bp), "\n"), " ", 2) + if len(ss) != 2 { + r.err = SyntaxError("cannot split into two: " + string(bp)) + return false + } + if ss[0] == "nonce" { + r.state = protocolV1ReceivePackRequestStateScanCertNonce + goto transition + } + if ss[0] != "pushee" { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", string(bp))) + return false + } + r.state = protocolV1ReceivePackRequestStateScanCertNonce + r.curr = &ProtocolV1ReceivePackRequestChunk{ + Pushee: ss[1], + } + return true case protocolV1ReceivePackRequestStateScanCertNonce: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + ss := strings.SplitN(strings.TrimSuffix(string(bp), "\n"), " ", 2) + if len(ss) != 2 { + r.err = SyntaxError("cannot split into two: " + string(bp)) + return false + } + if ss[0] != "nonce" { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", string(bp))) + return false + } + r.state = protocolV1ReceivePackRequestStateScanOptionalCertPushOptions + r.curr = &ProtocolV1ReceivePackRequestChunk{ + Nonce: ss[1], + } + return true case protocolV1ReceivePackRequestStateScanOptionalCertPushOptions: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + if string(bp) == "\n" { + r.state = protocolV1ReceivePackRequestStateScanCertCommand + r.curr = &ProtocolV1ReceivePackRequestChunk{ + EndOfCertPushOptions: true, + } + return true + } + ss := strings.SplitN(strings.TrimSuffix(string(bp), "\n"), " ", 2) + if len(ss) != 2 { + r.err = SyntaxError("cannot split into two: " + string(bp)) + return false + } + if ss[0] != "push-option" { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", string(bp))) + return false + } + r.curr = &ProtocolV1ReceivePackRequestChunk{ + CertPushOption: ss[1], + } + return true case protocolV1ReceivePackRequestStateScanCertCommand: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + if string(bp) == "-----BEGIN PGP SIGNATURE-----\n" { + r.state = protocolV1ReceivePackRequestStateScanCertGPGLine + goto transition + } + ss := strings.SplitN(strings.TrimSuffix(string(bp), "\n"), " ", 3) + if len(ss) != 3 { + r.err = SyntaxError("cannot split into three: " + string(bp)) + return false + } + r.curr = &ProtocolV1ReceivePackRequestChunk{ + OldObjectID: ss[0], + NewObjectID: ss[1], + RefName: ss[2], + } + return true case protocolV1ReceivePackRequestStateScanCertGPGLine: + bp, ok := pkt.(BytesPacket) + if !ok { + r.err = SyntaxError(fmt.Sprintf("unexpected packet: %#v", pkt)) + return false + } + if string(bp) == "push-cert-end\n" { + r.state = protocolV1ReceivePackRequestStateScanPushOptions + r.curr = &ProtocolV1ReceivePackRequestChunk{ + EndOfPushCert: true, + } + return true + } + r.curr = &ProtocolV1ReceivePackRequestChunk{ + GPGSignaturePart: bp, + } + return true case protocolV1ReceivePackRequestStateScanOptionalPushOptions: if _, ok := pkt.(PackFileIndicatorPacket); ok { r.state = protocolV1ReceivePackRequestStateScanPackFile From b20ac42c6d17333a710bef4933f14051d8999d22 Mon Sep 17 00:00:00 2001 From: Son Luong Ngoc Date: Sun, 4 Jul 2021 15:45:37 +0000 Subject: [PATCH 4/4] Init go module Ensure that the project has a go.mod file for version tracking with Go module. --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c09bc29 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/google/gitprotocolio + +go 1.16