cloudfns: send email updates when the wiki changes
This change introduces two cloud functions: wikiwebhook and
sendwikidiff. The former handles GitHub Wiki change events
sent over HTTP and enqueues them on a pubsub topic for the
latter to pick up. sendwikidiff then checks out the wiki
repo and sends an email with the diff of the change to
golang-wikichanges@.
The reason it is split into two functions is due to GitHub’s
timeout limit on webhook handlers (ten seconds). In testing,
a cold boot of a function that does everything described
above would sometimes hit that limit.
Updates golang/go#27313
Change-Id: I1974e1434c7003482df724d6ea3b537e22231c36
Reviewed-on: https://go-review.googlesource.com/c/158642
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/cloudfns/wikiwebhook/README.md b/cloudfns/wikiwebhook/README.md
new file mode 100644
index 0000000..c3e9dcb
--- /dev/null
+++ b/cloudfns/wikiwebhook/README.md
@@ -0,0 +1,9 @@
+# wikiwebhook Cloud Function
+
+```sh
+gcloud functions deploy GitHubWikiChangeWebHook \
+ --project=symbolic-datum-552 \
+ --runtime go111 \
+ --trigger-http \
+ --set-env-vars=PUBSUB_TOPIC=github.webhooks.golang.go.wiki,GITHUB_WEBHOOK_SECRET=<github webhook secret>
+```
diff --git a/cloudfns/wikiwebhook/go.mod b/cloudfns/wikiwebhook/go.mod
new file mode 100644
index 0000000..72c81c2
--- /dev/null
+++ b/cloudfns/wikiwebhook/go.mod
@@ -0,0 +1,10 @@
+module golang.org/x/build/cloudfns/wikiwebhook
+
+require (
+ cloud.google.com/go v0.34.0
+ github.com/googleapis/gax-go v2.0.2+incompatible // indirect
+ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
+ google.golang.org/api v0.1.0 // indirect
+ google.golang.org/genproto v0.0.0-20190111180523-db91494dd46c // indirect
+ google.golang.org/grpc v1.18.0 // indirect
+)
diff --git a/cloudfns/wikiwebhook/go.sum b/cloudfns/wikiwebhook/go.sum
new file mode 100644
index 0000000..9de6b63
--- /dev/null
+++ b/cloudfns/wikiwebhook/go.sum
@@ -0,0 +1,64 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
+github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953 h1:LuZIitY8waaxUfNIdtajyE/YzA/zyf0YxXG27VpLrkg=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190111180523-db91494dd46c h1:LZllHYjdJnynBfmwysp+s4yhMzfc+3BzhdqzAMvwjoc=
+google.golang.org/genproto v0.0.0-20190111180523-db91494dd46c/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA=
+google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/cloudfns/wikiwebhook/wikiwebhook.go b/cloudfns/wikiwebhook/wikiwebhook.go
new file mode 100644
index 0000000..9daca95
--- /dev/null
+++ b/cloudfns/wikiwebhook/wikiwebhook.go
@@ -0,0 +1,99 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package wikiwebhook implements an Google Cloud Function HTTP handler that
+// expects GitHub webhook change events. Specifically, it reacts to wiki change
+// events and posts the payload to a pubsub topic.
+package wikiwebhook
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha1"
+ "encoding/hex"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+
+ "cloud.google.com/go/pubsub"
+)
+
+var (
+ githubSecret = os.Getenv("GITHUB_WEBHOOK_SECRET")
+ projectID = os.Getenv("GCP_PROJECT")
+ pubsubTopic = os.Getenv("PUBSUB_TOPIC")
+)
+
+func GitHubWikiChangeWebHook(w http.ResponseWriter, r *http.Request) {
+ defer r.Body.Close()
+ body, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Could not read request body: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ if !validSignature(body, []byte(githubSecret), r.Header.Get("X-Hub-Signature")) {
+ http.Error(w, "signature mismatch", http.StatusUnauthorized)
+ return
+ }
+
+ evt := r.Header.Get("X-GitHub-Event")
+ // Ping event is sent upon initial setup of the webhook.
+ if evt == "ping" {
+ fmt.Fprintf(w, "pong")
+ return
+ }
+ // See https://developer.github.com/v3/activity/events/types/#gollumevent.
+ if evt != "gollum" {
+ http.Error(w, fmt.Sprintf("incorrect event type %q", evt), http.StatusBadRequest)
+ return
+ }
+
+ id, err := publishToTopic(pubsubTopic, body)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Unable to publish to topic: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ fmt.Fprintf(w, "Message ID: %s\n", id)
+}
+
+// publishToTopic publishes body to the given topic.
+// It returns the ID of the message published.
+var publishToTopic = func(topic string, body []byte) (string, error) {
+ ctx := context.Background()
+ client, err := pubsub.NewClient(ctx, projectID)
+ if err != nil {
+ return "", fmt.Errorf("pubsub.NewClient: %v", err)
+ }
+
+ t := client.Topic(topic)
+ resp := t.Publish(ctx, &pubsub.Message{Data: body})
+ id, err := resp.Get(ctx)
+ if err != nil {
+ return "", fmt.Errorf("topic.Publish: %v", err)
+ }
+ return id, nil
+}
+
+// validSignature reports whether the HMAC-SHA1 of body with key matches sig,
+// which is in the form "sha1=<HMAC-SHA1 in hex>".
+func validSignature(body, key []byte, sig string) bool {
+ const prefix = "sha1="
+ if len(sig) < len(prefix) {
+ return false
+ }
+ sig = sig[len(prefix):]
+ mac := hmac.New(sha1.New, key)
+ mac.Write(body)
+ b, err := hex.DecodeString(sig)
+ if err != nil {
+ return false
+ }
+
+ // Use hmac.Equal to avoid timing attacks.
+ return hmac.Equal(mac.Sum(nil), b)
+}
diff --git a/cloudfns/wikiwebhook/wikiwebhook_test.go b/cloudfns/wikiwebhook/wikiwebhook_test.go
new file mode 100644
index 0000000..857995c
--- /dev/null
+++ b/cloudfns/wikiwebhook/wikiwebhook_test.go
@@ -0,0 +1,119 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wikiwebhook
+
+import (
+ "bytes"
+ "errors"
+ "io/ioutil"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestValidSignature(t *testing.T) {
+ testCases := []struct {
+ body, key []byte
+ sig string
+ matches bool
+ }{
+ {[]byte("body"), []byte("key"), "sha1=70bbf6819d1037aa94ca7e7f537cbea25fe49283", true},
+ {[]byte("body"), []byte("key"), "sha1=70bbf6819d1037aa94ca7e7f537cbea25fe49284", false},
+ {[]byte{}, []byte{}, "", false},
+ {[]byte{}, []byte{}, "sha1=not a valid hex string", false},
+ }
+ for _, tc := range testCases {
+ if matches := validSignature(tc.body, tc.key, tc.sig); matches != tc.matches {
+ t.Errorf("expected match = %v; got match = %v\nbody: %q, key: %q, sig: %q", tc.matches, matches, tc.body, tc.key, tc.sig)
+ }
+ }
+}
+
+func TestWebHook(t *testing.T) {
+ testCases := []struct {
+ desc string
+ body []byte
+ headers map[string]string
+ publishFn func(string, []byte) (string, error)
+ statusCode int
+ respBody []byte
+ }{
+ {
+ "invalid signature",
+ nil,
+ map[string]string{
+ "X-Hub-Signature": "sha1=invalid",
+ },
+ nil,
+ 401,
+ []byte("signature mismatch\n"),
+ },
+ {
+ "ping event",
+ nil,
+ map[string]string{
+ "X-Hub-Signature": "sha1=fbdb1d1b18aa6c08324b7d64b71fb76370690e1d",
+ "X-GitHub-Event": "ping",
+ },
+ nil,
+ 200,
+ []byte("pong"),
+ },
+ {
+ "wiki change event",
+ []byte("body"),
+ map[string]string{
+ "X-Hub-Signature": "sha1=cc5e6b2b046bc7401d071a3d9be9a1cf1869376d",
+ "X-GitHub-Event": "gollum",
+ },
+ func(topic string, body []byte) (string, error) {
+ if got, want := body, []byte("body"); !bytes.Equal(got, want) {
+ t.Errorf("unexpected body: got %q; expected %q", got, want)
+ }
+ return "42", nil
+ },
+ 200,
+ []byte("Message ID: 42\n"),
+ },
+ {
+ "error publishing topic",
+ nil,
+ map[string]string{
+ "X-Hub-Signature": "sha1=fbdb1d1b18aa6c08324b7d64b71fb76370690e1d",
+ "X-GitHub-Event": "gollum",
+ },
+ func(topic string, body []byte) (string, error) {
+ return "", errors.New("publishToTopic error")
+ },
+ 500,
+ []byte("publishToTopic error\n"),
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.desc, func(t *testing.T) {
+ oldFn := publishToTopic
+ defer func() { publishToTopic = oldFn }()
+ publishToTopic = tc.publishFn
+
+ req := httptest.NewRequest("GET", "http://cloudfunctionz.com/func", bytes.NewReader(tc.body))
+ for k, v := range tc.headers {
+ req.Header.Set(k, v)
+ }
+ w := httptest.NewRecorder()
+ GitHubWikiChangeWebHook(w, req)
+
+ resp := w.Result()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Errorf("ioutil.ReadAll: %v", err)
+ }
+ if got, want := resp.StatusCode, tc.statusCode; got != want {
+ t.Errorf("Unexpected status code: got %d; want %d", got, want)
+ }
+ if !bytes.Equal(body, tc.respBody) {
+ t.Errorf("Unexpected body: got %q; want %q", body, tc.respBody)
+ }
+ })
+ }
+}
