gddo-server: tee requests directly to pkg.go.dev Change-Id: Ibe8c66efd7771421709695e94a94eb5e945ba5f4 Reviewed-on: https://go-review.googlesource.com/c/gddo/+/274860 Trust: Julie Qiu <julie@golang.org> Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/gddo-server/main.go b/gddo-server/main.go index 99e98a7..f34fc0f 100644 --- a/gddo-server/main.go +++ b/gddo-server/main.go
@@ -1046,7 +1046,7 @@ if f, ok := w.(http.Flusher); ok { f.Flush() } - makePkgGoDevRequest(r, latency, s.isRobot(r), translateStatus(w2.status)) + s.teeRequestToPkgGoDev(r, latency, translateStatus(w2.status)) } func (s *server) logRequestStart(req *http.Request) { @@ -1074,6 +1074,39 @@ }) } +func (s *server) teeRequestToPkgGoDev(r *http.Request, latency time.Duration, status int) { + if !shouldTeeRequest(r.URL.Path) { + log.Printf("teeRequestToPkgGoDev(%q): not teeing request", r.URL.Path) + return + } + if os.Getenv("GDDO_TEE_REQUESTS_TO_PKGGODEV") == "true" { + gddoEvent, pkggodevEvent := teeRequestToPkgGoDev(r, latency, s.isRobot(r), status) + payload := map[string]interface{}{ + "godoc.org": gddoEvent, + "pkg.go.dev": pkggodevEvent, + } + + if s.gceLogger == nil { + for k, v := range payload { + log.Printf("%q", k) + log.Printf("%+v", v) + } + return + } + s.gceLogger.Log(logging.Entry{ + HTTPRequest: &logging.HTTPRequest{ + Request: r, + Latency: latency, + Status: status, + }, + Payload: payload, + Severity: logging.Info, + }) + return + } + teeRequestToTeeproxy(r, latency, s.isRobot(r), status) +} + func main() { ctx := context.Background() v, err := loadConfig(ctx, os.Args)
diff --git a/gddo-server/pkgsite.go b/gddo-server/pkgsite.go index b14e320..f209f37 100644 --- a/gddo-server/pkgsite.go +++ b/gddo-server/pkgsite.go
@@ -16,27 +16,70 @@ "path/filepath" "strings" "time" + + "golang.org/x/net/context/ctxhttp" ) -// makePkgGoDevRequest makes a request to the teeproxy with data about the +type pkggodevEvent struct { + Host string + Path string + Status int + URL string + Latency time.Duration + Error error +} + +func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) { + gddoEvent = newGDDOEvent(godocReq, latency, isRobot, status) + u := pkgGoDevURL(godocReq.URL) + + // Strip the utm_source from the URL. + vals := u.Query() + vals.Del("utm_source") + u.RawQuery = vals.Encode() + + pkgEvent = &pkggodevEvent{ + Host: u.Host, + Path: u.Path, + URL: u.String(), + } + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + pkgEvent.Status = http.StatusInternalServerError + pkgEvent.Error = err + return gddoEvent, pkgEvent + } + start := time.Now() + xfwd := req.Header.Get("X-Forwarded-for") + req.Header.Set("X-Godoc-Forwarded-for", xfwd) + resp, err := ctxhttp.Do(godocReq.Context(), http.DefaultClient, req) + if err != nil { + // Use StatusBadGateway to indicate the upstream error. + pkgEvent.Status = http.StatusBadGateway + pkgEvent.Error = err + return gddoEvent, pkgEvent + } + + pkgEvent.Status = resp.StatusCode + pkgEvent.Latency = time.Since(start) + return gddoEvent, pkgEvent + +} + +// teeRequestToTeeproxy makes a request to the teeproxy with data about the // godoc.org request. -func makePkgGoDevRequest(r *http.Request, latency time.Duration, isRobot bool, status int) { +func teeRequestToTeeproxy(r *http.Request, latency time.Duration, isRobot bool, status int) { var msg string defer func() { - log.Printf("makePkgGoDevRequest(%q): %s", r.URL.Path, msg) + log.Printf("teeRequestToTeeproxy(%q): %s", r.URL.Path, msg) }() - if !shouldTeeRequest(r.URL.Path) { - msg = "not teeing request" - return - } event := newGDDOEvent(r, latency, isRobot, status) b, err := json.Marshal(event) if err != nil { msg = fmt.Sprintf("json.Marshal(%v): %v", event, err) return } - teeproxyURL := url.URL{Scheme: "https", Host: teeproxyHost} if _, err := http.Post(teeproxyURL.String(), jsonMIMEType, bytes.NewReader(b)); err != nil { msg = fmt.Sprintf("http.Post(%q, %q, %v): %v", teeproxyURL.String(), jsonMIMEType, event, err) @@ -59,6 +102,7 @@ ".js": true, ".txt": true, ".xml": true, + ".ico": true, } // shouldTeeRequest reports whether a request should be teed to pkg.go.dev. @@ -86,6 +130,7 @@ Latency time.Duration IsRobot bool UsePkgGoDev bool + Error error } func newGDDOEvent(r *http.Request, latency time.Duration, isRobot bool, status int) *gddoEvent {
diff --git a/gddo-server/pkgsite_test.go b/gddo-server/pkgsite_test.go index 8515db6..f825f5b 100644 --- a/gddo-server/pkgsite_test.go +++ b/gddo-server/pkgsite_test.go
@@ -175,12 +175,12 @@ name string url string cookie *http.Cookie - want *gddoEvent + want *requestEvent }{ { name: "home page request", url: "https://godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "", UsePkgGoDev: false, @@ -190,7 +190,7 @@ name: "home page request with cookie on should redirect", url: "https://godoc.org", cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"}, - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "", UsePkgGoDev: true, @@ -199,7 +199,7 @@ { name: "about page request", url: "https://godoc.org/-/about", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/-/about", UsePkgGoDev: false, @@ -208,7 +208,7 @@ { name: "request with search query parameter", url: "https://godoc.org/?q=test", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/", UsePkgGoDev: false, @@ -217,7 +217,7 @@ { name: "package page request", url: "https://godoc.org/net/http", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", UsePkgGoDev: false, @@ -227,7 +227,7 @@ name: "package page request with wrong cookie on should not redirect", url: "https://godoc.org/net/http", cookie: &http.Cookie{Name: "bogus-cookie", Value: "on"}, - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", UsePkgGoDev: false, @@ -237,7 +237,7 @@ name: "package page request with query parameter off should not redirect", url: "https://godoc.org/net/http?redirect=off", cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"}, - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", UsePkgGoDev: false, @@ -247,7 +247,7 @@ name: "package page request with cookie on should redirect", url: "https://godoc.org/net/http", cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"}, - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", UsePkgGoDev: true, @@ -257,7 +257,7 @@ name: "package page request with query parameter on should redirect", url: "https://godoc.org/net/http?redirect=on", cookie: &http.Cookie{Name: "pkggodev-redirect", Value: ""}, - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", UsePkgGoDev: true, @@ -266,7 +266,7 @@ { name: "api request", url: "https://api.godoc.org/imports/net/http", - want: &gddoEvent{ + want: &requestEvent{ Host: "api.godoc.org", Path: "/imports/net/http", UsePkgGoDev: false, @@ -276,7 +276,7 @@ name: "api requests should never redirect, even with cookie on", url: "https://api.godoc.org/imports/net/http", cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"}, - want: &gddoEvent{ + want: &requestEvent{ Host: "api.godoc.org", Path: "/imports/net/http", UsePkgGoDev: false, @@ -286,7 +286,7 @@ name: "api requests should never redirect, even with query parameter on", url: "https://api.godoc.org/imports/net/http?redirect=on", cookie: &http.Cookie{Name: "pkggodev-redirect", Value: ""}, - want: &gddoEvent{ + want: &requestEvent{ Host: "api.godoc.org", Path: "/imports/net/http", UsePkgGoDev: false, @@ -318,13 +318,13 @@ name string requestURI string host string - want *gddoEvent + want *requestEvent }{ { name: "absolute index path", requestURI: "https://godoc.org", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "", URL: "https://godoc.org", @@ -334,7 +334,7 @@ name: "absolute index path with trailing slash", requestURI: "https://godoc.org/", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/", URL: "https://godoc.org/", @@ -344,7 +344,7 @@ name: "relative index path", requestURI: "/", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/", URL: "https://godoc.org/", @@ -354,7 +354,7 @@ name: "absolute about path", requestURI: "https://godoc.org/-/about", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/-/about", URL: "https://godoc.org/-/about", @@ -364,7 +364,7 @@ name: "relative about path", requestURI: "/-/about", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/-/about", URL: "https://godoc.org/-/about", @@ -374,7 +374,7 @@ name: "absolute package path", requestURI: "https://godoc.org/net/http", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", URL: "https://godoc.org/net/http", @@ -384,7 +384,7 @@ name: "relative package path", requestURI: "/net/http", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", URL: "https://godoc.org/net/http", @@ -394,7 +394,7 @@ name: "absolute path with query parameters", requestURI: "https://godoc.org/net/http?q=test", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", URL: "https://godoc.org/net/http?q=test", @@ -404,7 +404,7 @@ name: "relative path with query parameters", requestURI: "/net/http?q=test", host: "godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "godoc.org", Path: "/net/http", URL: "https://godoc.org/net/http?q=test", @@ -414,7 +414,7 @@ name: "absolute api path", requestURI: "https://api.godoc.org/imports/net/http", host: "api.godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "api.godoc.org", Path: "/imports/net/http", URL: "https://api.godoc.org/imports/net/http", @@ -424,7 +424,7 @@ name: "relative api path", requestURI: "/imports/net/http", host: "api.godoc.org", - want: &gddoEvent{ + want: &requestEvent{ Host: "api.godoc.org", Path: "/imports/net/http", URL: "https://api.godoc.org/imports/net/http",
diff --git a/go.mod b/go.mod index 66b1931..97ede31 100644 --- a/go.mod +++ b/go.mod
@@ -29,6 +29,7 @@ github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1 github.com/spf13/viper v1.0.0 github.com/stretchr/testify v1.4.0 // indirect + golang.org/x/net v0.0.0-20190603091049-60506f45cf65 golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2 golang.org/x/sync v0.0.0-20170517211232-f52d1811a629 // indirect golang.org/x/time v0.0.0-20170424234030-8be79e1e0910 // indirect