gddo-server: tee requests directly to pkg.go.dev
Change-Id: Ibe8c66efd7771421709695e94a94eb5e945ba5f4
Reviewed-on: https://go-review.googlesource.com/c/gddo/+/274860
Trust: Julie Qiu <julie@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/gddo-server/main.go b/gddo-server/main.go
index 99e98a7..f34fc0f 100644
--- a/gddo-server/main.go
+++ b/gddo-server/main.go
@@ -1046,7 +1046,7 @@
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
- makePkgGoDevRequest(r, latency, s.isRobot(r), translateStatus(w2.status))
+ s.teeRequestToPkgGoDev(r, latency, translateStatus(w2.status))
}
func (s *server) logRequestStart(req *http.Request) {
@@ -1074,6 +1074,39 @@
})
}
+func (s *server) teeRequestToPkgGoDev(r *http.Request, latency time.Duration, status int) {
+ if !shouldTeeRequest(r.URL.Path) {
+ log.Printf("teeRequestToPkgGoDev(%q): not teeing request", r.URL.Path)
+ return
+ }
+ if os.Getenv("GDDO_TEE_REQUESTS_TO_PKGGODEV") == "true" {
+ gddoEvent, pkggodevEvent := teeRequestToPkgGoDev(r, latency, s.isRobot(r), status)
+ payload := map[string]interface{}{
+ "godoc.org": gddoEvent,
+ "pkg.go.dev": pkggodevEvent,
+ }
+
+ if s.gceLogger == nil {
+ for k, v := range payload {
+ log.Printf("%q", k)
+ log.Printf("%+v", v)
+ }
+ return
+ }
+ s.gceLogger.Log(logging.Entry{
+ HTTPRequest: &logging.HTTPRequest{
+ Request: r,
+ Latency: latency,
+ Status: status,
+ },
+ Payload: payload,
+ Severity: logging.Info,
+ })
+ return
+ }
+ teeRequestToTeeproxy(r, latency, s.isRobot(r), status)
+}
+
func main() {
ctx := context.Background()
v, err := loadConfig(ctx, os.Args)
diff --git a/gddo-server/pkgsite.go b/gddo-server/pkgsite.go
index b14e320..f209f37 100644
--- a/gddo-server/pkgsite.go
+++ b/gddo-server/pkgsite.go
@@ -16,27 +16,70 @@
"path/filepath"
"strings"
"time"
+
+ "golang.org/x/net/context/ctxhttp"
)
-// makePkgGoDevRequest makes a request to the teeproxy with data about the
+type pkggodevEvent struct {
+ Host string
+ Path string
+ Status int
+ URL string
+ Latency time.Duration
+ Error error
+}
+
+func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) {
+ gddoEvent = newGDDOEvent(godocReq, latency, isRobot, status)
+ u := pkgGoDevURL(godocReq.URL)
+
+ // Strip the utm_source from the URL.
+ vals := u.Query()
+ vals.Del("utm_source")
+ u.RawQuery = vals.Encode()
+
+ pkgEvent = &pkggodevEvent{
+ Host: u.Host,
+ Path: u.Path,
+ URL: u.String(),
+ }
+ req, err := http.NewRequest("GET", u.String(), nil)
+ if err != nil {
+ pkgEvent.Status = http.StatusInternalServerError
+ pkgEvent.Error = err
+ return gddoEvent, pkgEvent
+ }
+ start := time.Now()
+ xfwd := req.Header.Get("X-Forwarded-for")
+ req.Header.Set("X-Godoc-Forwarded-for", xfwd)
+ resp, err := ctxhttp.Do(godocReq.Context(), http.DefaultClient, req)
+ if err != nil {
+ // Use StatusBadGateway to indicate the upstream error.
+ pkgEvent.Status = http.StatusBadGateway
+ pkgEvent.Error = err
+ return gddoEvent, pkgEvent
+ }
+
+ pkgEvent.Status = resp.StatusCode
+ pkgEvent.Latency = time.Since(start)
+ return gddoEvent, pkgEvent
+
+}
+
+// teeRequestToTeeproxy makes a request to the teeproxy with data about the
// godoc.org request.
-func makePkgGoDevRequest(r *http.Request, latency time.Duration, isRobot bool, status int) {
+func teeRequestToTeeproxy(r *http.Request, latency time.Duration, isRobot bool, status int) {
var msg string
defer func() {
- log.Printf("makePkgGoDevRequest(%q): %s", r.URL.Path, msg)
+ log.Printf("teeRequestToTeeproxy(%q): %s", r.URL.Path, msg)
}()
- if !shouldTeeRequest(r.URL.Path) {
- msg = "not teeing request"
- return
- }
event := newGDDOEvent(r, latency, isRobot, status)
b, err := json.Marshal(event)
if err != nil {
msg = fmt.Sprintf("json.Marshal(%v): %v", event, err)
return
}
-
teeproxyURL := url.URL{Scheme: "https", Host: teeproxyHost}
if _, err := http.Post(teeproxyURL.String(), jsonMIMEType, bytes.NewReader(b)); err != nil {
msg = fmt.Sprintf("http.Post(%q, %q, %v): %v", teeproxyURL.String(), jsonMIMEType, event, err)
@@ -59,6 +102,7 @@
".js": true,
".txt": true,
".xml": true,
+ ".ico": true,
}
// shouldTeeRequest reports whether a request should be teed to pkg.go.dev.
@@ -86,6 +130,7 @@
Latency time.Duration
IsRobot bool
UsePkgGoDev bool
+ Error error
}
func newGDDOEvent(r *http.Request, latency time.Duration, isRobot bool, status int) *gddoEvent {
diff --git a/gddo-server/pkgsite_test.go b/gddo-server/pkgsite_test.go
index 8515db6..f825f5b 100644
--- a/gddo-server/pkgsite_test.go
+++ b/gddo-server/pkgsite_test.go
@@ -175,12 +175,12 @@
name string
url string
cookie *http.Cookie
- want *gddoEvent
+ want *requestEvent
}{
{
name: "home page request",
url: "https://godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "",
UsePkgGoDev: false,
@@ -190,7 +190,7 @@
name: "home page request with cookie on should redirect",
url: "https://godoc.org",
cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "",
UsePkgGoDev: true,
@@ -199,7 +199,7 @@
{
name: "about page request",
url: "https://godoc.org/-/about",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/-/about",
UsePkgGoDev: false,
@@ -208,7 +208,7 @@
{
name: "request with search query parameter",
url: "https://godoc.org/?q=test",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/",
UsePkgGoDev: false,
@@ -217,7 +217,7 @@
{
name: "package page request",
url: "https://godoc.org/net/http",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
UsePkgGoDev: false,
@@ -227,7 +227,7 @@
name: "package page request with wrong cookie on should not redirect",
url: "https://godoc.org/net/http",
cookie: &http.Cookie{Name: "bogus-cookie", Value: "on"},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
UsePkgGoDev: false,
@@ -237,7 +237,7 @@
name: "package page request with query parameter off should not redirect",
url: "https://godoc.org/net/http?redirect=off",
cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
UsePkgGoDev: false,
@@ -247,7 +247,7 @@
name: "package page request with cookie on should redirect",
url: "https://godoc.org/net/http",
cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
UsePkgGoDev: true,
@@ -257,7 +257,7 @@
name: "package page request with query parameter on should redirect",
url: "https://godoc.org/net/http?redirect=on",
cookie: &http.Cookie{Name: "pkggodev-redirect", Value: ""},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
UsePkgGoDev: true,
@@ -266,7 +266,7 @@
{
name: "api request",
url: "https://api.godoc.org/imports/net/http",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "api.godoc.org",
Path: "/imports/net/http",
UsePkgGoDev: false,
@@ -276,7 +276,7 @@
name: "api requests should never redirect, even with cookie on",
url: "https://api.godoc.org/imports/net/http",
cookie: &http.Cookie{Name: "pkggodev-redirect", Value: "on"},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "api.godoc.org",
Path: "/imports/net/http",
UsePkgGoDev: false,
@@ -286,7 +286,7 @@
name: "api requests should never redirect, even with query parameter on",
url: "https://api.godoc.org/imports/net/http?redirect=on",
cookie: &http.Cookie{Name: "pkggodev-redirect", Value: ""},
- want: &gddoEvent{
+ want: &requestEvent{
Host: "api.godoc.org",
Path: "/imports/net/http",
UsePkgGoDev: false,
@@ -318,13 +318,13 @@
name string
requestURI string
host string
- want *gddoEvent
+ want *requestEvent
}{
{
name: "absolute index path",
requestURI: "https://godoc.org",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "",
URL: "https://godoc.org",
@@ -334,7 +334,7 @@
name: "absolute index path with trailing slash",
requestURI: "https://godoc.org/",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/",
URL: "https://godoc.org/",
@@ -344,7 +344,7 @@
name: "relative index path",
requestURI: "/",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/",
URL: "https://godoc.org/",
@@ -354,7 +354,7 @@
name: "absolute about path",
requestURI: "https://godoc.org/-/about",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/-/about",
URL: "https://godoc.org/-/about",
@@ -364,7 +364,7 @@
name: "relative about path",
requestURI: "/-/about",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/-/about",
URL: "https://godoc.org/-/about",
@@ -374,7 +374,7 @@
name: "absolute package path",
requestURI: "https://godoc.org/net/http",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
URL: "https://godoc.org/net/http",
@@ -384,7 +384,7 @@
name: "relative package path",
requestURI: "/net/http",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
URL: "https://godoc.org/net/http",
@@ -394,7 +394,7 @@
name: "absolute path with query parameters",
requestURI: "https://godoc.org/net/http?q=test",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
URL: "https://godoc.org/net/http?q=test",
@@ -404,7 +404,7 @@
name: "relative path with query parameters",
requestURI: "/net/http?q=test",
host: "godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "godoc.org",
Path: "/net/http",
URL: "https://godoc.org/net/http?q=test",
@@ -414,7 +414,7 @@
name: "absolute api path",
requestURI: "https://api.godoc.org/imports/net/http",
host: "api.godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "api.godoc.org",
Path: "/imports/net/http",
URL: "https://api.godoc.org/imports/net/http",
@@ -424,7 +424,7 @@
name: "relative api path",
requestURI: "/imports/net/http",
host: "api.godoc.org",
- want: &gddoEvent{
+ want: &requestEvent{
Host: "api.godoc.org",
Path: "/imports/net/http",
URL: "https://api.godoc.org/imports/net/http",
diff --git a/go.mod b/go.mod
index 66b1931..97ede31 100644
--- a/go.mod
+++ b/go.mod
@@ -29,6 +29,7 @@
github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1
github.com/spf13/viper v1.0.0
github.com/stretchr/testify v1.4.0 // indirect
+ golang.org/x/net v0.0.0-20190603091049-60506f45cf65
golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2
golang.org/x/sync v0.0.0-20170517211232-f52d1811a629 // indirect
golang.org/x/time v0.0.0-20170424234030-8be79e1e0910 // indirect
