gddo-server: fetch path on pkg.go.dev 404
When a godoc.org requests results in a 200, but a 404 on pkg.go.dev,
make a fetch request to pkg.go.dev to fetch that path and log the
output.
Change-Id: I1ca12732317d7e90b42eff022357067d21f84f9b
Reviewed-on: https://go-review.googlesource.com/c/gddo/+/279038
Trust: Julie Qiu <julie@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/gddo-server/pkgsite.go b/gddo-server/pkgsite.go
index e41c866..3ee9e71 100644
--- a/gddo-server/pkgsite.go
+++ b/gddo-server/pkgsite.go
@@ -7,6 +7,9 @@
package main
import (
+ "context"
+ "fmt"
+ "io/ioutil"
"net/http"
"net/url"
"path/filepath"
@@ -22,7 +25,10 @@
Status int
URL string
Latency time.Duration
- Error error
+ Error string
+ // If a request 404s, make a request to fetch it and store the response.
+ FetchStatus int
+ FetchResponse string
}
func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) {
@@ -39,27 +45,41 @@
Path: u.Path,
URL: u.String(),
}
- req, err := http.NewRequest("GET", u.String(), nil)
- if err != nil {
- pkgEvent.Status = http.StatusInternalServerError
- pkgEvent.Error = err
- return gddoEvent, pkgEvent
- }
start := time.Now()
+ status, errResp := makeRequest(godocReq.Context(), u.String())
+ pkgEvent.Status = status
+ pkgEvent.Latency = time.Since(start)
+ // The response will always be an error here if not empty.
+ pkgEvent.Error = errResp
+
+ if pkgEvent.Status == http.StatusNotFound && gddoEvent.Status == http.StatusOK {
+ // If the request was successful on godoc.org but returned a 404 on
+ // pkg.go.dev make a fetch request.
+ status, body := makeRequest(godocReq.Context(), "/fetch"+u.String())
+ pkgEvent.FetchStatus = status
+ pkgEvent.FetchResponse = body
+ }
+ return gddoEvent, pkgEvent
+}
+
+func makeRequest(ctx context.Context, url string) (int, string) {
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return http.StatusInternalServerError, fmt.Sprintf("http.NewRequest: %v", err)
+ }
xfwd := req.Header.Get("X-Forwarded-for")
req.Header.Set("X-Godoc-Forwarded-for", xfwd)
- resp, err := ctxhttp.Do(godocReq.Context(), http.DefaultClient, req)
+ resp, err := ctxhttp.Do(ctx, http.DefaultClient, req)
if err != nil {
// Use StatusBadGateway to indicate the upstream error.
- pkgEvent.Status = http.StatusBadGateway
- pkgEvent.Error = err
- return gddoEvent, pkgEvent
+ return http.StatusBadGateway, err.Error()
}
-
- pkgEvent.Status = resp.StatusCode
- pkgEvent.Latency = time.Since(start)
- return gddoEvent, pkgEvent
-
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return resp.StatusCode, fmt.Sprintf("can't read body: %v", err)
+ }
+ return resp.StatusCode, string(body)
}
// doNotTeeURLsToPkgGoDev are paths that should not be teed to pkg.go.dev.
