blob: 5c883f86b9927706eab0111b2831e4d2e86d8de1 [file] [log] [blame]
// Copyright 2020 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.
package main
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"path/filepath"
"strings"
"time"
"golang.org/x/mod/module"
"golang.org/x/net/context/ctxhttp"
)
type pkggodevEvent struct {
Host string
Path string
Status int
URL string
Latency time.Duration
Error string
// If a request 404s, make a request to fetch it and store the response.
FetchStatus int
FetchResponse string
}
func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) {
gddoEvent = newGDDOEvent(godocReq, latency, isRobot, status)
u := pkgGoDevURL(godocReq.URL)
// Strip the utm_source from the URL.
vals := u.Query()
vals.Del("utm_source")
u.RawQuery = vals.Encode()
pkgEvent = &pkggodevEvent{
Host: u.Host,
Path: u.Path,
URL: u.String(),
}
start := time.Now()
status, _, err := makeRequest(godocReq.Context(), u.String())
pkgEvent.Status = status
pkgEvent.Latency = time.Since(start)
if err != nil {
pkgEvent.Error = err.Error()
}
if pkgEvent.Status == http.StatusNotFound && gddoEvent.Status == http.StatusOK {
// If the request was successful on godoc.org but returned a 404 on
// pkg.go.dev make a fetch request.
status, body, err := makeRequest(godocReq.Context(), "/fetch"+u.String())
pkgEvent.FetchStatus = status
if err != nil {
pkgEvent.Error = err.Error()
}
pkgEvent.FetchResponse = body
}
return gddoEvent, pkgEvent
}
func makeRequest(ctx context.Context, url string) (int, string, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return http.StatusInternalServerError, "", fmt.Errorf("http.NewRequest: %v", err)
}
xfwd := req.Header.Get("X-Forwarded-for")
req.Header.Set("X-Godoc-Forwarded-for", xfwd)
log.Printf("sending request to pkg.go.dev: %q", url)
resp, err := ctxhttp.Do(ctx, http.DefaultClient, req)
if err != nil {
// Use StatusBadGateway to indicate the upstream error.
return http.StatusBadGateway, "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return resp.StatusCode, "", fmt.Errorf("can't read body: %v", err)
}
return resp.StatusCode, string(body), nil
}
// doNotTeeURLsToPkgGoDev are paths that should not be teed to pkg.go.dev.
var doNotTeeURLsToPkgGoDev = map[string]bool{
"/-/bot": true,
"/-/refresh": true,
}
// doNotTeeExtsToPkgGoDev are URL extensions that should not be teed to
// pkg.go.dev.
var doNotTeeExtsToPkgGoDev = map[string]bool{
".css": true,
".html": true,
".js": true,
".txt": true,
".xml": true,
".ico": true,
}
// shouldTeeRequest reports whether a request should be teed to pkg.go.dev.
func shouldTeeRequest(u string) bool {
// Don't tee App Engine requests to pkg.go.dev.
if strings.HasPrefix(u, "/_ah/") {
return false
}
ext := filepath.Ext(u)
if doNotTeeExtsToPkgGoDev[ext] {
return false
}
if doNotTeeURLsToPkgGoDev[u] {
return false
}
return true
}
type gddoEvent struct {
Host string
Path string
Status int
URL string
Header http.Header
Latency time.Duration
IsRobot bool
UsePkgGoDev bool
Error error
}
func newGDDOEvent(r *http.Request, latency time.Duration, isRobot bool, status int) *gddoEvent {
targetURL := url.URL{
Scheme: "https",
Host: r.URL.Host,
Path: r.URL.Path,
RawQuery: r.URL.RawQuery,
}
if targetURL.Host == "" && r.Host != "" {
targetURL.Host = r.Host
}
return &gddoEvent{
Host: targetURL.Host,
Path: r.URL.Path,
Status: status,
URL: targetURL.String(),
Header: r.Header,
Latency: latency,
IsRobot: isRobot,
UsePkgGoDev: shouldRedirectToPkgGoDev(r),
}
}
func userReturningFromPkgGoDev(req *http.Request) bool {
return req.FormValue("utm_source") == "backtogodoc"
}
const (
pkgGoDevRedirectCookie = "pkggodev-redirect"
pkgGoDevRedirectParam = "redirect"
pkgGoDevRedirectOn = "on"
pkgGoDevRedirectOff = "off"
pkgGoDevHost = "pkg.go.dev"
)
func shouldRedirectToPkgGoDev(req *http.Request) bool {
// API requests are not redirected.
if strings.HasPrefix(req.URL.Host, "api") {
return false
}
redirectParam := req.FormValue(pkgGoDevRedirectParam)
if redirectParam == pkgGoDevRedirectOn || redirectParam == pkgGoDevRedirectOff {
return redirectParam == pkgGoDevRedirectOn
}
cookie, err := req.Cookie(pkgGoDevRedirectCookie)
return (err == nil && cookie.Value == pkgGoDevRedirectOn)
}
// pkgGoDevRedirectHandler redirects requests from godoc.org to pkg.go.dev,
// based on whether a cookie is set for pkggodev-redirect. The cookie
// can be turned on/off using a query param.
func pkgGoDevRedirectHandler(f func(http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) error {
return func(w http.ResponseWriter, r *http.Request) error {
if shouldRedirectRequest(r) {
http.Redirect(w, r, pkgGoDevURL(r.URL).String(), http.StatusFound)
return nil
}
if userReturningFromPkgGoDev(r) {
return f(w, r)
}
redirectParam := r.FormValue(pkgGoDevRedirectParam)
if redirectParam == pkgGoDevRedirectOn {
cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: redirectParam, Path: "/"}
http.SetCookie(w, cookie)
}
if redirectParam == pkgGoDevRedirectOff {
cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: "", MaxAge: -1, Path: "/"}
http.SetCookie(w, cookie)
}
if !shouldRedirectToPkgGoDev(r) {
return f(w, r)
}
http.Redirect(w, r, pkgGoDevURL(r.URL).String(), http.StatusFound)
return nil
}
}
// alwaysRedirectPaths is a set of paths that are always redirected to
// pkg.go.dev.
var alwaysRedirectPaths = map[string]bool{
"/-/about": true,
}
func shouldRedirectRequest(r *http.Request) bool {
// Requests to api.godoc.org and talks.godoc.org are not redirected.
if strings.HasPrefix(r.URL.Host, "api") || strings.HasPrefix(r.URL.Host, "talks") {
return false
}
// Badge SVGs will be redirected last.
_, isSVG := r.URL.Query()["status.svg"]
_, isPNG := r.URL.Query()["status.png"]
if isSVG || isPNG {
return false
}
if alwaysRedirectPaths[r.URL.Path] {
return true
}
// TODO: redirect based on rollout percentage.
return false
}
const goGithubRepoURLPath = "/github.com/golang/go"
func pkgGoDevURL(godocURL *url.URL) *url.URL {
u := &url.URL{Scheme: "https", Host: pkgGoDevHost}
q := url.Values{"utm_source": []string{"godoc"}}
if strings.Contains(godocURL.Path, "/vendor/") || strings.HasSuffix(godocURL.Path, "/vendor") {
u.Path = "/"
u.RawQuery = q.Encode()
return u
}
if strings.HasPrefix(godocURL.Path, goGithubRepoURLPath) ||
strings.HasPrefix(godocURL.Path, goGithubRepoURLPath+"/src") {
u.Path = strings.TrimPrefix(strings.TrimPrefix(godocURL.Path, goGithubRepoURLPath), "/src")
if u.Path == "" {
u.Path = "/std"
}
u.RawQuery = q.Encode()
return u
}
_, isSVG := godocURL.Query()["status.svg"]
_, isPNG := godocURL.Query()["status.png"]
if isSVG || isPNG {
u.Path = "/badge" + godocURL.Path
u.RawQuery = q.Encode()
return u
}
switch godocURL.Path {
case "/-/go":
u.Path = "/std"
case "/-/about":
u.Path = "/about"
case "/C":
u.Path = "/C"
case "/":
if qparam := godocURL.Query().Get("q"); qparam != "" {
u.Path = "/search"
q.Set("q", qparam)
} else {
u.Path = "/"
}
case "":
u.Path = ""
case "/-/subrepo":
u.Path = "/search"
q.Set("q", "golang.org/x")
default:
{
// If the import path is invalid, redirect to
// https://golang.org/issue/43036, so that the users has more context
// on why this path does not work on pkg.go.dev.
if err := module.CheckImportPath(strings.TrimPrefix(godocURL.Path, "/")); err != nil {
u.Host = "golang.org"
u.Path = "/issue/43036"
return u
}
u.Path = godocURL.Path
if _, ok := godocURL.Query()["imports"]; ok {
q.Set("tab", "imports")
} else if _, ok := godocURL.Query()["importers"]; ok {
q.Set("tab", "importedby")
}
}
}
u.RawQuery = q.Encode()
return u
}