gddo-server/pkgsite.go

// Copyright 2020 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

package main

import (
	"context"
	"fmt"
	"hash/fnv"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/gddo/gddo-server/dynconfig"
	"github.com/golang/gddo/gddo-server/poller"
	"golang.org/x/mod/module"
	"golang.org/x/net/context/ctxhttp"
)

type pkggodevEvent struct {
	Host    string
	Path    string
	Status  int
	URL     string
	Latency time.Duration
	Error   string
	// If a request 404s, make a request to fetch it and store the response.
	FetchStatus   int
	FetchResponse string
}

func teeRequestToPkgGoDev(godocReq *http.Request, latency time.Duration, isRobot bool, status int) (gddoEvent *gddoEvent, pkgEvent *pkggodevEvent) {
	gddoEvent = newGDDOEvent(godocReq, latency, isRobot, status)
	u := pkgGoDevURL(godocReq.URL)

	// Strip the utm_source from the URL.
	vals := u.Query()
	vals.Del("utm_source")
	u.RawQuery = vals.Encode()

	pkgEvent = &pkggodevEvent{
		Host: u.Host,
		Path: u.Path,
		URL:  u.String(),
	}
	start := time.Now()
	status, _, err := makeRequest(godocReq.Context(), u.String())
	pkgEvent.Status = status
	pkgEvent.Latency = time.Since(start)
	if err != nil {
		pkgEvent.Error = err.Error()
	}

	if pkgEvent.Status == http.StatusNotFound && gddoEvent.Status == http.StatusOK {
		// If the request was successful on godoc.org but returned a 404 on
		// pkg.go.dev make a fetch request.
		status, body, err := makeRequest(godocReq.Context(), "/fetch"+u.String())
		pkgEvent.FetchStatus = status
		if err != nil {
			pkgEvent.Error = err.Error()
		}
		pkgEvent.FetchResponse = body
	}
	return gddoEvent, pkgEvent
}

func makeRequest(ctx context.Context, url string) (int, string, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return http.StatusInternalServerError, "", fmt.Errorf("http.NewRequest: %v", err)
	}
	xfwd := req.Header.Get("X-Forwarded-for")
	req.Header.Set("X-Godoc-Forwarded-for", xfwd)

	log.Printf("sending request to pkg.go.dev: %q", url)
	resp, err := ctxhttp.Do(ctx, http.DefaultClient, req)
	if err != nil {
		// Use StatusBadGateway to indicate the upstream error.
		return http.StatusBadGateway, "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return resp.StatusCode, "", fmt.Errorf("can't read body: %v", err)
	}
	return resp.StatusCode, string(body), nil
}

// doNotTeeURLsToPkgGoDev are paths that should not be teed to pkg.go.dev.
var doNotTeeURLsToPkgGoDev = map[string]bool{
	"/-/bot":     true,
	"/-/refresh": true,
}

// doNotTeeExtsToPkgGoDev are URL extensions that should not be teed to
// pkg.go.dev.
var doNotTeeExtsToPkgGoDev = map[string]bool{
	".css":  true,
	".eot":  true,
	".html": true,
	".ico":  true,
	".js":   true,
	".ttf":  true,
	".txt":  true,
	".woff": true,
	".xml":  true,
}

// shouldTeeRequest reports whether a request should be teed to pkg.go.dev.
func shouldTeeRequest(u string) bool {
	// Don't tee App Engine requests to pkg.go.dev.
	if strings.HasPrefix(u, "/_ah/") {
		return false
	}
	if strings.HasPrefix(u, "/fonts/") {
		return false
	}
	ext := filepath.Ext(u)
	if doNotTeeExtsToPkgGoDev[ext] {
		return false
	}
	if doNotTeeURLsToPkgGoDev[u] {
		return false
	}
	return true
}

type gddoEvent struct {
	Host        string
	Path        string
	Status      int
	URL         string
	Header      http.Header
	Latency     time.Duration
	IsRobot     bool
	UsePkgGoDev bool
	Error       error
}

func newGDDOEvent(r *http.Request, latency time.Duration, isRobot bool, status int) *gddoEvent {
	targetURL := url.URL{
		Scheme:   "https",
		Host:     r.URL.Host,
		Path:     r.URL.Path,
		RawQuery: r.URL.RawQuery,
	}
	if targetURL.Host == "" && r.Host != "" {
		targetURL.Host = r.Host
	}
	return &gddoEvent{
		Host:        targetURL.Host,
		Path:        r.URL.Path,
		Status:      status,
		URL:         targetURL.String(),
		Header:      r.Header,
		Latency:     latency,
		IsRobot:     isRobot,
		UsePkgGoDev: shouldRedirectToPkgGoDev(r),
	}
}

func userReturningFromPkgGoDev(req *http.Request) bool {
	return req.FormValue("utm_source") == "backtogodoc"
}

const (
	pkgGoDevRedirectCookie = "pkggodev-redirect"
	pkgGoDevRedirectParam  = "redirect"
	pkgGoDevRedirectOn     = "on"
	pkgGoDevRedirectOff    = "off"
	pkgGoDevHost           = "pkg.go.dev"
)

func shouldRedirectToPkgGoDev(req *http.Request) bool {
	// API requests are not redirected.
	if strings.HasPrefix(req.URL.Host, "api") {
		return false
	}
	redirectParam := req.FormValue(pkgGoDevRedirectParam)
	if redirectParam == pkgGoDevRedirectOn || redirectParam == pkgGoDevRedirectOff {
		return redirectParam == pkgGoDevRedirectOn
	}
	cookie, err := req.Cookie(pkgGoDevRedirectCookie)
	return (err == nil && cookie.Value == pkgGoDevRedirectOn)
}

// pkgGoDevRedirectHandler redirects requests from godoc.org to pkg.go.dev,
// based on whether a cookie is set for pkggodev-redirect. The cookie
// can be turned on/off using a query param.
func pkgGoDevRedirectHandler(configPoller *poller.Poller, f func(http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) error {
	return func(w http.ResponseWriter, r *http.Request) error {
		redirectURL := pkgGoDevURL(r.URL).String()
		if configPoller != nil {
			if shouldRedirectURL(r, configPoller) {
				http.Redirect(w, r, redirectURL, http.StatusFound)
				return nil
			}
		}

		if userReturningFromPkgGoDev(r) {
			return f(w, r)
		}
		redirectParam := r.FormValue(pkgGoDevRedirectParam)
		if redirectParam == pkgGoDevRedirectOn {
			cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: redirectParam, Path: "/"}
			http.SetCookie(w, cookie)
		}
		if redirectParam == pkgGoDevRedirectOff {
			cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: "", MaxAge: -1, Path: "/"}
			http.SetCookie(w, cookie)
		}
		if !shouldRedirectToPkgGoDev(r) {
			return f(w, r)
		}
		http.Redirect(w, r, redirectURL, http.StatusFound)
		return nil
	}
}

var vcsHostsWithThreeElementRepoName = map[string]bool{
	"bitbucket.org": true,
	"gitea.com":     true,
	"gitee.com":     true,
	"github.com":    true,
	"gitlab.com":    true,
	"golang.org":    true,
}

// shouldRedirectURL reports whether a request to the given URL should be
// redirected to pkg.go.dev.
func shouldRedirectURL(r *http.Request, poller *poller.Poller) bool {
	if poller == nil {
		return false
	}
	cfg := poller.Current().(*dynconfig.DynamicConfig)
	return shouldRedirectURLForSnapshot(r, cfg)
}

func shouldRedirectURLForSnapshot(r *http.Request, cfg *dynconfig.DynamicConfig) bool {
	if cfg == nil {
		log.Printf("shouldRedirectURLForSnapshot(%q): cfg is nil", r.URL.Path)
		return false
	}

	// Requests to api.godoc.org and talks.godoc.org are not redirected.
	if strings.HasPrefix(r.URL.Host, "api") || strings.HasPrefix(r.URL.Host, "talks") {
		return false
	}

	_, isSVG := r.URL.Query()["status.svg"]
	_, isPNG := r.URL.Query()["status.png"]
	if isSVG || isPNG {
		return cfg.RedirectBadges
	}
	for _, p := range cfg.RedirectPaths {
		if r.URL.Path == p {
			return true
		} else if !strings.HasPrefix(r.URL.Path, "/-/") && strings.HasPrefix(r.URL.Path, p+"/") {
			return true
		}
	}
	q := strings.TrimSpace(r.Form.Get("q"))
	if r.URL.Path == "/" || r.URL.Path == "" {
		if q == "" && cfg.RedirectHomepage {
			return true
		}
		if q != "" && cfg.RedirectSearch {
			return true
		}
		return false
	}
	if isStdlibURLPath(r.URL.Path) && cfg.RedirectStdlib {
		return true
	}

	if cfg.RedirectRollout >= 100 {
		return true
	}
	if cfg.RedirectRollout == 0 {
		return false
	}
	parts := strings.Split(r.URL.Path, "/")
	prefix := parts[1]
	if _, ok := vcsHostsWithThreeElementRepoName[prefix]; ok {
		prefix = strings.Join(parts[1:3], "/")
	}
	h := fnv.New32a()
	fmt.Fprintf(h, "%s", prefix)
	return uint(h.Sum32()%100) < cfg.RedirectRollout
}

func isStdlibURLPath(path string) bool {
	if strings.HasPrefix(path, "/-/") || strings.HasPrefix(path, "/fonts/") {
		return false
	}
	path = strings.TrimPrefix(path, "/")
	if i := strings.IndexByte(path, '/'); i != -1 {
		path = path[:i]
	}
	return !strings.Contains(path, ".")
}

const goGithubRepoURLPath = "/github.com/golang/go"

func pkgGoDevURL(godocURL *url.URL) *url.URL {
	u := &url.URL{Scheme: "https", Host: pkgGoDevHost}
	q := url.Values{"utm_source": []string{"godoc"}}

	if strings.Contains(godocURL.Path, "/vendor/") || strings.HasSuffix(godocURL.Path, "/vendor") {
		u.Path = "/"
		u.RawQuery = q.Encode()
		return u
	}

	if strings.HasPrefix(godocURL.Path, goGithubRepoURLPath) ||
		strings.HasPrefix(godocURL.Path, goGithubRepoURLPath+"/src") {
		u.Path = strings.TrimPrefix(strings.TrimPrefix(godocURL.Path, goGithubRepoURLPath), "/src")
		if u.Path == "" {
			u.Path = "/std"
		}
		u.RawQuery = q.Encode()
		return u
	}

	_, isSVG := godocURL.Query()["status.svg"]
	_, isPNG := godocURL.Query()["status.png"]
	if isSVG || isPNG {
		u.Path = "/badge" + godocURL.Path
		u.RawQuery = q.Encode()
		return u
	}

	switch godocURL.Path {
	case "/-/go":
		u.Path = "/std"
	case "/-/about":
		u.Path = "/about"
	case "/C":
		u.Path = "/C"
	case "/":
		if qparam := godocURL.Query().Get("q"); qparam != "" {
			u.Path = "/search"
			q.Set("q", qparam)
		} else {
			u.Path = "/"
		}
	case "":
		u.Path = ""
	case "/-/subrepo":
		u.Path = "/search"
		q.Set("q", "golang.org/x")
	default:
		{
			godocURL.Path = strings.TrimSuffix(godocURL.Path, "/")
			// If the import path is invalid, redirect to
			// https://golang.org/issue/43036, so that the users has more context
			// on why this path does not work on pkg.go.dev.
			if err := module.CheckImportPath(strings.TrimPrefix(godocURL.Path, "/")); err != nil {
				u.Host = "golang.org"
				u.Path = "/issue/43036"
				return u
			}

			u.Path = godocURL.Path
			if _, ok := godocURL.Query()["imports"]; ok {
				q.Set("tab", "imports")
			} else if _, ok := godocURL.Query()["importers"]; ok {
				q.Set("tab", "importedby")
			}
		}
	}

	u.RawQuery = q.Encode()
	return u
}