blob: 6c0c5c4b0a8819d99a0ca860a818b22e5a24f8c4 [file] [log] [blame]
// Copyright 2017 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.
// Command gddo-server is the GoPkgDoc server.
package main
import (
"bytes"
"context"
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"go/build"
"html/template"
"io"
"log"
"net/http"
"os"
"path"
"regexp"
"runtime/debug"
"sort"
"strconv"
"strings"
"time"
"cloud.google.com/go/logging"
"cloud.google.com/go/pubsub"
"cloud.google.com/go/trace"
"github.com/spf13/viper"
"github.com/golang/gddo/database"
"github.com/golang/gddo/doc"
"github.com/golang/gddo/gosrc"
"github.com/golang/gddo/httputil"
"github.com/golang/gddo/internal/health"
)
const (
jsonMIMEType = "application/json; charset=utf-8"
textMIMEType = "text/plain; charset=utf-8"
htmlMIMEType = "text/html; charset=utf-8"
)
var errUpdateTimeout = errors.New("refresh timeout")
type httpError struct {
status int // HTTP status code.
err error // Optional reason for the HTTP error.
}
func (err *httpError) Error() string {
if err.err != nil {
return fmt.Sprintf("status %d, reason %s", err.status, err.err.Error())
}
return fmt.Sprintf("Status %d", err.status)
}
const (
humanRequest = iota
robotRequest
queryRequest
refreshRequest
apiRequest
)
type crawlResult struct {
pdoc *doc.Package
err error
}
// getDoc gets the package documentation from the database or from the version
// control system as needed.
func (s *server) getDoc(ctx context.Context, path string, requestType int) (*doc.Package, []database.Package, error) {
if path == "-" {
// A hack in the database package uses the path "-" to represent the
// next document to crawl. Block "-" here so that requests to /- always
// return not found.
return nil, nil, &httpError{status: http.StatusNotFound}
}
pdoc, pkgs, nextCrawl, err := s.db.Get(ctx, path)
if err != nil {
return nil, nil, err
}
needsCrawl := false
switch requestType {
case queryRequest, apiRequest:
needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0
case humanRequest:
needsCrawl = nextCrawl.Before(time.Now())
case robotRequest:
needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0
}
if !needsCrawl {
return pdoc, pkgs, nil
}
c := make(chan crawlResult, 1)
go func() {
pdoc, err := s.crawlDoc(ctx, "web ", path, pdoc, len(pkgs) > 0, nextCrawl)
c <- crawlResult{pdoc, err}
}()
timeout := s.v.GetDuration(ConfigGetTimeout)
if pdoc == nil {
timeout = s.v.GetDuration(ConfigFirstGetTimeout)
}
select {
case cr := <-c:
err = cr.err
if err == nil {
pdoc = cr.pdoc
}
case <-time.After(timeout):
err = errUpdateTimeout
}
switch {
case err == nil:
return pdoc, pkgs, nil
case gosrc.IsNotFound(err):
return nil, nil, err
case pdoc != nil:
log.Printf("Serving %q from database after error getting doc: %v", path, err)
return pdoc, pkgs, nil
case err == errUpdateTimeout:
log.Printf("Serving %q as not found after timeout getting doc", path)
return nil, nil, &httpError{status: http.StatusNotFound}
default:
return nil, nil, err
}
}
func templateExt(req *http.Request) string {
if httputil.NegotiateContentType(req, []string{"text/html", "text/plain"}, "text/html") == "text/plain" {
return ".txt"
}
return ".html"
}
var robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)|(?:^Python-urllib)|(?:^Go )|(?:^Java/)`)
func (s *server) isRobot(req *http.Request) bool {
if robotPat.MatchString(req.Header.Get("User-Agent")) {
return true
}
host := httputil.StripPort(req.RemoteAddr)
n, err := s.db.IncrementCounter(host, 1)
if err != nil {
log.Printf("error incrementing counter for %s, %v", host, err)
return false
}
if n > s.v.GetFloat64(ConfigRobotThreshold) {
log.Printf("robot %.2f %s %s", n, host, req.Header.Get("User-Agent"))
return true
}
return false
}
func popularLinkReferral(req *http.Request) bool {
return strings.HasSuffix(req.Header.Get("Referer"), "//"+req.Host+"/")
}
func isView(req *http.Request, key string) bool {
rq := req.URL.RawQuery
return strings.HasPrefix(rq, key) &&
(len(rq) == len(key) || rq[len(key)] == '=' || rq[len(key)] == '&')
}
// httpEtag returns the package entity tag used in HTTP transactions.
func (s *server) httpEtag(pdoc *doc.Package, pkgs []database.Package, importerCount int, flashMessages []flashMessage) string {
b := make([]byte, 0, 128)
b = strconv.AppendInt(b, pdoc.Updated.Unix(), 16)
b = append(b, 0)
b = append(b, pdoc.Etag...)
if importerCount >= 8 {
importerCount = 8
}
b = append(b, 0)
b = strconv.AppendInt(b, int64(importerCount), 16)
for _, pkg := range pkgs {
b = append(b, 0)
b = append(b, pkg.Path...)
b = append(b, 0)
b = append(b, pkg.Synopsis...)
}
if s.v.GetBool(ConfigSidebar) {
b = append(b, "\000xsb"...)
}
for _, m := range flashMessages {
b = append(b, 0)
b = append(b, m.ID...)
for _, a := range m.Args {
b = append(b, 1)
b = append(b, a...)
}
}
h := md5.New()
h.Write(b)
b = h.Sum(b[:0])
return fmt.Sprintf("\"%x\"", b)
}
func (s *server) servePackage(resp http.ResponseWriter, req *http.Request) error {
p := path.Clean(req.URL.Path)
if strings.HasPrefix(p, "/pkg/") {
p = p[len("/pkg"):]
}
if p != req.URL.Path {
http.Redirect(resp, req, p, http.StatusMovedPermanently)
return nil
}
if isView(req, "status.svg") {
s.statusSVG.ServeHTTP(resp, req)
return nil
}
if isView(req, "status.png") {
s.statusPNG.ServeHTTP(resp, req)
return nil
}
requestType := humanRequest
if s.isRobot(req) {
requestType = robotRequest
}
importPath := strings.TrimPrefix(req.URL.Path, "/")
pdoc, pkgs, err := s.getDoc(req.Context(), importPath, requestType)
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
// To prevent dumb clients from following redirect loops, respond with
// status 404 if the target document is not found.
if _, _, err := s.getDoc(req.Context(), e.Redirect, requestType); gosrc.IsNotFound(err) {
return &httpError{status: http.StatusNotFound}
}
u := "/" + e.Redirect
if req.URL.RawQuery != "" {
u += "?" + req.URL.RawQuery
}
setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
http.Redirect(resp, req, u, http.StatusFound)
return nil
}
if err != nil {
return err
}
flashMessages := getFlashMessages(resp, req)
if pdoc == nil {
if len(pkgs) == 0 {
return &httpError{status: http.StatusNotFound}
}
pdocChild, _, _, err := s.db.Get(req.Context(), pkgs[0].Path)
if err != nil {
return err
}
pdoc = &doc.Package{
ProjectName: pdocChild.ProjectName,
ProjectRoot: pdocChild.ProjectRoot,
ProjectURL: pdocChild.ProjectURL,
ImportPath: importPath,
}
}
showPkgGoDevRedirectToast := userReturningFromPkgGoDev(req)
switch {
case isView(req, "imports"):
if pdoc.Name == "" {
return &httpError{status: http.StatusNotFound}
}
pkgs, err = s.db.Packages(pdoc.Imports)
if err != nil {
return err
}
return s.templates.execute(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{
"flashMessages": flashMessages,
"pkgs": pkgs,
"pdoc": newTDoc(s.v, pdoc),
"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
})
case isView(req, "tools"):
proto := "http"
if req.Host == "godoc.org" {
proto = "https"
}
return s.templates.execute(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{
"flashMessages": flashMessages,
"uri": fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath),
"pdoc": newTDoc(s.v, pdoc),
"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
})
case isView(req, "importers"):
if pdoc.Name == "" {
return &httpError{status: http.StatusNotFound}
}
pkgs, err = s.db.Importers(importPath)
if err != nil {
return err
}
template := "importers.html"
if requestType == robotRequest {
// Hide back links from robots.
template = "importers_robot.html"
}
return s.templates.execute(resp, template, http.StatusOK, nil, map[string]interface{}{
"flashMessages": flashMessages,
"pkgs": pkgs,
"pdoc": newTDoc(s.v, pdoc),
"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
})
case isView(req, "import-graph"):
if requestType == robotRequest {
return &httpError{status: http.StatusForbidden}
}
if pdoc.Name == "" {
return &httpError{status: http.StatusNotFound}
}
// Throttle ?import-graph requests.
select {
case s.importGraphSem <- struct{}{}:
default:
return &httpError{status: http.StatusTooManyRequests}
}
defer func() { <-s.importGraphSem }()
hide := database.ShowAllDeps
switch req.Form.Get("hide") {
case "1":
hide = database.HideStandardDeps
case "2":
hide = database.HideStandardAll
}
pkgs, edges, err := s.db.ImportGraph(pdoc, hide)
if err != nil {
return err
}
b, err := renderGraph(pdoc, pkgs, edges)
if err != nil {
return err
}
return s.templates.execute(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{
"flashMessages": flashMessages,
"svg": template.HTML(b),
"pdoc": newTDoc(s.v, pdoc),
"hide": hide,
"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
})
case isView(req, "play"):
u, err := s.playURL(pdoc, req.Form.Get("play"), req.Header.Get("X-AppEngine-Country"))
if err != nil {
return err
}
http.Redirect(resp, req, u, http.StatusMovedPermanently)
return nil
case req.Form.Get("view") != "":
// Redirect deprecated view= queries.
var q string
switch view := req.Form.Get("view"); view {
case "imports", "importers":
q = view
case "import-graph":
if req.Form.Get("hide") == "1" {
q = "import-graph&hide=1"
} else {
q = "import-graph"
}
}
if q != "" {
u := *req.URL
u.RawQuery = q
http.Redirect(resp, req, u.String(), http.StatusMovedPermanently)
return nil
}
return &httpError{status: http.StatusNotFound}
default:
importerCount := 0
if pdoc.Name != "" {
importerCount, err = s.db.ImporterCount(importPath)
if err != nil {
return err
}
}
etag := s.httpEtag(pdoc, pkgs, importerCount, flashMessages)
status := http.StatusOK
if req.Header.Get("If-None-Match") == etag {
status = http.StatusNotModified
}
if requestType == humanRequest &&
pdoc.Name != "" && // not a directory
pdoc.ProjectRoot != "" && // not a standard package
!pdoc.IsCmd &&
len(pdoc.Errors) == 0 &&
!popularLinkReferral(req) {
if err := s.db.IncrementPopularScore(pdoc.ImportPath); err != nil {
log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err)
}
}
if s.gceLogger != nil {
s.gceLogger.LogEvent(resp, req, nil)
}
template := "dir"
switch {
case pdoc.IsCmd:
template = "cmd"
case pdoc.Name != "":
template = "pkg"
}
template += templateExt(req)
return s.templates.execute(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{
"flashMessages": flashMessages,
"pkgs": pkgs,
"pdoc": newTDoc(s.v, pdoc),
"importerCount": importerCount,
"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
})
}
}
func (s *server) serveRefresh(resp http.ResponseWriter, req *http.Request) error {
importPath := req.Form.Get("path")
_, pkgs, _, err := s.db.Get(req.Context(), importPath)
if err != nil {
return err
}
c := make(chan error, 1)
go func() {
_, err := s.crawlDoc(req.Context(), "rfrsh", importPath, nil, len(pkgs) > 0, time.Time{})
c <- err
}()
select {
case err = <-c:
case <-time.After(s.v.GetDuration(ConfigGetTimeout)):
err = errUpdateTimeout
}
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
importPath = e.Redirect
err = nil
} else if err != nil {
setFlashMessages(resp, []flashMessage{{ID: "refresh", Args: []string{errorText(err)}}})
}
http.Redirect(resp, req, "/"+importPath, http.StatusFound)
return nil
}
func (s *server) serveGoIndex(resp http.ResponseWriter, req *http.Request) error {
pkgs, err := s.db.GoIndex()
if err != nil {
return err
}
return s.templates.execute(resp, "std.html", http.StatusOK, nil, map[string]interface{}{
"pkgs": pkgs,
})
}
func (s *server) serveGoSubrepoIndex(resp http.ResponseWriter, req *http.Request) error {
pkgs, err := s.db.GoSubrepoIndex()
if err != nil {
return err
}
return s.templates.execute(resp, "subrepo.html", http.StatusOK, nil, map[string]interface{}{
"pkgs": pkgs,
})
}
type byPath struct {
pkgs []database.Package
rank []int
}
func (bp *byPath) Len() int { return len(bp.pkgs) }
func (bp *byPath) Less(i, j int) bool { return bp.pkgs[i].Path < bp.pkgs[j].Path }
func (bp *byPath) Swap(i, j int) {
bp.pkgs[i], bp.pkgs[j] = bp.pkgs[j], bp.pkgs[i]
bp.rank[i], bp.rank[j] = bp.rank[j], bp.rank[i]
}
type byRank struct {
pkgs []database.Package
rank []int
}
func (br *byRank) Len() int { return len(br.pkgs) }
func (br *byRank) Less(i, j int) bool { return br.rank[i] < br.rank[j] }
func (br *byRank) Swap(i, j int) {
br.pkgs[i], br.pkgs[j] = br.pkgs[j], br.pkgs[i]
br.rank[i], br.rank[j] = br.rank[j], br.rank[i]
}
func (s *server) popular() ([]database.Package, error) {
const n = 25
pkgs, err := s.db.Popular(2 * n)
if err != nil {
return nil, err
}
rank := make([]int, len(pkgs))
for i := range pkgs {
rank[i] = i
}
sort.Sort(&byPath{pkgs, rank})
j := 0
prev := "."
for i, pkg := range pkgs {
if strings.HasPrefix(pkg.Path, prev) {
if rank[j-1] < rank[i] {
rank[j-1] = rank[i]
}
continue
}
prev = pkg.Path + "/"
pkgs[j] = pkg
rank[j] = rank[i]
j++
}
pkgs = pkgs[:j]
sort.Sort(&byRank{pkgs, rank})
if len(pkgs) > n {
pkgs = pkgs[:n]
}
sort.Sort(&byPath{pkgs, rank})
return pkgs, nil
}
func (s *server) serveHome(resp http.ResponseWriter, req *http.Request) error {
if req.URL.Path != "/" {
return s.servePackage(resp, req)
}
q := strings.TrimSpace(req.Form.Get("q"))
if q == "" {
pkgs, err := s.popular()
if err != nil {
return err
}
return s.templates.execute(resp, "home"+templateExt(req), http.StatusOK, nil,
map[string]interface{}{
"Popular": pkgs,
"showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
})
}
if path, ok := isBrowseURL(q); ok {
q = path
}
if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
pdoc, pkgs, err := s.getDoc(req.Context(), q, queryRequest)
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
return nil
}
if err == nil && (pdoc != nil || len(pkgs) > 0) {
http.Redirect(resp, req, "/"+q, http.StatusFound)
return nil
}
}
pkgs, err := s.db.Search(req.Context(), q)
if err != nil {
return err
}
if s.gceLogger != nil {
// Log up to top 10 packages we served upon a search.
logPkgs := pkgs
if len(pkgs) > 10 {
logPkgs = pkgs[:10]
}
s.gceLogger.LogEvent(resp, req, logPkgs)
}
return s.templates.execute(resp, "results"+templateExt(req), http.StatusOK, nil,
map[string]interface{}{
"q": q,
"pkgs": pkgs,
"showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
})
}
func (s *server) serveAbout(resp http.ResponseWriter, req *http.Request) error {
return s.templates.execute(resp, "about.html", http.StatusOK, nil,
map[string]interface{}{
"Host": req.Host,
"showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
})
}
func (s *server) serveBot(resp http.ResponseWriter, req *http.Request) error {
return s.templates.execute(resp, "bot.html", http.StatusOK, nil, nil)
}
func logError(req *http.Request, err error, rv interface{}) {
if err != nil {
var buf bytes.Buffer
fmt.Fprintf(&buf, "Error serving %s: %v\n", req.URL, err)
if rv != nil {
fmt.Fprintln(&buf, rv)
buf.Write(debug.Stack())
}
log.Print(buf.String())
}
}
func (s *server) serveAPISearch(resp http.ResponseWriter, req *http.Request) error {
q := strings.TrimSpace(req.Form.Get("q"))
var pkgs []database.Package
if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
pdoc, _, err := s.getDoc(req.Context(), q, apiRequest)
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
pdoc, _, err = s.getDoc(req.Context(), e.Redirect, robotRequest)
}
if err == nil && pdoc != nil {
pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}}
}
}
if pkgs == nil {
var err error
pkgs, err = s.db.Search(req.Context(), q)
if err != nil {
return err
}
}
var data = struct {
Results []database.Package `json:"results"`
}{
pkgs,
}
resp.Header().Set("Content-Type", jsonMIMEType)
return json.NewEncoder(resp).Encode(&data)
}
func (s *server) serveAPIPackages(resp http.ResponseWriter, req *http.Request) error {
pkgs, err := s.db.AllPackages()
if err != nil {
return err
}
data := struct {
Results []database.Package `json:"results"`
}{
pkgs,
}
resp.Header().Set("Content-Type", jsonMIMEType)
return json.NewEncoder(resp).Encode(&data)
}
func (s *server) serveAPIImporters(resp http.ResponseWriter, req *http.Request) error {
importPath := strings.TrimPrefix(req.URL.Path, "/importers/")
pkgs, err := s.db.Importers(importPath)
if err != nil {
return err
}
data := struct {
Results []database.Package `json:"results"`
}{
pkgs,
}
resp.Header().Set("Content-Type", jsonMIMEType)
return json.NewEncoder(resp).Encode(&data)
}
func (s *server) serveAPIImports(resp http.ResponseWriter, req *http.Request) error {
importPath := strings.TrimPrefix(req.URL.Path, "/imports/")
pdoc, _, err := s.getDoc(req.Context(), importPath, robotRequest)
if err != nil {
return err
}
if pdoc == nil || pdoc.Name == "" {
return &httpError{status: http.StatusNotFound}
}
imports, err := s.db.Packages(pdoc.Imports)
if err != nil {
return err
}
testImports, err := s.db.Packages(pdoc.TestImports)
if err != nil {
return err
}
data := struct {
Imports []database.Package `json:"imports"`
TestImports []database.Package `json:"testImports"`
}{
imports,
testImports,
}
resp.Header().Set("Content-Type", jsonMIMEType)
return json.NewEncoder(resp).Encode(&data)
}
func serveAPIHome(resp http.ResponseWriter, req *http.Request) error {
return &httpError{status: http.StatusNotFound}
}
type requestCleaner struct {
h http.Handler
trustProxyHeaders bool
}
func (rc requestCleaner) ServeHTTP(w http.ResponseWriter, req *http.Request) {
req2 := new(http.Request)
*req2 = *req
if rc.trustProxyHeaders {
if s := req.Header.Get("X-Forwarded-For"); s != "" {
req2.RemoteAddr = s
}
}
req2.Body = http.MaxBytesReader(w, req.Body, 2048)
req2.ParseForm()
rc.h.ServeHTTP(w, req2)
}
type errorHandler struct {
fn func(resp http.ResponseWriter, req *http.Request) error
errFn httputil.Error
}
func (eh errorHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
defer func() {
if rv := recover(); rv != nil {
err := errors.New("handler panic")
logError(req, err, rv)
eh.errFn(resp, req, http.StatusInternalServerError, err)
}
}()
rb := new(httputil.ResponseBuffer)
err := eh.fn(rb, req)
if err == nil {
rb.WriteTo(resp)
} else if e, ok := err.(*httpError); ok {
if e.status >= 500 {
logError(req, err, nil)
}
eh.errFn(resp, req, e.status, e.err)
} else if gosrc.IsNotFound(err) {
eh.errFn(resp, req, http.StatusNotFound, nil)
} else {
logError(req, err, nil)
eh.errFn(resp, req, http.StatusInternalServerError, err)
}
}
func errorText(err error) string {
if err == errUpdateTimeout {
return "Timeout getting package files from the version control system."
}
if e, ok := err.(*gosrc.RemoteError); ok {
return "Error getting package files from " + e.Host + "."
}
return "Internal server error."
}
func (s *server) handleError(resp http.ResponseWriter, req *http.Request, status int, err error) {
switch status {
case http.StatusNotFound:
s.templates.execute(resp, "notfound"+templateExt(req), status, nil, map[string]interface{}{
"flashMessages": getFlashMessages(resp, req),
})
default:
resp.Header().Set("Content-Type", textMIMEType)
resp.WriteHeader(http.StatusInternalServerError)
io.WriteString(resp, errorText(err))
}
}
func handleAPIError(resp http.ResponseWriter, req *http.Request, status int, err error) {
var data struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
data.Error.Message = http.StatusText(status)
resp.Header().Set("Content-Type", jsonMIMEType)
resp.WriteHeader(status)
json.NewEncoder(resp).Encode(&data)
}
// httpsRedirectHandler redirects all requests with an X-Forwarded-Proto: http
// handler to their https equivalent.
type httpsRedirectHandler struct {
h http.Handler
}
func (h httpsRedirectHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if req.Header.Get("X-Forwarded-Proto") == "http" {
u := *req.URL
u.Scheme = "https"
u.Host = req.Host
http.Redirect(resp, req, u.String(), http.StatusFound)
return
}
h.h.ServeHTTP(resp, req)
}
type rootHandler []struct {
prefix string
h http.Handler
}
func (m rootHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
var h http.Handler
for _, ph := range m {
if strings.HasPrefix(req.Host, ph.prefix) {
h = ph.h
break
}
}
h.ServeHTTP(resp, req)
}
// otherDomainHandler redirects to another domain keeping the rest of the URL.
type otherDomainHandler struct {
scheme string
targetDomain string
}
func (h otherDomainHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
u := *req.URL
u.Scheme = h.scheme
u.Host = h.targetDomain
http.Redirect(w, req, u.String(), http.StatusFound)
}
func defaultBase(path string) string {
p, err := build.Default.Import(path, "", build.FindOnly)
if err != nil {
return "."
}
return p.Dir
}
type server struct {
v *viper.Viper
db *database.Database
httpClient *http.Client
gceLogger *GCELogger
templates templateMap
traceClient *trace.Client
crawlTopic *pubsub.Topic
statusPNG http.Handler
statusSVG http.Handler
root rootHandler
// A semaphore to limit concurrent ?import-graph requests.
importGraphSem chan struct{}
}
func newServer(ctx context.Context, v *viper.Viper) (*server, error) {
s := &server{
v: v,
httpClient: newHTTPClient(v),
importGraphSem: make(chan struct{}, 10),
}
var err error
if proj := s.v.GetString(ConfigProject); proj != "" {
if s.traceClient, err = trace.NewClient(ctx, proj); err != nil {
return nil, err
}
sp, err := trace.NewLimitedSampler(s.v.GetFloat64(ConfigTraceSamplerFraction), s.v.GetFloat64(ConfigTraceSamplerMaxQPS))
if err != nil {
return nil, err
}
s.traceClient.SetSamplingPolicy(sp)
// This topic should be created in the cloud console.
ps, err := pubsub.NewClient(ctx, proj)
if err != nil {
return nil, err
}
s.crawlTopic = ps.Topic(ConfigCrawlPubSubTopic)
}
assets := v.GetString(ConfigAssetsDir)
staticServer := httputil.StaticServer{
Dir: assets,
MaxAge: time.Hour,
MIMETypes: map[string]string{
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
},
}
s.statusPNG = staticServer.FileHandler("status.png")
s.statusSVG = staticServer.FileHandler("status.svg")
apiHandler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
return requestCleaner{
h: errorHandler{
fn: f,
errFn: handleAPIError,
},
trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
}
}
apiMux := http.NewServeMux()
apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
apiMux.Handle("/search", apiHandler(s.serveAPISearch))
apiMux.Handle("/packages", apiHandler(s.serveAPIPackages))
apiMux.Handle("/importers/", apiHandler(s.serveAPIImporters))
apiMux.Handle("/imports/", apiHandler(s.serveAPIImports))
apiMux.Handle("/", apiHandler(serveAPIHome))
mux := http.NewServeMux()
mux.Handle("/-/site.js", staticServer.FilesHandler(
"third_party/jquery.timeago.js",
"site.js"))
mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
if s.v.GetBool(ConfigSidebar) {
mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
}
mux.Handle("/-/", http.NotFoundHandler())
handler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
return requestCleaner{
h: errorHandler{
fn: f,
errFn: s.handleError,
},
trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
}
}
mux.Handle("/-/about", handler(pkgGoDevRedirectHandler(s.serveAbout)))
mux.Handle("/-/bot", handler(s.serveBot))
mux.Handle("/-/go", handler(pkgGoDevRedirectHandler(s.serveGoIndex)))
mux.Handle("/-/subrepo", handler(s.serveGoSubrepoIndex))
mux.Handle("/-/refresh", handler(s.serveRefresh))
mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
mux.Handle("/code.jquery.com/", http.NotFoundHandler())
mux.Handle("/", handler(pkgGoDevRedirectHandler(s.serveHome)))
ahMux := http.NewServeMux()
ready := new(health.Handler)
ahMux.HandleFunc("/_ah/health", health.HandleLive)
ahMux.Handle("/_ah/ready", ready)
mainMux := http.NewServeMux()
mainMux.Handle("/_ah/", ahMux)
mainMux.Handle("/", s.traceClient.HTTPHandler(mux))
s.root = rootHandler{
{"api.", httpsRedirectHandler{s.traceClient.HTTPHandler(apiMux)}},
{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
{"", httpsRedirectHandler{mainMux}},
}
cacheBusters := &httputil.CacheBusters{Handler: mux}
s.templates, err = parseTemplates(assets, cacheBusters, v)
if err != nil {
return nil, err
}
s.db, err = database.New(
v.GetString(ConfigDBServer),
v.GetDuration(ConfigDBIdleTimeout),
v.GetBool(ConfigDBLog),
v.GetString(ConfigGAERemoteAPI),
)
if err != nil {
return nil, fmt.Errorf("open database: %v", err)
}
ready.Add(s.db)
if gceLogName := v.GetString(ConfigGCELogName); gceLogName != "" {
logc, err := logging.NewClient(ctx, v.GetString(ConfigProject))
if err != nil {
return nil, fmt.Errorf("create cloud logging client: %v", err)
}
logger := logc.Logger(gceLogName)
if err := logc.Ping(ctx); err != nil {
return nil, fmt.Errorf("pinging cloud logging: %v", err)
}
s.gceLogger = newGCELogger(logger)
}
return s, nil
}
type responseWriter struct {
http.ResponseWriter
status int
}
func (rw *responseWriter) WriteHeader(code int) {
rw.status = code
rw.ResponseWriter.WriteHeader(code)
}
func translateStatus(code int) int {
if code == 0 {
return http.StatusOK
}
return code
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
start := time.Now()
s.logRequestStart(r)
w2 := &responseWriter{ResponseWriter: w}
s.root.ServeHTTP(w2, r)
latency := time.Since(start)
s.logRequestEnd(r, latency)
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
s.teeRequestToPkgGoDev(r, latency, translateStatus(w2.status))
}
func (s *server) logRequestStart(req *http.Request) {
if s.gceLogger == nil {
return
}
s.gceLogger.Log(logging.Entry{
HTTPRequest: &logging.HTTPRequest{Request: req},
Payload: fmt.Sprintf("%s request start", req.Host),
Severity: logging.Info,
})
}
func (s *server) logRequestEnd(req *http.Request, latency time.Duration) {
if s.gceLogger == nil {
return
}
s.gceLogger.Log(logging.Entry{
HTTPRequest: &logging.HTTPRequest{
Request: req,
Latency: latency,
},
Payload: fmt.Sprintf("%s request end", req.Host),
Severity: logging.Info,
})
}
func (s *server) teeRequestToPkgGoDev(r *http.Request, latency time.Duration, status int) {
if !shouldTeeRequest(r.URL.Path) {
log.Printf("teeRequestToPkgGoDev(%q): not teeing request", r.URL.Path)
return
}
val := os.Getenv("GDDO_TEE_REQUESTS_TO_PKGGODEV")
if strings.ToLower(val) != "true" {
log.Printf("not sending requests to pkg.go.dev (GDDO_TEE_REQUESTS_TO_PKGGODEV = %q)", val)
return
}
gddoEvent, pkggodevEvent := teeRequestToPkgGoDev(r, latency, s.isRobot(r), status)
payload := map[string]interface{}{
"godoc.org": gddoEvent,
"pkg.go.dev": pkggodevEvent,
}
if s.gceLogger == nil {
for k, v := range payload {
log.Printf("%q", k)
log.Printf("%+v", v)
}
return
}
s.gceLogger.Log(logging.Entry{
HTTPRequest: &logging.HTTPRequest{
Request: r,
Latency: latency,
Status: status,
},
Payload: payload,
Severity: logging.Info,
})
return
}
func main() {
ctx := context.Background()
v, err := loadConfig(ctx, os.Args)
if err != nil {
log.Fatal(ctx, "load config", "error", err.Error())
}
doc.SetDefaultGOOS(v.GetString(ConfigDefaultGOOS))
s, err := newServer(ctx, v)
if err != nil {
log.Fatal("error creating server:", err)
}
go func() {
for range time.Tick(s.v.GetDuration(ConfigCrawlInterval)) {
if err := s.doCrawl(ctx); err != nil {
log.Printf("Task Crawl: %v", err)
}
}
}()
go func() {
for range time.Tick(s.v.GetDuration(ConfigGithubInterval)) {
if err := s.readGitHubUpdates(ctx); err != nil {
log.Printf("Task GitHub updates: %v", err)
}
}
}()
http.Handle("/", s)
log.Fatal(http.ListenAndServe(s.v.GetString(ConfigBindAddress), s))
}