gddo-server: split main.go into two files
Pure code in motion.
Change-Id: Idd7e08bac2a1fdb9e2c79ca9a126becf30ea4235
Reviewed-on: https://go-review.googlesource.com/c/gddo/+/285835
Trust: Julie Qiu <julie@golang.org>
Run-TryBot: Julie Qiu <julie@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/gddo-server/main.go b/gddo-server/main.go
index 57eac28..2fc0a69 100644
--- a/gddo-server/main.go
+++ b/gddo-server/main.go
@@ -8,1115 +8,15 @@
package main
import (
- "bytes"
"context"
- "crypto/md5"
- "encoding/json"
- "errors"
- "fmt"
- "go/build"
- "html/template"
- "io"
"log"
"net/http"
"os"
- "path"
- "regexp"
- "runtime/debug"
- "sort"
- "strconv"
- "strings"
"time"
- "cloud.google.com/go/logging"
- "cloud.google.com/go/pubsub"
- "cloud.google.com/go/trace"
- "github.com/spf13/viper"
-
- "github.com/golang/gddo/database"
"github.com/golang/gddo/doc"
- "github.com/golang/gddo/gosrc"
- "github.com/golang/gddo/httputil"
- "github.com/golang/gddo/internal/health"
)
-const (
- jsonMIMEType = "application/json; charset=utf-8"
- textMIMEType = "text/plain; charset=utf-8"
- htmlMIMEType = "text/html; charset=utf-8"
-)
-
-var errUpdateTimeout = errors.New("refresh timeout")
-
-type httpError struct {
- status int // HTTP status code.
- err error // Optional reason for the HTTP error.
-}
-
-func (err *httpError) Error() string {
- if err.err != nil {
- return fmt.Sprintf("status %d, reason %s", err.status, err.err.Error())
- }
- return fmt.Sprintf("Status %d", err.status)
-}
-
-const (
- humanRequest = iota
- robotRequest
- queryRequest
- refreshRequest
- apiRequest
-)
-
-type crawlResult struct {
- pdoc *doc.Package
- err error
-}
-
-// getDoc gets the package documentation from the database or from the version
-// control system as needed.
-func (s *server) getDoc(ctx context.Context, path string, requestType int) (*doc.Package, []database.Package, error) {
- if path == "-" {
- // A hack in the database package uses the path "-" to represent the
- // next document to crawl. Block "-" here so that requests to /- always
- // return not found.
- return nil, nil, &httpError{status: http.StatusNotFound}
- }
-
- pdoc, pkgs, nextCrawl, err := s.db.Get(ctx, path)
- if err != nil {
- return nil, nil, err
- }
-
- needsCrawl := false
- switch requestType {
- case queryRequest, apiRequest:
- needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0
- case humanRequest:
- needsCrawl = nextCrawl.Before(time.Now())
- case robotRequest:
- needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0
- }
-
- if !needsCrawl {
- return pdoc, pkgs, nil
- }
-
- c := make(chan crawlResult, 1)
- go func() {
- pdoc, err := s.crawlDoc(ctx, "web ", path, pdoc, len(pkgs) > 0, nextCrawl)
- c <- crawlResult{pdoc, err}
- }()
-
- timeout := s.v.GetDuration(ConfigGetTimeout)
- if pdoc == nil {
- timeout = s.v.GetDuration(ConfigFirstGetTimeout)
- }
-
- select {
- case cr := <-c:
- err = cr.err
- if err == nil {
- pdoc = cr.pdoc
- }
- case <-time.After(timeout):
- err = errUpdateTimeout
- }
-
- switch {
- case err == nil:
- return pdoc, pkgs, nil
- case gosrc.IsNotFound(err):
- return nil, nil, err
- case pdoc != nil:
- log.Printf("Serving %q from database after error getting doc: %v", path, err)
- return pdoc, pkgs, nil
- case err == errUpdateTimeout:
- log.Printf("Serving %q as not found after timeout getting doc", path)
- return nil, nil, &httpError{status: http.StatusNotFound}
- default:
- return nil, nil, err
- }
-}
-
-func templateExt(req *http.Request) string {
- if httputil.NegotiateContentType(req, []string{"text/html", "text/plain"}, "text/html") == "text/plain" {
- return ".txt"
- }
- return ".html"
-}
-
-var robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)|(?:^Python-urllib)|(?:^Go )|(?:^Java/)`)
-
-func (s *server) isRobot(req *http.Request) bool {
- if robotPat.MatchString(req.Header.Get("User-Agent")) {
- return true
- }
- host := httputil.StripPort(req.RemoteAddr)
- n, err := s.db.IncrementCounter(host, 1)
- if err != nil {
- log.Printf("error incrementing counter for %s, %v", host, err)
- return false
- }
- if n > s.v.GetFloat64(ConfigRobotThreshold) {
- log.Printf("robot %.2f %s %s", n, host, req.Header.Get("User-Agent"))
- return true
- }
- return false
-}
-
-func popularLinkReferral(req *http.Request) bool {
- return strings.HasSuffix(req.Header.Get("Referer"), "//"+req.Host+"/")
-}
-
-func isView(req *http.Request, key string) bool {
- rq := req.URL.RawQuery
- return strings.HasPrefix(rq, key) &&
- (len(rq) == len(key) || rq[len(key)] == '=' || rq[len(key)] == '&')
-}
-
-// httpEtag returns the package entity tag used in HTTP transactions.
-func (s *server) httpEtag(pdoc *doc.Package, pkgs []database.Package, importerCount int, flashMessages []flashMessage) string {
- b := make([]byte, 0, 128)
- b = strconv.AppendInt(b, pdoc.Updated.Unix(), 16)
- b = append(b, 0)
- b = append(b, pdoc.Etag...)
- if importerCount >= 8 {
- importerCount = 8
- }
- b = append(b, 0)
- b = strconv.AppendInt(b, int64(importerCount), 16)
- for _, pkg := range pkgs {
- b = append(b, 0)
- b = append(b, pkg.Path...)
- b = append(b, 0)
- b = append(b, pkg.Synopsis...)
- }
- if s.v.GetBool(ConfigSidebar) {
- b = append(b, "\000xsb"...)
- }
- for _, m := range flashMessages {
- b = append(b, 0)
- b = append(b, m.ID...)
- for _, a := range m.Args {
- b = append(b, 1)
- b = append(b, a...)
- }
- }
- h := md5.New()
- h.Write(b)
- b = h.Sum(b[:0])
- return fmt.Sprintf("\"%x\"", b)
-}
-
-func (s *server) servePackage(resp http.ResponseWriter, req *http.Request) error {
- p := path.Clean(req.URL.Path)
- if strings.HasPrefix(p, "/pkg/") {
- p = p[len("/pkg"):]
- }
- if p != req.URL.Path {
- http.Redirect(resp, req, p, http.StatusMovedPermanently)
- return nil
- }
-
- if isView(req, "status.svg") {
- s.statusSVG.ServeHTTP(resp, req)
- return nil
- }
-
- if isView(req, "status.png") {
- s.statusPNG.ServeHTTP(resp, req)
- return nil
- }
-
- requestType := humanRequest
- if s.isRobot(req) {
- requestType = robotRequest
- }
-
- importPath := strings.TrimPrefix(req.URL.Path, "/")
- pdoc, pkgs, err := s.getDoc(req.Context(), importPath, requestType)
-
- if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
- // To prevent dumb clients from following redirect loops, respond with
- // status 404 if the target document is not found.
- if _, _, err := s.getDoc(req.Context(), e.Redirect, requestType); gosrc.IsNotFound(err) {
- return &httpError{status: http.StatusNotFound}
- }
- u := "/" + e.Redirect
- if req.URL.RawQuery != "" {
- u += "?" + req.URL.RawQuery
- }
- setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
- http.Redirect(resp, req, u, http.StatusFound)
- return nil
- }
- if err != nil {
- return err
- }
-
- flashMessages := getFlashMessages(resp, req)
-
- if pdoc == nil {
- if len(pkgs) == 0 {
- return &httpError{status: http.StatusNotFound}
- }
- pdocChild, _, _, err := s.db.Get(req.Context(), pkgs[0].Path)
- if err != nil {
- return err
- }
- pdoc = &doc.Package{
- ProjectName: pdocChild.ProjectName,
- ProjectRoot: pdocChild.ProjectRoot,
- ProjectURL: pdocChild.ProjectURL,
- ImportPath: importPath,
- }
- }
-
- showPkgGoDevRedirectToast := userReturningFromPkgGoDev(req)
-
- switch {
- case isView(req, "imports"):
- if pdoc.Name == "" {
- return &httpError{status: http.StatusNotFound}
- }
- pkgs, err = s.db.Packages(pdoc.Imports)
- if err != nil {
- return err
- }
- return s.templates.execute(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{
- "flashMessages": flashMessages,
- "pkgs": pkgs,
- "pdoc": newTDoc(s.v, pdoc),
- "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
- })
- case isView(req, "tools"):
- proto := "http"
- if req.Host == "godoc.org" {
- proto = "https"
- }
- return s.templates.execute(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{
- "flashMessages": flashMessages,
- "uri": fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath),
- "pdoc": newTDoc(s.v, pdoc),
- "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
- })
- case isView(req, "importers"):
- if pdoc.Name == "" {
- return &httpError{status: http.StatusNotFound}
- }
- pkgs, err = s.db.Importers(importPath)
- if err != nil {
- return err
- }
- template := "importers.html"
- if requestType == robotRequest {
- // Hide back links from robots.
- template = "importers_robot.html"
- }
- return s.templates.execute(resp, template, http.StatusOK, nil, map[string]interface{}{
- "flashMessages": flashMessages,
- "pkgs": pkgs,
- "pdoc": newTDoc(s.v, pdoc),
- "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
- })
- case isView(req, "import-graph"):
- if requestType == robotRequest {
- return &httpError{status: http.StatusForbidden}
- }
- if pdoc.Name == "" {
- return &httpError{status: http.StatusNotFound}
- }
-
- // Throttle ?import-graph requests.
- select {
- case s.importGraphSem <- struct{}{}:
- default:
- return &httpError{status: http.StatusTooManyRequests}
- }
- defer func() { <-s.importGraphSem }()
-
- hide := database.ShowAllDeps
- switch req.Form.Get("hide") {
- case "1":
- hide = database.HideStandardDeps
- case "2":
- hide = database.HideStandardAll
- }
- pkgs, edges, err := s.db.ImportGraph(pdoc, hide)
- if err != nil {
- return err
- }
- b, err := renderGraph(pdoc, pkgs, edges)
- if err != nil {
- return err
- }
- return s.templates.execute(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{
- "flashMessages": flashMessages,
- "svg": template.HTML(b),
- "pdoc": newTDoc(s.v, pdoc),
- "hide": hide,
- "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
- })
- case isView(req, "play"):
- u, err := s.playURL(pdoc, req.Form.Get("play"), req.Header.Get("X-AppEngine-Country"))
- if err != nil {
- return err
- }
- http.Redirect(resp, req, u, http.StatusMovedPermanently)
- return nil
- case req.Form.Get("view") != "":
- // Redirect deprecated view= queries.
- var q string
- switch view := req.Form.Get("view"); view {
- case "imports", "importers":
- q = view
- case "import-graph":
- if req.Form.Get("hide") == "1" {
- q = "import-graph&hide=1"
- } else {
- q = "import-graph"
- }
- }
- if q != "" {
- u := *req.URL
- u.RawQuery = q
- http.Redirect(resp, req, u.String(), http.StatusMovedPermanently)
- return nil
- }
- return &httpError{status: http.StatusNotFound}
- default:
- importerCount := 0
- if pdoc.Name != "" {
- importerCount, err = s.db.ImporterCount(importPath)
- if err != nil {
- return err
- }
- }
-
- etag := s.httpEtag(pdoc, pkgs, importerCount, flashMessages)
- status := http.StatusOK
- if req.Header.Get("If-None-Match") == etag {
- status = http.StatusNotModified
- }
-
- if requestType == humanRequest &&
- pdoc.Name != "" && // not a directory
- pdoc.ProjectRoot != "" && // not a standard package
- !pdoc.IsCmd &&
- len(pdoc.Errors) == 0 &&
- !popularLinkReferral(req) {
- if err := s.db.IncrementPopularScore(pdoc.ImportPath); err != nil {
- log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err)
- }
- }
- if s.gceLogger != nil {
- s.gceLogger.LogEvent(resp, req, nil)
- }
-
- template := "dir"
- switch {
- case pdoc.IsCmd:
- template = "cmd"
- case pdoc.Name != "":
- template = "pkg"
- }
- template += templateExt(req)
-
- return s.templates.execute(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{
- "flashMessages": flashMessages,
- "pkgs": pkgs,
- "pdoc": newTDoc(s.v, pdoc),
- "importerCount": importerCount,
- "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
- })
- }
-}
-
-func (s *server) serveRefresh(resp http.ResponseWriter, req *http.Request) error {
- importPath := req.Form.Get("path")
- _, pkgs, _, err := s.db.Get(req.Context(), importPath)
- if err != nil {
- return err
- }
- c := make(chan error, 1)
- go func() {
- _, err := s.crawlDoc(req.Context(), "rfrsh", importPath, nil, len(pkgs) > 0, time.Time{})
- c <- err
- }()
- select {
- case err = <-c:
- case <-time.After(s.v.GetDuration(ConfigGetTimeout)):
- err = errUpdateTimeout
- }
- if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
- setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
- importPath = e.Redirect
- err = nil
- } else if err != nil {
- setFlashMessages(resp, []flashMessage{{ID: "refresh", Args: []string{errorText(err)}}})
- }
- http.Redirect(resp, req, "/"+importPath, http.StatusFound)
- return nil
-}
-
-func (s *server) serveGoIndex(resp http.ResponseWriter, req *http.Request) error {
- pkgs, err := s.db.GoIndex()
- if err != nil {
- return err
- }
- return s.templates.execute(resp, "std.html", http.StatusOK, nil, map[string]interface{}{
- "pkgs": pkgs,
- })
-}
-
-func (s *server) serveGoSubrepoIndex(resp http.ResponseWriter, req *http.Request) error {
- pkgs, err := s.db.GoSubrepoIndex()
- if err != nil {
- return err
- }
- return s.templates.execute(resp, "subrepo.html", http.StatusOK, nil, map[string]interface{}{
- "pkgs": pkgs,
- })
-}
-
-type byPath struct {
- pkgs []database.Package
- rank []int
-}
-
-func (bp *byPath) Len() int { return len(bp.pkgs) }
-func (bp *byPath) Less(i, j int) bool { return bp.pkgs[i].Path < bp.pkgs[j].Path }
-func (bp *byPath) Swap(i, j int) {
- bp.pkgs[i], bp.pkgs[j] = bp.pkgs[j], bp.pkgs[i]
- bp.rank[i], bp.rank[j] = bp.rank[j], bp.rank[i]
-}
-
-type byRank struct {
- pkgs []database.Package
- rank []int
-}
-
-func (br *byRank) Len() int { return len(br.pkgs) }
-func (br *byRank) Less(i, j int) bool { return br.rank[i] < br.rank[j] }
-func (br *byRank) Swap(i, j int) {
- br.pkgs[i], br.pkgs[j] = br.pkgs[j], br.pkgs[i]
- br.rank[i], br.rank[j] = br.rank[j], br.rank[i]
-}
-
-func (s *server) popular() ([]database.Package, error) {
- const n = 25
-
- pkgs, err := s.db.Popular(2 * n)
- if err != nil {
- return nil, err
- }
-
- rank := make([]int, len(pkgs))
- for i := range pkgs {
- rank[i] = i
- }
-
- sort.Sort(&byPath{pkgs, rank})
-
- j := 0
- prev := "."
- for i, pkg := range pkgs {
- if strings.HasPrefix(pkg.Path, prev) {
- if rank[j-1] < rank[i] {
- rank[j-1] = rank[i]
- }
- continue
- }
- prev = pkg.Path + "/"
- pkgs[j] = pkg
- rank[j] = rank[i]
- j++
- }
- pkgs = pkgs[:j]
-
- sort.Sort(&byRank{pkgs, rank})
-
- if len(pkgs) > n {
- pkgs = pkgs[:n]
- }
-
- sort.Sort(&byPath{pkgs, rank})
-
- return pkgs, nil
-}
-
-func (s *server) serveHome(resp http.ResponseWriter, req *http.Request) error {
- if req.URL.Path != "/" {
- return s.servePackage(resp, req)
- }
-
- q := strings.TrimSpace(req.Form.Get("q"))
- if q == "" {
- pkgs, err := s.popular()
- if err != nil {
- return err
- }
-
- return s.templates.execute(resp, "home"+templateExt(req), http.StatusOK, nil,
- map[string]interface{}{
- "Popular": pkgs,
-
- "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
- })
- }
-
- if path, ok := isBrowseURL(q); ok {
- q = path
- }
-
- if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
- pdoc, pkgs, err := s.getDoc(req.Context(), q, queryRequest)
- if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
- http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
- return nil
- }
- if err == nil && (pdoc != nil || len(pkgs) > 0) {
- http.Redirect(resp, req, "/"+q, http.StatusFound)
- return nil
- }
- }
-
- pkgs, err := s.db.Search(req.Context(), q)
- if err != nil {
- return err
- }
- if s.gceLogger != nil {
- // Log up to top 10 packages we served upon a search.
- logPkgs := pkgs
- if len(pkgs) > 10 {
- logPkgs = pkgs[:10]
- }
- s.gceLogger.LogEvent(resp, req, logPkgs)
- }
-
- return s.templates.execute(resp, "results"+templateExt(req), http.StatusOK, nil,
- map[string]interface{}{
- "q": q,
- "pkgs": pkgs,
-
- "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
- })
-}
-
-func (s *server) serveAbout(resp http.ResponseWriter, req *http.Request) error {
- return s.templates.execute(resp, "about.html", http.StatusOK, nil,
- map[string]interface{}{
- "Host": req.Host,
-
- "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
- })
-}
-
-func (s *server) serveBot(resp http.ResponseWriter, req *http.Request) error {
- return s.templates.execute(resp, "bot.html", http.StatusOK, nil, nil)
-}
-
-func logError(req *http.Request, err error, rv interface{}) {
- if err != nil {
- var buf bytes.Buffer
- fmt.Fprintf(&buf, "Error serving %s: %v\n", req.URL, err)
- if rv != nil {
- fmt.Fprintln(&buf, rv)
- buf.Write(debug.Stack())
- }
- log.Print(buf.String())
- }
-}
-
-func (s *server) serveAPISearch(resp http.ResponseWriter, req *http.Request) error {
- q := strings.TrimSpace(req.Form.Get("q"))
-
- var pkgs []database.Package
-
- if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
- pdoc, _, err := s.getDoc(req.Context(), q, apiRequest)
- if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
- pdoc, _, err = s.getDoc(req.Context(), e.Redirect, robotRequest)
- }
- if err == nil && pdoc != nil {
- pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}}
- }
- }
-
- if pkgs == nil {
- var err error
- pkgs, err = s.db.Search(req.Context(), q)
- if err != nil {
- return err
- }
- }
-
- var data = struct {
- Results []database.Package `json:"results"`
- }{
- pkgs,
- }
- resp.Header().Set("Content-Type", jsonMIMEType)
- return json.NewEncoder(resp).Encode(&data)
-}
-
-func (s *server) serveAPIPackages(resp http.ResponseWriter, req *http.Request) error {
- pkgs, err := s.db.AllPackages()
- if err != nil {
- return err
- }
- data := struct {
- Results []database.Package `json:"results"`
- }{
- pkgs,
- }
- resp.Header().Set("Content-Type", jsonMIMEType)
- return json.NewEncoder(resp).Encode(&data)
-}
-
-func (s *server) serveAPIImporters(resp http.ResponseWriter, req *http.Request) error {
- importPath := strings.TrimPrefix(req.URL.Path, "/importers/")
- pkgs, err := s.db.Importers(importPath)
- if err != nil {
- return err
- }
- data := struct {
- Results []database.Package `json:"results"`
- }{
- pkgs,
- }
- resp.Header().Set("Content-Type", jsonMIMEType)
- return json.NewEncoder(resp).Encode(&data)
-}
-
-func (s *server) serveAPIImports(resp http.ResponseWriter, req *http.Request) error {
- importPath := strings.TrimPrefix(req.URL.Path, "/imports/")
- pdoc, _, err := s.getDoc(req.Context(), importPath, robotRequest)
- if err != nil {
- return err
- }
- if pdoc == nil || pdoc.Name == "" {
- return &httpError{status: http.StatusNotFound}
- }
- imports, err := s.db.Packages(pdoc.Imports)
- if err != nil {
- return err
- }
- testImports, err := s.db.Packages(pdoc.TestImports)
- if err != nil {
- return err
- }
- data := struct {
- Imports []database.Package `json:"imports"`
- TestImports []database.Package `json:"testImports"`
- }{
- imports,
- testImports,
- }
- resp.Header().Set("Content-Type", jsonMIMEType)
- return json.NewEncoder(resp).Encode(&data)
-}
-
-func serveAPIHome(resp http.ResponseWriter, req *http.Request) error {
- return &httpError{status: http.StatusNotFound}
-}
-
-type requestCleaner struct {
- h http.Handler
- trustProxyHeaders bool
-}
-
-func (rc requestCleaner) ServeHTTP(w http.ResponseWriter, req *http.Request) {
- req2 := new(http.Request)
- *req2 = *req
- if rc.trustProxyHeaders {
- if s := req.Header.Get("X-Forwarded-For"); s != "" {
- req2.RemoteAddr = s
- }
- }
- req2.Body = http.MaxBytesReader(w, req.Body, 2048)
- req2.ParseForm()
- rc.h.ServeHTTP(w, req2)
-}
-
-type errorHandler struct {
- fn func(resp http.ResponseWriter, req *http.Request) error
- errFn httputil.Error
-}
-
-func (eh errorHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- defer func() {
- if rv := recover(); rv != nil {
- err := errors.New("handler panic")
- logError(req, err, rv)
- eh.errFn(resp, req, http.StatusInternalServerError, err)
- }
- }()
-
- rb := new(httputil.ResponseBuffer)
- err := eh.fn(rb, req)
- if err == nil {
- rb.WriteTo(resp)
- } else if e, ok := err.(*httpError); ok {
- if e.status >= 500 {
- logError(req, err, nil)
- }
- eh.errFn(resp, req, e.status, e.err)
- } else if gosrc.IsNotFound(err) {
- eh.errFn(resp, req, http.StatusNotFound, nil)
- } else {
- logError(req, err, nil)
- eh.errFn(resp, req, http.StatusInternalServerError, err)
- }
-}
-
-func errorText(err error) string {
- if err == errUpdateTimeout {
- return "Timeout getting package files from the version control system."
- }
- if e, ok := err.(*gosrc.RemoteError); ok {
- return "Error getting package files from " + e.Host + "."
- }
- return "Internal server error."
-}
-
-func (s *server) handleError(resp http.ResponseWriter, req *http.Request, status int, err error) {
- switch status {
- case http.StatusNotFound:
- s.templates.execute(resp, "notfound"+templateExt(req), status, nil, map[string]interface{}{
- "flashMessages": getFlashMessages(resp, req),
- })
- default:
- resp.Header().Set("Content-Type", textMIMEType)
- resp.WriteHeader(http.StatusInternalServerError)
- io.WriteString(resp, errorText(err))
- }
-}
-
-func handleAPIError(resp http.ResponseWriter, req *http.Request, status int, err error) {
- var data struct {
- Error struct {
- Message string `json:"message"`
- } `json:"error"`
- }
- data.Error.Message = http.StatusText(status)
- resp.Header().Set("Content-Type", jsonMIMEType)
- resp.WriteHeader(status)
- json.NewEncoder(resp).Encode(&data)
-}
-
-// httpsRedirectHandler redirects all requests with an X-Forwarded-Proto: http
-// handler to their https equivalent.
-type httpsRedirectHandler struct {
- h http.Handler
-}
-
-func (h httpsRedirectHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- if req.Header.Get("X-Forwarded-Proto") == "http" {
- u := *req.URL
- u.Scheme = "https"
- u.Host = req.Host
- http.Redirect(resp, req, u.String(), http.StatusFound)
- return
- }
- h.h.ServeHTTP(resp, req)
-}
-
-type rootHandler []struct {
- prefix string
- h http.Handler
-}
-
-func (m rootHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- var h http.Handler
- for _, ph := range m {
- if strings.HasPrefix(req.Host, ph.prefix) {
- h = ph.h
- break
- }
- }
-
- h.ServeHTTP(resp, req)
-}
-
-// otherDomainHandler redirects to another domain keeping the rest of the URL.
-type otherDomainHandler struct {
- scheme string
- targetDomain string
-}
-
-func (h otherDomainHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
- u := *req.URL
- u.Scheme = h.scheme
- u.Host = h.targetDomain
- http.Redirect(w, req, u.String(), http.StatusFound)
-}
-
-func defaultBase(path string) string {
- p, err := build.Default.Import(path, "", build.FindOnly)
- if err != nil {
- return "."
- }
- return p.Dir
-}
-
-type server struct {
- v *viper.Viper
- db *database.Database
- httpClient *http.Client
- gceLogger *GCELogger
- templates templateMap
- traceClient *trace.Client
- crawlTopic *pubsub.Topic
-
- statusPNG http.Handler
- statusSVG http.Handler
-
- root rootHandler
-
- // A semaphore to limit concurrent ?import-graph requests.
- importGraphSem chan struct{}
-}
-
-func newServer(ctx context.Context, v *viper.Viper) (*server, error) {
- s := &server{
- v: v,
- httpClient: newHTTPClient(v),
- importGraphSem: make(chan struct{}, 10),
- }
-
- var err error
- if proj := s.v.GetString(ConfigProject); proj != "" {
- if s.traceClient, err = trace.NewClient(ctx, proj); err != nil {
- return nil, err
- }
- sp, err := trace.NewLimitedSampler(s.v.GetFloat64(ConfigTraceSamplerFraction), s.v.GetFloat64(ConfigTraceSamplerMaxQPS))
- if err != nil {
- return nil, err
- }
- s.traceClient.SetSamplingPolicy(sp)
-
- // This topic should be created in the cloud console.
- ps, err := pubsub.NewClient(ctx, proj)
- if err != nil {
- return nil, err
- }
- s.crawlTopic = ps.Topic(ConfigCrawlPubSubTopic)
- }
-
- assets := v.GetString(ConfigAssetsDir)
- staticServer := httputil.StaticServer{
- Dir: assets,
- MaxAge: time.Hour,
- MIMETypes: map[string]string{
- ".css": "text/css; charset=utf-8",
- ".js": "text/javascript; charset=utf-8",
- },
- }
- s.statusPNG = staticServer.FileHandler("status.png")
- s.statusSVG = staticServer.FileHandler("status.svg")
-
- apiHandler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
- return requestCleaner{
- h: errorHandler{
- fn: f,
- errFn: handleAPIError,
- },
- trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
- }
- }
- apiMux := http.NewServeMux()
- apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
- apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
- apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
- apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
- apiMux.Handle("/search", apiHandler(s.serveAPISearch))
- apiMux.Handle("/packages", apiHandler(s.serveAPIPackages))
- apiMux.Handle("/importers/", apiHandler(s.serveAPIImporters))
- apiMux.Handle("/imports/", apiHandler(s.serveAPIImports))
- apiMux.Handle("/", apiHandler(serveAPIHome))
-
- mux := http.NewServeMux()
- mux.Handle("/-/site.js", staticServer.FilesHandler(
- "third_party/jquery.timeago.js",
- "site.js"))
- mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
- mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
- mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
- mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
- if s.v.GetBool(ConfigSidebar) {
- mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
- }
- mux.Handle("/-/", http.NotFoundHandler())
-
- handler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
- return requestCleaner{
- h: errorHandler{
- fn: f,
- errFn: s.handleError,
- },
- trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
- }
- }
-
- mux.Handle("/-/about", handler(pkgGoDevRedirectHandler(s.serveAbout)))
- mux.Handle("/-/bot", handler(pkgGoDevRedirectHandler(s.serveBot)))
- mux.Handle("/-/go", handler(pkgGoDevRedirectHandler(s.serveGoIndex)))
- mux.Handle("/-/subrepo", handler(pkgGoDevRedirectHandler(s.serveGoSubrepoIndex)))
- mux.Handle("/-/refresh", handler(s.serveRefresh))
- mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
- mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
- mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
- mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
- mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
- mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
- mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
- mux.Handle("/code.jquery.com/", http.NotFoundHandler())
- mux.Handle("/", handler(pkgGoDevRedirectHandler(s.serveHome)))
-
- ahMux := http.NewServeMux()
- ready := new(health.Handler)
- ahMux.HandleFunc("/_ah/health", health.HandleLive)
- ahMux.Handle("/_ah/ready", ready)
-
- mainMux := http.NewServeMux()
- mainMux.Handle("/_ah/", ahMux)
- mainMux.Handle("/", s.traceClient.HTTPHandler(mux))
-
- s.root = rootHandler{
- {"api.", httpsRedirectHandler{s.traceClient.HTTPHandler(apiMux)}},
- {"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
- {"", httpsRedirectHandler{mainMux}},
- }
-
- cacheBusters := &httputil.CacheBusters{Handler: mux}
- s.templates, err = parseTemplates(assets, cacheBusters, v)
- if err != nil {
- return nil, err
- }
- s.db, err = database.New(
- v.GetString(ConfigDBServer),
- v.GetDuration(ConfigDBIdleTimeout),
- v.GetBool(ConfigDBLog),
- v.GetString(ConfigGAERemoteAPI),
- )
- if err != nil {
- return nil, fmt.Errorf("open database: %v", err)
- }
- ready.Add(s.db)
- if gceLogName := v.GetString(ConfigGCELogName); gceLogName != "" {
- logc, err := logging.NewClient(ctx, v.GetString(ConfigProject))
- if err != nil {
- return nil, fmt.Errorf("create cloud logging client: %v", err)
- }
- logger := logc.Logger(gceLogName)
- if err := logc.Ping(ctx); err != nil {
- return nil, fmt.Errorf("pinging cloud logging: %v", err)
- }
- s.gceLogger = newGCELogger(logger)
- }
- return s, nil
-}
-
-type responseWriter struct {
- http.ResponseWriter
- status int
-}
-
-func (rw *responseWriter) WriteHeader(code int) {
- rw.status = code
- rw.ResponseWriter.WriteHeader(code)
-}
-
-func translateStatus(code int) int {
- if code == 0 {
- return http.StatusOK
- }
- return code
-}
-
-func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- start := time.Now()
- s.logRequestStart(r)
- w2 := &responseWriter{ResponseWriter: w}
- s.root.ServeHTTP(w2, r)
- latency := time.Since(start)
- s.logRequestEnd(r, latency)
- if f, ok := w.(http.Flusher); ok {
- f.Flush()
- }
- s.teeRequestToPkgGoDev(r, latency, translateStatus(w2.status))
-}
-
-func (s *server) logRequestStart(req *http.Request) {
- if s.gceLogger == nil {
- return
- }
- s.gceLogger.Log(logging.Entry{
- HTTPRequest: &logging.HTTPRequest{Request: req},
- Payload: fmt.Sprintf("%s request start", req.Host),
- Severity: logging.Info,
- })
-}
-
-func (s *server) logRequestEnd(req *http.Request, latency time.Duration) {
- if s.gceLogger == nil {
- return
- }
- s.gceLogger.Log(logging.Entry{
- HTTPRequest: &logging.HTTPRequest{
- Request: req,
- Latency: latency,
- },
- Payload: fmt.Sprintf("%s request end", req.Host),
- Severity: logging.Info,
- })
-}
-
-func (s *server) teeRequestToPkgGoDev(r *http.Request, latency time.Duration, status int) {
- if shouldRedirectRequest(r) {
- log.Printf("shouldRedirectToPkgGoDev(%q, %q)= true: not teeing request because it is redirected to pkg.go.dev", r.URL.Host, r.URL.Path)
- return
- }
- if !shouldTeeRequest(r.URL.Path) {
- log.Printf("s.teeRequestToPkgGoDev: shouldTeeRequest(%q): not teeing request", r.URL.Path)
- return
- }
- val := os.Getenv("GDDO_TEE_REQUESTS_TO_PKGGODEV")
- if strings.ToLower(val) != "true" {
- log.Printf("s.teeRequestToPkgGoDev: not sending requests to pkg.go.dev (GDDO_TEE_REQUESTS_TO_PKGGODEV = %q)", val)
- return
- }
-
- log.Printf("s.teeRequestToPkgGoDev: teeRequestToPkgGoDev for %q (%d)", r.URL.Path, status)
- gddoEvent, pkggodevEvent := teeRequestToPkgGoDev(r, latency, s.isRobot(r), status)
- payload := map[string]interface{}{
- "godoc.org": gddoEvent,
- "pkg.go.dev": pkggodevEvent,
- }
- log.Printf("s.teeRequestToPkgGoDev: logging gddoEvent and pkggodevEvent: %q", r.URL.Path)
- if s.gceLogger == nil {
- log.Printf("no s.gceLogger: %q", r.URL.Path)
- for k, v := range payload {
- log.Printf("%q", k)
- log.Printf("%+v", v)
- }
- return
- }
- log.Printf("s.teeRequestToPkgGoDev: s.gceLogger.Log:%q", r.URL.Path)
- s.gceLogger.Log(logging.Entry{
- HTTPRequest: &logging.HTTPRequest{
- Request: r,
- Latency: latency,
- Status: status,
- },
- Payload: payload,
- Severity: logging.Info,
- })
- return
-}
-
func main() {
ctx := context.Background()
v, err := loadConfig(ctx, os.Args)
diff --git a/gddo-server/server.go b/gddo-server/server.go
new file mode 100644
index 0000000..07ebf1f
--- /dev/null
+++ b/gddo-server/server.go
@@ -0,0 +1,1117 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+// Command gddo-server is the GoPkgDoc server.
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/md5"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "go/build"
+ "html/template"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "path"
+ "regexp"
+ "runtime/debug"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "cloud.google.com/go/logging"
+ "cloud.google.com/go/pubsub"
+ "cloud.google.com/go/trace"
+ "github.com/golang/gddo/database"
+ "github.com/golang/gddo/doc"
+ "github.com/golang/gddo/gosrc"
+ "github.com/golang/gddo/httputil"
+ "github.com/golang/gddo/internal/health"
+ "github.com/spf13/viper"
+)
+
+const (
+ jsonMIMEType = "application/json; charset=utf-8"
+ textMIMEType = "text/plain; charset=utf-8"
+ htmlMIMEType = "text/html; charset=utf-8"
+)
+
+var errUpdateTimeout = errors.New("refresh timeout")
+
+type httpError struct {
+ status int // HTTP status code.
+ err error // Optional reason for the HTTP error.
+}
+
+func (err *httpError) Error() string {
+ if err.err != nil {
+ return fmt.Sprintf("status %d, reason %s", err.status, err.err.Error())
+ }
+ return fmt.Sprintf("Status %d", err.status)
+}
+
+const (
+ humanRequest = iota
+ robotRequest
+ queryRequest
+ refreshRequest
+ apiRequest
+)
+
+type crawlResult struct {
+ pdoc *doc.Package
+ err error
+}
+
+// getDoc gets the package documentation from the database or from the version
+// control system as needed.
+func (s *server) getDoc(ctx context.Context, path string, requestType int) (*doc.Package, []database.Package, error) {
+ if path == "-" {
+ // A hack in the database package uses the path "-" to represent the
+ // next document to crawl. Block "-" here so that requests to /- always
+ // return not found.
+ return nil, nil, &httpError{status: http.StatusNotFound}
+ }
+
+ pdoc, pkgs, nextCrawl, err := s.db.Get(ctx, path)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ needsCrawl := false
+ switch requestType {
+ case queryRequest, apiRequest:
+ needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0
+ case humanRequest:
+ needsCrawl = nextCrawl.Before(time.Now())
+ case robotRequest:
+ needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0
+ }
+
+ if !needsCrawl {
+ return pdoc, pkgs, nil
+ }
+
+ c := make(chan crawlResult, 1)
+ go func() {
+ pdoc, err := s.crawlDoc(ctx, "web ", path, pdoc, len(pkgs) > 0, nextCrawl)
+ c <- crawlResult{pdoc, err}
+ }()
+
+ timeout := s.v.GetDuration(ConfigGetTimeout)
+ if pdoc == nil {
+ timeout = s.v.GetDuration(ConfigFirstGetTimeout)
+ }
+
+ select {
+ case cr := <-c:
+ err = cr.err
+ if err == nil {
+ pdoc = cr.pdoc
+ }
+ case <-time.After(timeout):
+ err = errUpdateTimeout
+ }
+
+ switch {
+ case err == nil:
+ return pdoc, pkgs, nil
+ case gosrc.IsNotFound(err):
+ return nil, nil, err
+ case pdoc != nil:
+ log.Printf("Serving %q from database after error getting doc: %v", path, err)
+ return pdoc, pkgs, nil
+ case err == errUpdateTimeout:
+ log.Printf("Serving %q as not found after timeout getting doc", path)
+ return nil, nil, &httpError{status: http.StatusNotFound}
+ default:
+ return nil, nil, err
+ }
+}
+
+func templateExt(req *http.Request) string {
+ if httputil.NegotiateContentType(req, []string{"text/html", "text/plain"}, "text/html") == "text/plain" {
+ return ".txt"
+ }
+ return ".html"
+}
+
+var robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)|(?:^Python-urllib)|(?:^Go )|(?:^Java/)`)
+
+func (s *server) isRobot(req *http.Request) bool {
+ if robotPat.MatchString(req.Header.Get("User-Agent")) {
+ return true
+ }
+ host := httputil.StripPort(req.RemoteAddr)
+ n, err := s.db.IncrementCounter(host, 1)
+ if err != nil {
+ log.Printf("error incrementing counter for %s, %v", host, err)
+ return false
+ }
+ if n > s.v.GetFloat64(ConfigRobotThreshold) {
+ log.Printf("robot %.2f %s %s", n, host, req.Header.Get("User-Agent"))
+ return true
+ }
+ return false
+}
+
+func popularLinkReferral(req *http.Request) bool {
+ return strings.HasSuffix(req.Header.Get("Referer"), "//"+req.Host+"/")
+}
+
+func isView(req *http.Request, key string) bool {
+ rq := req.URL.RawQuery
+ return strings.HasPrefix(rq, key) &&
+ (len(rq) == len(key) || rq[len(key)] == '=' || rq[len(key)] == '&')
+}
+
+// httpEtag returns the package entity tag used in HTTP transactions.
+func (s *server) httpEtag(pdoc *doc.Package, pkgs []database.Package, importerCount int, flashMessages []flashMessage) string {
+ b := make([]byte, 0, 128)
+ b = strconv.AppendInt(b, pdoc.Updated.Unix(), 16)
+ b = append(b, 0)
+ b = append(b, pdoc.Etag...)
+ if importerCount >= 8 {
+ importerCount = 8
+ }
+ b = append(b, 0)
+ b = strconv.AppendInt(b, int64(importerCount), 16)
+ for _, pkg := range pkgs {
+ b = append(b, 0)
+ b = append(b, pkg.Path...)
+ b = append(b, 0)
+ b = append(b, pkg.Synopsis...)
+ }
+ if s.v.GetBool(ConfigSidebar) {
+ b = append(b, "\000xsb"...)
+ }
+ for _, m := range flashMessages {
+ b = append(b, 0)
+ b = append(b, m.ID...)
+ for _, a := range m.Args {
+ b = append(b, 1)
+ b = append(b, a...)
+ }
+ }
+ h := md5.New()
+ h.Write(b)
+ b = h.Sum(b[:0])
+ return fmt.Sprintf("\"%x\"", b)
+}
+
+func (s *server) servePackage(resp http.ResponseWriter, req *http.Request) error {
+ p := path.Clean(req.URL.Path)
+ if strings.HasPrefix(p, "/pkg/") {
+ p = p[len("/pkg"):]
+ }
+ if p != req.URL.Path {
+ http.Redirect(resp, req, p, http.StatusMovedPermanently)
+ return nil
+ }
+
+ if isView(req, "status.svg") {
+ s.statusSVG.ServeHTTP(resp, req)
+ return nil
+ }
+
+ if isView(req, "status.png") {
+ s.statusPNG.ServeHTTP(resp, req)
+ return nil
+ }
+
+ requestType := humanRequest
+ if s.isRobot(req) {
+ requestType = robotRequest
+ }
+
+ importPath := strings.TrimPrefix(req.URL.Path, "/")
+ pdoc, pkgs, err := s.getDoc(req.Context(), importPath, requestType)
+
+ if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+ // To prevent dumb clients from following redirect loops, respond with
+ // status 404 if the target document is not found.
+ if _, _, err := s.getDoc(req.Context(), e.Redirect, requestType); gosrc.IsNotFound(err) {
+ return &httpError{status: http.StatusNotFound}
+ }
+ u := "/" + e.Redirect
+ if req.URL.RawQuery != "" {
+ u += "?" + req.URL.RawQuery
+ }
+ setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
+ http.Redirect(resp, req, u, http.StatusFound)
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+
+ flashMessages := getFlashMessages(resp, req)
+
+ if pdoc == nil {
+ if len(pkgs) == 0 {
+ return &httpError{status: http.StatusNotFound}
+ }
+ pdocChild, _, _, err := s.db.Get(req.Context(), pkgs[0].Path)
+ if err != nil {
+ return err
+ }
+ pdoc = &doc.Package{
+ ProjectName: pdocChild.ProjectName,
+ ProjectRoot: pdocChild.ProjectRoot,
+ ProjectURL: pdocChild.ProjectURL,
+ ImportPath: importPath,
+ }
+ }
+
+ showPkgGoDevRedirectToast := userReturningFromPkgGoDev(req)
+
+ switch {
+ case isView(req, "imports"):
+ if pdoc.Name == "" {
+ return &httpError{status: http.StatusNotFound}
+ }
+ pkgs, err = s.db.Packages(pdoc.Imports)
+ if err != nil {
+ return err
+ }
+ return s.templates.execute(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{
+ "flashMessages": flashMessages,
+ "pkgs": pkgs,
+ "pdoc": newTDoc(s.v, pdoc),
+ "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
+ })
+ case isView(req, "tools"):
+ proto := "http"
+ if req.Host == "godoc.org" {
+ proto = "https"
+ }
+ return s.templates.execute(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{
+ "flashMessages": flashMessages,
+ "uri": fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath),
+ "pdoc": newTDoc(s.v, pdoc),
+ "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
+ })
+ case isView(req, "importers"):
+ if pdoc.Name == "" {
+ return &httpError{status: http.StatusNotFound}
+ }
+ pkgs, err = s.db.Importers(importPath)
+ if err != nil {
+ return err
+ }
+ template := "importers.html"
+ if requestType == robotRequest {
+ // Hide back links from robots.
+ template = "importers_robot.html"
+ }
+ return s.templates.execute(resp, template, http.StatusOK, nil, map[string]interface{}{
+ "flashMessages": flashMessages,
+ "pkgs": pkgs,
+ "pdoc": newTDoc(s.v, pdoc),
+ "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
+ })
+ case isView(req, "import-graph"):
+ if requestType == robotRequest {
+ return &httpError{status: http.StatusForbidden}
+ }
+ if pdoc.Name == "" {
+ return &httpError{status: http.StatusNotFound}
+ }
+
+ // Throttle ?import-graph requests.
+ select {
+ case s.importGraphSem <- struct{}{}:
+ default:
+ return &httpError{status: http.StatusTooManyRequests}
+ }
+ defer func() { <-s.importGraphSem }()
+
+ hide := database.ShowAllDeps
+ switch req.Form.Get("hide") {
+ case "1":
+ hide = database.HideStandardDeps
+ case "2":
+ hide = database.HideStandardAll
+ }
+ pkgs, edges, err := s.db.ImportGraph(pdoc, hide)
+ if err != nil {
+ return err
+ }
+ b, err := renderGraph(pdoc, pkgs, edges)
+ if err != nil {
+ return err
+ }
+ return s.templates.execute(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{
+ "flashMessages": flashMessages,
+ "svg": template.HTML(b),
+ "pdoc": newTDoc(s.v, pdoc),
+ "hide": hide,
+ "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
+ })
+ case isView(req, "play"):
+ u, err := s.playURL(pdoc, req.Form.Get("play"), req.Header.Get("X-AppEngine-Country"))
+ if err != nil {
+ return err
+ }
+ http.Redirect(resp, req, u, http.StatusMovedPermanently)
+ return nil
+ case req.Form.Get("view") != "":
+ // Redirect deprecated view= queries.
+ var q string
+ switch view := req.Form.Get("view"); view {
+ case "imports", "importers":
+ q = view
+ case "import-graph":
+ if req.Form.Get("hide") == "1" {
+ q = "import-graph&hide=1"
+ } else {
+ q = "import-graph"
+ }
+ }
+ if q != "" {
+ u := *req.URL
+ u.RawQuery = q
+ http.Redirect(resp, req, u.String(), http.StatusMovedPermanently)
+ return nil
+ }
+ return &httpError{status: http.StatusNotFound}
+ default:
+ importerCount := 0
+ if pdoc.Name != "" {
+ importerCount, err = s.db.ImporterCount(importPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ etag := s.httpEtag(pdoc, pkgs, importerCount, flashMessages)
+ status := http.StatusOK
+ if req.Header.Get("If-None-Match") == etag {
+ status = http.StatusNotModified
+ }
+
+ if requestType == humanRequest &&
+ pdoc.Name != "" && // not a directory
+ pdoc.ProjectRoot != "" && // not a standard package
+ !pdoc.IsCmd &&
+ len(pdoc.Errors) == 0 &&
+ !popularLinkReferral(req) {
+ if err := s.db.IncrementPopularScore(pdoc.ImportPath); err != nil {
+ log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err)
+ }
+ }
+ if s.gceLogger != nil {
+ s.gceLogger.LogEvent(resp, req, nil)
+ }
+
+ template := "dir"
+ switch {
+ case pdoc.IsCmd:
+ template = "cmd"
+ case pdoc.Name != "":
+ template = "pkg"
+ }
+ template += templateExt(req)
+
+ return s.templates.execute(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{
+ "flashMessages": flashMessages,
+ "pkgs": pkgs,
+ "pdoc": newTDoc(s.v, pdoc),
+ "importerCount": importerCount,
+ "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
+ })
+ }
+}
+
+func (s *server) serveRefresh(resp http.ResponseWriter, req *http.Request) error {
+ importPath := req.Form.Get("path")
+ _, pkgs, _, err := s.db.Get(req.Context(), importPath)
+ if err != nil {
+ return err
+ }
+ c := make(chan error, 1)
+ go func() {
+ _, err := s.crawlDoc(req.Context(), "rfrsh", importPath, nil, len(pkgs) > 0, time.Time{})
+ c <- err
+ }()
+ select {
+ case err = <-c:
+ case <-time.After(s.v.GetDuration(ConfigGetTimeout)):
+ err = errUpdateTimeout
+ }
+ if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+ setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
+ importPath = e.Redirect
+ err = nil
+ } else if err != nil {
+ setFlashMessages(resp, []flashMessage{{ID: "refresh", Args: []string{errorText(err)}}})
+ }
+ http.Redirect(resp, req, "/"+importPath, http.StatusFound)
+ return nil
+}
+
+func (s *server) serveGoIndex(resp http.ResponseWriter, req *http.Request) error {
+ pkgs, err := s.db.GoIndex()
+ if err != nil {
+ return err
+ }
+ return s.templates.execute(resp, "std.html", http.StatusOK, nil, map[string]interface{}{
+ "pkgs": pkgs,
+ })
+}
+
+func (s *server) serveGoSubrepoIndex(resp http.ResponseWriter, req *http.Request) error {
+ pkgs, err := s.db.GoSubrepoIndex()
+ if err != nil {
+ return err
+ }
+ return s.templates.execute(resp, "subrepo.html", http.StatusOK, nil, map[string]interface{}{
+ "pkgs": pkgs,
+ })
+}
+
+type byPath struct {
+ pkgs []database.Package
+ rank []int
+}
+
+func (bp *byPath) Len() int { return len(bp.pkgs) }
+func (bp *byPath) Less(i, j int) bool { return bp.pkgs[i].Path < bp.pkgs[j].Path }
+func (bp *byPath) Swap(i, j int) {
+ bp.pkgs[i], bp.pkgs[j] = bp.pkgs[j], bp.pkgs[i]
+ bp.rank[i], bp.rank[j] = bp.rank[j], bp.rank[i]
+}
+
+type byRank struct {
+ pkgs []database.Package
+ rank []int
+}
+
+func (br *byRank) Len() int { return len(br.pkgs) }
+func (br *byRank) Less(i, j int) bool { return br.rank[i] < br.rank[j] }
+func (br *byRank) Swap(i, j int) {
+ br.pkgs[i], br.pkgs[j] = br.pkgs[j], br.pkgs[i]
+ br.rank[i], br.rank[j] = br.rank[j], br.rank[i]
+}
+
+func (s *server) popular() ([]database.Package, error) {
+ const n = 25
+
+ pkgs, err := s.db.Popular(2 * n)
+ if err != nil {
+ return nil, err
+ }
+
+ rank := make([]int, len(pkgs))
+ for i := range pkgs {
+ rank[i] = i
+ }
+
+ sort.Sort(&byPath{pkgs, rank})
+
+ j := 0
+ prev := "."
+ for i, pkg := range pkgs {
+ if strings.HasPrefix(pkg.Path, prev) {
+ if rank[j-1] < rank[i] {
+ rank[j-1] = rank[i]
+ }
+ continue
+ }
+ prev = pkg.Path + "/"
+ pkgs[j] = pkg
+ rank[j] = rank[i]
+ j++
+ }
+ pkgs = pkgs[:j]
+
+ sort.Sort(&byRank{pkgs, rank})
+
+ if len(pkgs) > n {
+ pkgs = pkgs[:n]
+ }
+
+ sort.Sort(&byPath{pkgs, rank})
+
+ return pkgs, nil
+}
+
+func (s *server) serveHome(resp http.ResponseWriter, req *http.Request) error {
+ if req.URL.Path != "/" {
+ return s.servePackage(resp, req)
+ }
+
+ q := strings.TrimSpace(req.Form.Get("q"))
+ if q == "" {
+ pkgs, err := s.popular()
+ if err != nil {
+ return err
+ }
+
+ return s.templates.execute(resp, "home"+templateExt(req), http.StatusOK, nil,
+ map[string]interface{}{
+ "Popular": pkgs,
+
+ "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
+ })
+ }
+
+ if path, ok := isBrowseURL(q); ok {
+ q = path
+ }
+
+ if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
+ pdoc, pkgs, err := s.getDoc(req.Context(), q, queryRequest)
+ if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+ http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
+ return nil
+ }
+ if err == nil && (pdoc != nil || len(pkgs) > 0) {
+ http.Redirect(resp, req, "/"+q, http.StatusFound)
+ return nil
+ }
+ }
+
+ pkgs, err := s.db.Search(req.Context(), q)
+ if err != nil {
+ return err
+ }
+ if s.gceLogger != nil {
+ // Log up to top 10 packages we served upon a search.
+ logPkgs := pkgs
+ if len(pkgs) > 10 {
+ logPkgs = pkgs[:10]
+ }
+ s.gceLogger.LogEvent(resp, req, logPkgs)
+ }
+
+ return s.templates.execute(resp, "results"+templateExt(req), http.StatusOK, nil,
+ map[string]interface{}{
+ "q": q,
+ "pkgs": pkgs,
+
+ "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
+ })
+}
+
+func (s *server) serveAbout(resp http.ResponseWriter, req *http.Request) error {
+ return s.templates.execute(resp, "about.html", http.StatusOK, nil,
+ map[string]interface{}{
+ "Host": req.Host,
+
+ "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
+ })
+}
+
+func (s *server) serveBot(resp http.ResponseWriter, req *http.Request) error {
+ return s.templates.execute(resp, "bot.html", http.StatusOK, nil, nil)
+}
+
+func logError(req *http.Request, err error, rv interface{}) {
+ if err != nil {
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "Error serving %s: %v\n", req.URL, err)
+ if rv != nil {
+ fmt.Fprintln(&buf, rv)
+ buf.Write(debug.Stack())
+ }
+ log.Print(buf.String())
+ }
+}
+
+func (s *server) serveAPISearch(resp http.ResponseWriter, req *http.Request) error {
+ q := strings.TrimSpace(req.Form.Get("q"))
+
+ var pkgs []database.Package
+
+ if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
+ pdoc, _, err := s.getDoc(req.Context(), q, apiRequest)
+ if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+ pdoc, _, err = s.getDoc(req.Context(), e.Redirect, robotRequest)
+ }
+ if err == nil && pdoc != nil {
+ pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}}
+ }
+ }
+
+ if pkgs == nil {
+ var err error
+ pkgs, err = s.db.Search(req.Context(), q)
+ if err != nil {
+ return err
+ }
+ }
+
+ var data = struct {
+ Results []database.Package `json:"results"`
+ }{
+ pkgs,
+ }
+ resp.Header().Set("Content-Type", jsonMIMEType)
+ return json.NewEncoder(resp).Encode(&data)
+}
+
+func (s *server) serveAPIPackages(resp http.ResponseWriter, req *http.Request) error {
+ pkgs, err := s.db.AllPackages()
+ if err != nil {
+ return err
+ }
+ data := struct {
+ Results []database.Package `json:"results"`
+ }{
+ pkgs,
+ }
+ resp.Header().Set("Content-Type", jsonMIMEType)
+ return json.NewEncoder(resp).Encode(&data)
+}
+
+func (s *server) serveAPIImporters(resp http.ResponseWriter, req *http.Request) error {
+ importPath := strings.TrimPrefix(req.URL.Path, "/importers/")
+ pkgs, err := s.db.Importers(importPath)
+ if err != nil {
+ return err
+ }
+ data := struct {
+ Results []database.Package `json:"results"`
+ }{
+ pkgs,
+ }
+ resp.Header().Set("Content-Type", jsonMIMEType)
+ return json.NewEncoder(resp).Encode(&data)
+}
+
+func (s *server) serveAPIImports(resp http.ResponseWriter, req *http.Request) error {
+ importPath := strings.TrimPrefix(req.URL.Path, "/imports/")
+ pdoc, _, err := s.getDoc(req.Context(), importPath, robotRequest)
+ if err != nil {
+ return err
+ }
+ if pdoc == nil || pdoc.Name == "" {
+ return &httpError{status: http.StatusNotFound}
+ }
+ imports, err := s.db.Packages(pdoc.Imports)
+ if err != nil {
+ return err
+ }
+ testImports, err := s.db.Packages(pdoc.TestImports)
+ if err != nil {
+ return err
+ }
+ data := struct {
+ Imports []database.Package `json:"imports"`
+ TestImports []database.Package `json:"testImports"`
+ }{
+ imports,
+ testImports,
+ }
+ resp.Header().Set("Content-Type", jsonMIMEType)
+ return json.NewEncoder(resp).Encode(&data)
+}
+
+func serveAPIHome(resp http.ResponseWriter, req *http.Request) error {
+ return &httpError{status: http.StatusNotFound}
+}
+
+type requestCleaner struct {
+ h http.Handler
+ trustProxyHeaders bool
+}
+
+func (rc requestCleaner) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ req2 := new(http.Request)
+ *req2 = *req
+ if rc.trustProxyHeaders {
+ if s := req.Header.Get("X-Forwarded-For"); s != "" {
+ req2.RemoteAddr = s
+ }
+ }
+ req2.Body = http.MaxBytesReader(w, req.Body, 2048)
+ req2.ParseForm()
+ rc.h.ServeHTTP(w, req2)
+}
+
+type errorHandler struct {
+ fn func(resp http.ResponseWriter, req *http.Request) error
+ errFn httputil.Error
+}
+
+func (eh errorHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ defer func() {
+ if rv := recover(); rv != nil {
+ err := errors.New("handler panic")
+ logError(req, err, rv)
+ eh.errFn(resp, req, http.StatusInternalServerError, err)
+ }
+ }()
+
+ rb := new(httputil.ResponseBuffer)
+ err := eh.fn(rb, req)
+ if err == nil {
+ rb.WriteTo(resp)
+ } else if e, ok := err.(*httpError); ok {
+ if e.status >= 500 {
+ logError(req, err, nil)
+ }
+ eh.errFn(resp, req, e.status, e.err)
+ } else if gosrc.IsNotFound(err) {
+ eh.errFn(resp, req, http.StatusNotFound, nil)
+ } else {
+ logError(req, err, nil)
+ eh.errFn(resp, req, http.StatusInternalServerError, err)
+ }
+}
+
+func errorText(err error) string {
+ if err == errUpdateTimeout {
+ return "Timeout getting package files from the version control system."
+ }
+ if e, ok := err.(*gosrc.RemoteError); ok {
+ return "Error getting package files from " + e.Host + "."
+ }
+ return "Internal server error."
+}
+
+func (s *server) handleError(resp http.ResponseWriter, req *http.Request, status int, err error) {
+ switch status {
+ case http.StatusNotFound:
+ s.templates.execute(resp, "notfound"+templateExt(req), status, nil, map[string]interface{}{
+ "flashMessages": getFlashMessages(resp, req),
+ })
+ default:
+ resp.Header().Set("Content-Type", textMIMEType)
+ resp.WriteHeader(http.StatusInternalServerError)
+ io.WriteString(resp, errorText(err))
+ }
+}
+
+func handleAPIError(resp http.ResponseWriter, req *http.Request, status int, err error) {
+ var data struct {
+ Error struct {
+ Message string `json:"message"`
+ } `json:"error"`
+ }
+ data.Error.Message = http.StatusText(status)
+ resp.Header().Set("Content-Type", jsonMIMEType)
+ resp.WriteHeader(status)
+ json.NewEncoder(resp).Encode(&data)
+}
+
+// httpsRedirectHandler redirects all requests with an X-Forwarded-Proto: http
+// handler to their https equivalent.
+type httpsRedirectHandler struct {
+ h http.Handler
+}
+
+func (h httpsRedirectHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if req.Header.Get("X-Forwarded-Proto") == "http" {
+ u := *req.URL
+ u.Scheme = "https"
+ u.Host = req.Host
+ http.Redirect(resp, req, u.String(), http.StatusFound)
+ return
+ }
+ h.h.ServeHTTP(resp, req)
+}
+
+type rootHandler []struct {
+ prefix string
+ h http.Handler
+}
+
+func (m rootHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ var h http.Handler
+ for _, ph := range m {
+ if strings.HasPrefix(req.Host, ph.prefix) {
+ h = ph.h
+ break
+ }
+ }
+
+ h.ServeHTTP(resp, req)
+}
+
+// otherDomainHandler redirects to another domain keeping the rest of the URL.
+type otherDomainHandler struct {
+ scheme string
+ targetDomain string
+}
+
+func (h otherDomainHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ u := *req.URL
+ u.Scheme = h.scheme
+ u.Host = h.targetDomain
+ http.Redirect(w, req, u.String(), http.StatusFound)
+}
+
+func defaultBase(path string) string {
+ p, err := build.Default.Import(path, "", build.FindOnly)
+ if err != nil {
+ return "."
+ }
+ return p.Dir
+}
+
+type server struct {
+ v *viper.Viper
+ db *database.Database
+ httpClient *http.Client
+ gceLogger *GCELogger
+ templates templateMap
+ traceClient *trace.Client
+ crawlTopic *pubsub.Topic
+
+ statusPNG http.Handler
+ statusSVG http.Handler
+
+ root rootHandler
+
+ // A semaphore to limit concurrent ?import-graph requests.
+ importGraphSem chan struct{}
+}
+
+func newServer(ctx context.Context, v *viper.Viper) (*server, error) {
+ s := &server{
+ v: v,
+ httpClient: newHTTPClient(v),
+ importGraphSem: make(chan struct{}, 10),
+ }
+
+ var err error
+ if proj := s.v.GetString(ConfigProject); proj != "" {
+ if s.traceClient, err = trace.NewClient(ctx, proj); err != nil {
+ return nil, err
+ }
+ sp, err := trace.NewLimitedSampler(s.v.GetFloat64(ConfigTraceSamplerFraction), s.v.GetFloat64(ConfigTraceSamplerMaxQPS))
+ if err != nil {
+ return nil, err
+ }
+ s.traceClient.SetSamplingPolicy(sp)
+
+ // This topic should be created in the cloud console.
+ ps, err := pubsub.NewClient(ctx, proj)
+ if err != nil {
+ return nil, err
+ }
+ s.crawlTopic = ps.Topic(ConfigCrawlPubSubTopic)
+ }
+
+ assets := v.GetString(ConfigAssetsDir)
+ staticServer := httputil.StaticServer{
+ Dir: assets,
+ MaxAge: time.Hour,
+ MIMETypes: map[string]string{
+ ".css": "text/css; charset=utf-8",
+ ".js": "text/javascript; charset=utf-8",
+ },
+ }
+ s.statusPNG = staticServer.FileHandler("status.png")
+ s.statusSVG = staticServer.FileHandler("status.svg")
+
+ apiHandler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
+ return requestCleaner{
+ h: errorHandler{
+ fn: f,
+ errFn: handleAPIError,
+ },
+ trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
+ }
+ }
+ apiMux := http.NewServeMux()
+ apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
+ apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
+ apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
+ apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
+ apiMux.Handle("/search", apiHandler(s.serveAPISearch))
+ apiMux.Handle("/packages", apiHandler(s.serveAPIPackages))
+ apiMux.Handle("/importers/", apiHandler(s.serveAPIImporters))
+ apiMux.Handle("/imports/", apiHandler(s.serveAPIImports))
+ apiMux.Handle("/", apiHandler(serveAPIHome))
+
+ mux := http.NewServeMux()
+ mux.Handle("/-/site.js", staticServer.FilesHandler(
+ "third_party/jquery.timeago.js",
+ "site.js"))
+ mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
+ mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
+ mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
+ mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
+ if s.v.GetBool(ConfigSidebar) {
+ mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
+ }
+ mux.Handle("/-/", http.NotFoundHandler())
+
+ handler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
+ return requestCleaner{
+ h: errorHandler{
+ fn: f,
+ errFn: s.handleError,
+ },
+ trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
+ }
+ }
+
+ mux.Handle("/-/about", handler(pkgGoDevRedirectHandler(s.serveAbout)))
+ mux.Handle("/-/bot", handler(pkgGoDevRedirectHandler(s.serveBot)))
+ mux.Handle("/-/go", handler(pkgGoDevRedirectHandler(s.serveGoIndex)))
+ mux.Handle("/-/subrepo", handler(pkgGoDevRedirectHandler(s.serveGoSubrepoIndex)))
+ mux.Handle("/-/refresh", handler(s.serveRefresh))
+ mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
+ mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
+ mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
+ mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
+ mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
+ mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
+ mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
+ mux.Handle("/code.jquery.com/", http.NotFoundHandler())
+ mux.Handle("/", handler(pkgGoDevRedirectHandler(s.serveHome)))
+
+ ahMux := http.NewServeMux()
+ ready := new(health.Handler)
+ ahMux.HandleFunc("/_ah/health", health.HandleLive)
+ ahMux.Handle("/_ah/ready", ready)
+
+ mainMux := http.NewServeMux()
+ mainMux.Handle("/_ah/", ahMux)
+ mainMux.Handle("/", s.traceClient.HTTPHandler(mux))
+
+ s.root = rootHandler{
+ {"api.", httpsRedirectHandler{s.traceClient.HTTPHandler(apiMux)}},
+ {"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
+ {"", httpsRedirectHandler{mainMux}},
+ }
+
+ cacheBusters := &httputil.CacheBusters{Handler: mux}
+ s.templates, err = parseTemplates(assets, cacheBusters, v)
+ if err != nil {
+ return nil, err
+ }
+ s.db, err = database.New(
+ v.GetString(ConfigDBServer),
+ v.GetDuration(ConfigDBIdleTimeout),
+ v.GetBool(ConfigDBLog),
+ v.GetString(ConfigGAERemoteAPI),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("open database: %v", err)
+ }
+ ready.Add(s.db)
+ if gceLogName := v.GetString(ConfigGCELogName); gceLogName != "" {
+ logc, err := logging.NewClient(ctx, v.GetString(ConfigProject))
+ if err != nil {
+ return nil, fmt.Errorf("create cloud logging client: %v", err)
+ }
+ logger := logc.Logger(gceLogName)
+ if err := logc.Ping(ctx); err != nil {
+ return nil, fmt.Errorf("pinging cloud logging: %v", err)
+ }
+ s.gceLogger = newGCELogger(logger)
+ }
+ return s, nil
+}
+
+type responseWriter struct {
+ http.ResponseWriter
+ status int
+}
+
+func (rw *responseWriter) WriteHeader(code int) {
+ rw.status = code
+ rw.ResponseWriter.WriteHeader(code)
+}
+
+func translateStatus(code int) int {
+ if code == 0 {
+ return http.StatusOK
+ }
+ return code
+}
+
+func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ start := time.Now()
+ s.logRequestStart(r)
+ w2 := &responseWriter{ResponseWriter: w}
+ s.root.ServeHTTP(w2, r)
+ latency := time.Since(start)
+ s.logRequestEnd(r, latency)
+ if f, ok := w.(http.Flusher); ok {
+ f.Flush()
+ }
+ s.teeRequestToPkgGoDev(r, latency, translateStatus(w2.status))
+}
+
+func (s *server) logRequestStart(req *http.Request) {
+ if s.gceLogger == nil {
+ return
+ }
+ s.gceLogger.Log(logging.Entry{
+ HTTPRequest: &logging.HTTPRequest{Request: req},
+ Payload: fmt.Sprintf("%s request start", req.Host),
+ Severity: logging.Info,
+ })
+}
+
+func (s *server) logRequestEnd(req *http.Request, latency time.Duration) {
+ if s.gceLogger == nil {
+ return
+ }
+ s.gceLogger.Log(logging.Entry{
+ HTTPRequest: &logging.HTTPRequest{
+ Request: req,
+ Latency: latency,
+ },
+ Payload: fmt.Sprintf("%s request end", req.Host),
+ Severity: logging.Info,
+ })
+}
+
+func (s *server) teeRequestToPkgGoDev(r *http.Request, latency time.Duration, status int) {
+ if shouldRedirectRequest(r) {
+ log.Printf("shouldRedirectToPkgGoDev(%q, %q)= true: not teeing request because it is redirected to pkg.go.dev", r.URL.Host, r.URL.Path)
+ return
+ }
+ if !shouldTeeRequest(r.URL.Path) {
+ log.Printf("s.teeRequestToPkgGoDev: shouldTeeRequest(%q): not teeing request", r.URL.Path)
+ return
+ }
+ val := os.Getenv("GDDO_TEE_REQUESTS_TO_PKGGODEV")
+ if strings.ToLower(val) != "true" {
+ log.Printf("s.teeRequestToPkgGoDev: not sending requests to pkg.go.dev (GDDO_TEE_REQUESTS_TO_PKGGODEV = %q)", val)
+ return
+ }
+
+ log.Printf("s.teeRequestToPkgGoDev: teeRequestToPkgGoDev for %q (%d)", r.URL.Path, status)
+ gddoEvent, pkggodevEvent := teeRequestToPkgGoDev(r, latency, s.isRobot(r), status)
+ payload := map[string]interface{}{
+ "godoc.org": gddoEvent,
+ "pkg.go.dev": pkggodevEvent,
+ }
+ log.Printf("s.teeRequestToPkgGoDev: logging gddoEvent and pkggodevEvent: %q", r.URL.Path)
+ if s.gceLogger == nil {
+ log.Printf("no s.gceLogger: %q", r.URL.Path)
+ for k, v := range payload {
+ log.Printf("%q", k)
+ log.Printf("%+v", v)
+ }
+ return
+ }
+ log.Printf("s.teeRequestToPkgGoDev: s.gceLogger.Log:%q", r.URL.Path)
+ s.gceLogger.Log(logging.Entry{
+ HTTPRequest: &logging.HTTPRequest{
+ Request: r,
+ Latency: latency,
+ Status: status,
+ },
+ Payload: payload,
+ Severity: logging.Info,
+ })
+ return
+}
