cmd/golangorg/server.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Golangorg serves the golang.org web sites.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"sync/atomic"
	"time"

	"cloud.google.com/go/datastore"
	"golang.org/x/build/repos"
	"golang.org/x/tools/playground"
	"golang.org/x/website"
	"golang.org/x/website/internal/backport/archive/zip"
	"golang.org/x/website/internal/backport/html/template"
	"golang.org/x/website/internal/backport/io/fs"
	"golang.org/x/website/internal/backport/osfs"
	"golang.org/x/website/internal/codewalk"
	"golang.org/x/website/internal/dl"
	"golang.org/x/website/internal/env"
	"golang.org/x/website/internal/gitfs"
	"golang.org/x/website/internal/history"
	"golang.org/x/website/internal/memcache"
	"golang.org/x/website/internal/pkgdoc"
	"golang.org/x/website/internal/proxy"
	"golang.org/x/website/internal/redirect"
	"golang.org/x/website/internal/short"
	"golang.org/x/website/internal/web"
	"golang.org/x/website/internal/webtest"
)

var (
	httpAddr   = flag.String("http", "localhost:6060", "HTTP service address")
	verbose    = flag.Bool("v", false, "verbose mode")
	goroot     = flag.String("goroot", runtime.GOROOT(), "Go root directory")
	contentDir = flag.String("content", "", "path to _content directory")

	runningOnAppEngine = os.Getenv("PORT") != ""

	googleAnalytics string
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: golangorg\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	if *contentDir == "" {
		repoRoot := "../.."
		if _, err := os.Stat("_content"); err == nil {
			repoRoot = "."
		}
		*contentDir = filepath.Join(repoRoot, "_content")
	}

	if runningOnAppEngine {
		log.Print("golang.org server starting")
		*goroot = "_goroot.zip"
		log.SetFlags(log.Lshortfile | log.LstdFlags)
		port := "8080"
		if p := os.Getenv("PORT"); p != "" {
			port = p
		}
		*httpAddr = ":" + port
	}

	flag.Usage = usage
	flag.Parse()

	// Check usage.
	if flag.NArg() > 0 {
		fmt.Fprintln(os.Stderr, "Unexpected arguments.")
		usage()
	}
	if *httpAddr == "" {
		fmt.Fprintln(os.Stderr, "-http must be set")
		usage()
	}

	handler := NewHandler(*contentDir, *goroot)

	if os.Getenv("GODEV_IN_GO_DISCOVERY") != "" {
		// Running in go-discovery for a little longer, do not expect the golang-org prod setup.
		handler = webtest.HandlerWithCheck(handler, "/_readycheck",
			filepath.Join(*contentDir, "../cmd/golangorg/testdata/godev.txt"))
	} else {
		handler = webtest.HandlerWithCheck(handler, "/_readycheck",
			filepath.Join(*contentDir, "../cmd/golangorg/testdata/*.txt"))
	}

	if *verbose {
		log.Printf("golang.org server:")
		log.Printf("\tversion = %s", runtime.Version())
		log.Printf("\taddress = %s", *httpAddr)
		log.Printf("\tgoroot = %s", *goroot)
		handler = loggingHandler(handler)
	}

	// Start http server.
	fmt.Fprintf(os.Stderr, "serving http://%s\n", *httpAddr)
	if err := http.ListenAndServe(*httpAddr, handler); err != nil {
		log.Fatalf("ListenAndServe %s: %v", *httpAddr, err)
	}
}

var isTestBinary = false

// NewHandler returns the http.Handler for the web site,
// given the directory where the content can be found
// (can be "", in which case an internal copy is used)
// and the directory or zip file of the GOROOT.
func NewHandler(contentDir, goroot string) http.Handler {
	mux := http.NewServeMux()

	// Serve files from _content, falling back to GOROOT.
	var content fs.FS
	if contentDir != "" {
		content = osfs.DirFS(contentDir)
	} else {
		content = website.Content
	}

	var gorootFS fs.FS
	if strings.HasSuffix(goroot, ".zip") {
		z, err := zip.OpenReader(goroot)
		if err != nil {
			log.Fatal(err)
		}
		gorootFS = &seekableFS{z}
	} else {
		gorootFS = osfs.DirFS(goroot)
	}

	site, err := newSite(mux, "", content, gorootFS)
	if err != nil {
		log.Fatalf("newSite: %v", err)
	}
	if _, err := newSite(mux, "golang.google.cn", content, gorootFS); err != nil {
		log.Fatalf("newSite: %v", err)
	}

	// tip.golang.org serves content from the very latest Git commit
	// of the main Go repo, instead of the one the app is bundled with.
	var tipGoroot atomicFS
	if _, err := newSite(mux, "tip.golang.org", content, &tipGoroot); err != nil {
		log.Fatalf("loading tip site: %v", err)
	}

	// beta.golang.org is an old name for tip.
	mux.Handle("beta.golang.org/", redirectPrefix("https://tip.golang.org/"))

	// m.golang.org is an old shortcut for golang.org mail.
	// Gmail itself can serve this redirect, but only on HTTP (not HTTPS).
	// Golang.org's HSTS header tells browsers to use HTTPS for all subdomains,
	// which broke the redirect.
	mux.Handle("m.golang.org/", http.RedirectHandler("https://mail.google.com/a/golang.org/", http.StatusMovedPermanently))

	if !isTestBinary {
		go watchTip(&tipGoroot)
	}

	mux.Handle("/compile", playground.Proxy())
	mux.Handle("/fmt", http.HandlerFunc(fmtHandler))
	mux.Handle("/x/", http.HandlerFunc(xHandler))
	redirect.Register(mux)

	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "User-agent: *\nDisallow: /search\n")
	})

	if err := redirect.LoadChangeMap(filepath.Join(contentDir, "../cmd/golangorg/hg-git-mapping.bin")); err != nil {
		log.Fatalf("LoadChangeMap: %v", err)
	}

	if runningOnAppEngine {
		appEngineSetup(site, mux)
	} else {
		// Register a redirect handler for /dl/ to the golang.org download page.
		mux.Handle("/dl/", http.RedirectHandler("https://golang.org/dl/", http.StatusFound))
	}

	godev, err := godevHandler(filepath.Join(contentDir, "../go.dev/_content"))
	if err != nil {
		log.Fatalf("godevHandler: %v", err)
	}
	mux.Handle("go.dev/", godev)
	mux.Handle("learn.go.dev/", godev)

	var h http.Handler = mux
	if env.EnforceHosts() {
		h = hostEnforcerHandler(h)
	}
	h = hostPathHandler(h)
	return h
}

// newSite creates a new site for a given content and goroot file system pair
// and registers it in mux to handle requests for host.
// If host is the empty string, the registrations are for the wildcard host.
func newSite(mux *http.ServeMux, host string, content, goroot fs.FS) (*web.Site, error) {
	fsys := unionFS{content, &fixSpecsFS{goroot}}
	site := web.NewSite(fsys)
	site.Funcs(template.FuncMap{
		"googleAnalytics": func() string { return googleAnalytics },
		"googleCN":        func() bool { return host == "golang.google.cn" },
		"releases":        func() []*history.Major { return history.Majors },
		"version":         func() string { return runtime.Version() },
	})
	docs, err := pkgdoc.NewServer(fsys, site, googleCN)
	if err != nil {
		return nil, err
	}

	mux.Handle(host+"/", site)
	mux.Handle(host+"/cmd/", docs)
	mux.Handle(host+"/pkg/", docs)
	mux.Handle(host+"/doc/codewalk/", codewalk.NewServer(fsys, site))
	return site, nil
}

// watchTip is a background goroutine that watches the main Go repo for updates.
// When a new commit is available, watchTip downloads the new tree and calls
// tipGoroot.Set to install the new file system.
func watchTip(tipGoroot *atomicFS) {
	for {
		// watchTip1 runs until it panics (hopefully never).
		// If that happens, sleep 5 minutes and try again.
		watchTip1(tipGoroot)
		time.Sleep(5 * time.Minute)
	}
}

// watchTip1 does the actual work of watchTip and recovers from panics.
func watchTip1(tipGoroot *atomicFS) {
	defer func() {
		if e := recover(); e != nil {
			log.Printf("watchTip panic: %v\n%s", e, debug.Stack())
		}
	}()

	var r *gitfs.Repo
	for {
		var err error
		r, err = gitfs.NewRepo("https://go.googlesource.com/go")
		if err != nil {
			log.Printf("tip: %v", err)
			time.Sleep(1 * time.Minute)
			continue
		}
		break
	}

	var h gitfs.Hash
	for {
		var fsys fs.FS
		var err error
		h, fsys, err = r.Clone("HEAD")
		if err != nil {
			log.Printf("tip: %v", err)
			time.Sleep(1 * time.Minute)
			continue
		}
		tipGoroot.Set(fsys)
		break
	}

	for {
		time.Sleep(5 * time.Minute)
		h2, err := r.Resolve("HEAD")
		if err != nil {
			log.Printf("tip: %v", err)
			continue
		}
		if h2 != h {
			fsys, err := r.CloneHash(h2)
			if err != nil {
				log.Printf("tip: %v", err)
				time.Sleep(1 * time.Minute)
				continue
			}
			tipGoroot.Set(fsys)
			h = h2
		}
	}
}

func appEngineSetup(site *web.Site, mux *http.ServeMux) {
	googleAnalytics = os.Getenv("GOLANGORG_ANALYTICS")

	log.Printf("GODEV_IN_GO_DISCOVERY %q PROJECT %q", os.Getenv("GODEV_IN_GO_DISCOVERY"), os.Getenv("PROJECT_ID"))
	if os.Getenv("GODEV_IN_GO_DISCOVERY") != "" {
		// Running in go-discovery for a little longer, do not expect the golang-org prod setup.
		return
	}

	ctx := context.Background()

	datastoreClient, err := datastore.NewClient(ctx, "")
	if err != nil {
		if strings.Contains(err.Error(), "missing project") {
			log.Fatalf("Missing datastore project. Set the DATASTORE_PROJECT_ID env variable. Use `gcloud beta emulators datastore` to start a local datastore.")
		}
		log.Fatalf("datastore.NewClient: %v.", err)
	}

	redisAddr := os.Getenv("GOLANGORG_REDIS_ADDR")
	if redisAddr == "" {
		log.Fatalf("Missing redis server for golangorg in production mode. set GOLANGORG_REDIS_ADDR environment variable.")
	}
	memcacheClient := memcache.New(redisAddr)

	dl.RegisterHandlers(mux, site, datastoreClient, memcacheClient)
	short.RegisterHandlers(mux, datastoreClient, memcacheClient)
	proxy.RegisterHandlers(mux, googleCN)

	log.Println("AppEngine initialization complete")
}

type fmtResponse struct {
	Body  string
	Error string
}

// fmtHandler takes a Go program in its "body" form value, formats it with
// standard gofmt formatting, and writes a fmtResponse as a JSON object.
func fmtHandler(w http.ResponseWriter, r *http.Request) {
	resp := new(fmtResponse)
	body, err := format.Source([]byte(r.FormValue("body")))
	if err != nil {
		resp.Error = err.Error()
	} else {
		resp.Body = string(body)
	}
	w.Header().Set("Content-type", "application/json; charset=utf-8")
	json.NewEncoder(w).Encode(resp)
}

var validHosts = map[string]bool{
	"go.dev":           true,
	"learn.go.dev":     true,
	"golang.org":       true,
	"golang.google.cn": true,
	"m.golang.org":     true,
	"tip.golang.org":   true,
	"beta.golang.org":  true,
}

// hostEnforcerHandler redirects http://foo.golang.org/bar to https://golang.org/bar.
// It also forces all requests coming from China to use golang.google.cn.
func hostEnforcerHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		isHTTPS := r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" || r.URL.Scheme == "https"
		defaultHost := "golang.org"
		isValidHost := validHosts[strings.ToLower(r.Host)]

		if googleCN(r) {
			// golang.google.cn is the only web site in China.
			defaultHost = "golang.google.cn"
			isValidHost = strings.ToLower(r.Host) == defaultHost
		}

		if !isHTTPS || !isValidHost {
			r.URL.Scheme = "https"
			if isValidHost {
				r.URL.Host = r.Host
			} else {
				r.URL.Host = defaultHost
			}
			http.Redirect(w, r, r.URL.String(), http.StatusFound)
			return
		}

		w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
		h.ServeHTTP(w, r)
	})
}

// hostPathHandler infers the host from the first element of the URL path
// when the actual host is a testing domain (localhost or *.appspot.com).
// It also rewrites the output HTML in that case to link back to URLs on
// the test site.
func hostPathHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Host != "localhost" && !strings.HasPrefix(r.Host, "localhost:") && !strings.HasSuffix(r.Host, ".appspot.com") {
			h.ServeHTTP(w, r)
			return
		}

		elem, rest := strings.TrimPrefix(r.URL.Path, "/"), ""
		if i := strings.Index(elem, "/"); i >= 0 {
			elem, rest = elem[:i], elem[i+1:]
		}
		if !validHosts[elem] {
			u := "/golang.org" + r.URL.EscapedPath()
			if r.URL.RawQuery != "" {
				u += "?" + r.URL.RawQuery
			}
			http.Redirect(w, r, u, http.StatusTemporaryRedirect)
			return
		}

		r.Host = elem
		r.URL.Scheme = "https"
		r.URL.Host = elem
		r.URL.Path = "/" + rest
		lw := &linkRewriter{ResponseWriter: w, host: r.Host}
		h.ServeHTTP(lw, r)
		lw.Flush()
	})
}

// A linkRewriter is a ResponseWriter that rewrites links in HTML output.
// It rewrites relative links /foo to be /host/foo, and it rewrites any link
// https://h/foo, where h is in validHosts, to be /h/foo. This corrects the
// links to have the right form for the test server.
type linkRewriter struct {
	http.ResponseWriter
	host string
	buf  []byte
	ct   string // content-type
}

func (r *linkRewriter) Write(data []byte) (int, error) {
	if r.ct == "" {
		ct := r.Header().Get("Content-Type")
		if ct == "" {
			// Note: should use first 512 bytes, but first write is fine for our purposes.
			ct = http.DetectContentType(data)
		}
		r.ct = ct
	}
	if !strings.HasPrefix(r.ct, "text/html") {
		return r.ResponseWriter.Write(data)
	}
	r.buf = append(r.buf, data...)
	return len(data), nil
}

func (r *linkRewriter) Flush() {
	repl := []string{
		`href="/`, `href="/` + r.host + `/`,
		`src="/`, `src="/` + r.host + `/`,
	}
	for host := range validHosts {
		repl = append(repl, `href="https://`+host, `href="/`+host)
		repl = append(repl, `src="https://`+host, `src="/`+host)
	}
	strings.NewReplacer(repl...).WriteString(r.ResponseWriter, string(r.buf))
	r.buf = nil
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s\t%s", r.RemoteAddr, r.URL)
		h.ServeHTTP(w, r)
	})
}

func xHandler(w http.ResponseWriter, r *http.Request) {
	if !strings.HasPrefix(r.URL.Path, "/x/") {
		// Shouldn't happen if handler is registered correctly.
		http.Redirect(w, r, "https://pkg.go.dev/search?q=golang.org/x", http.StatusTemporaryRedirect)
		return
	}
	proj, suffix := strings.TrimPrefix(r.URL.Path, "/x/"), ""
	if i := strings.Index(proj, "/"); i != -1 {
		proj, suffix = proj[:i], proj[i:]
	}
	if proj == "" {
		http.Redirect(w, r, "https://pkg.go.dev/search?q=golang.org/x", http.StatusTemporaryRedirect)
		return
	}
	repo, ok := repos.ByGerritProject[proj]
	if !ok || !strings.HasPrefix(repo.ImportPath, "golang.org/x/") {
		http.NotFound(w, r)
		return
	}
	data := struct {
		Proj   string // Gerrit project ("net", "sys", etc)
		Suffix string // optional "/path" for requests like /x/PROJ/path
	}{proj, suffix}
	if err := xTemplate.Execute(w, data); err != nil {
		log.Println("xHandler:", err)
	}
}

var xTemplate = template.Must(template.New("x").Parse(`<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta name="go-import" content="golang.org/x/{{.Proj}} git https://go.googlesource.com/{{.Proj}}">
<meta name="go-source" content="golang.org/x/{{.Proj}} https://github.com/golang/{{.Proj}}/ https://github.com/golang/{{.Proj}}/tree/master{/dir} https://github.com/golang/{{.Proj}}/blob/master{/dir}/{file}#L{line}">
<meta http-equiv="refresh" content="0; url=https://pkg.go.dev/golang.org/x/{{.Proj}}{{.Suffix}}">
</head>
<body>
<a href="https://pkg.go.dev/golang.org/x/{{.Proj}}{{.Suffix}}">Redirecting to documentation...</a>
</body>
</html>
`))

var _ fs.ReadDirFS = unionFS{}

// A unionFS is an FS presenting the union of the file systems in the slice.
// If multiple file systems provide a particular file, Open uses the FS listed earlier in the slice.
// If multiple file systems provide a particular directory, ReadDir presents the
// concatenation of all the directories listed in the slice (with duplicates removed).
type unionFS []fs.FS

func (fsys unionFS) Open(name string) (fs.File, error) {
	var errOut error
	for _, sub := range fsys {
		f, err := sub.Open(name)
		if err == nil {
			// Note: Should technically check for directory
			// and return a synthetic directory that merges
			// reads from all the matching directories,
			// but all the directory reads in internal/godoc
			// come from fsys.ReadDir, which does that for us.
			// So we can ignore direct f.ReadDir calls.
			return f, nil
		}
		if errOut == nil {
			errOut = err
		}
	}
	return nil, errOut
}

func (fsys unionFS) ReadDir(name string) ([]fs.DirEntry, error) {
	var all []fs.DirEntry
	var seen map[string]bool // seen[name] is true if name is listed in all; lazily initialized
	var errOut error
	for _, sub := range fsys {
		list, err := fs.ReadDir(sub, name)
		if err != nil {
			errOut = err
		}
		if len(all) == 0 {
			all = append(all, list...)
		} else {
			if seen == nil {
				// Initialize seen only after we get two different directory listings.
				seen = make(map[string]bool)
				for _, d := range all {
					seen[d.Name()] = true
				}
			}
			for _, d := range list {
				name := d.Name()
				if !seen[name] {
					seen[name] = true
					all = append(all, d)
				}
			}
		}
	}
	if len(all) > 0 {
		return all, nil
	}
	return nil, errOut
}

// A fixSpecsFS is an FS mapping /ref/mem.html and /ref/spec.html to
// /doc/go_mem.html and /doc/go_spec.html.
var _ fs.FS = &fixSpecsFS{}

type fixSpecsFS struct {
	fs fs.FS
}

func (fsys fixSpecsFS) Open(name string) (fs.File, error) {
	switch name {
	case "ref/mem.html", "ref/spec.html":
		if f, err := fsys.fs.Open(name); err == nil {
			// Let Go distribution win if they move.
			return f, nil
		}
		// Otherwise fall back to doc/go_*.html
		name = "doc/go_" + strings.TrimPrefix(name, "ref/")
		return fsys.fs.Open(name)

	case "doc/go_mem.html", "doc/go_spec.html":
		data := []byte("<!--{\n\t\"Redirect\": \"/ref/" + strings.TrimPrefix(strings.TrimSuffix(name, ".html"), "doc/go_") + "\"\n}-->\n")
		return &memFile{path.Base(name), bytes.NewReader(data)}, nil
	}

	return fsys.fs.Open(name)
}

// A seekableFS is an FS wrapper that makes every file seekable
// by reading it entirely into memory when it is opened and then
// serving read operations (including seek) from the memory copy.
type seekableFS struct {
	fs fs.FS
}

func (s *seekableFS) Open(name string) (fs.File, error) {
	f, err := s.fs.Open(name)
	if err != nil {
		return nil, err
	}
	info, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, err
	}
	if info.IsDir() {
		return f, nil
	}
	data, err := ioutil.ReadAll(f)
	if err != nil {
		f.Close()
		return nil, err
	}
	var sf seekableFile
	sf.File = f
	sf.Reset(data)
	return &sf, nil
}

// A seekableFile is a fs.File augmented by an in-memory copy of the file data to allow use of Seek.
type seekableFile struct {
	bytes.Reader
	fs.File
}

// Read calls f.Reader.Read.
// Both f.Reader and f.File have Read methods - a conflict - so f inherits neither.
// This method calls the one we want.
func (f *seekableFile) Read(b []byte) (int, error) {
	return f.Reader.Read(b)
}

// A memFile is an fs.File implementation backed by in-memory data.
type memFile struct {
	name string
	*bytes.Reader
}

func (f *memFile) Stat() (fs.FileInfo, error) { return f, nil }
func (f *memFile) Name() string               { return f.name }
func (*memFile) Mode() fs.FileMode            { return 0444 }
func (*memFile) ModTime() time.Time           { return time.Time{} }
func (*memFile) IsDir() bool                  { return false }
func (*memFile) Sys() interface{}             { return nil }
func (*memFile) Close() error                 { return nil }

// An atomicFS is an fs.FS value safe for reading from multiple goroutines
// as well as updating (assigning a different fs.FS to use in future read requests).
type atomicFS struct {
	v atomic.Value
}

// Set sets the file system used by future calls to Open.
func (a *atomicFS) Set(fsys fs.FS) {
	a.v.Store(&fsys)
}

// Open returns fsys.Open(name) where fsys is the file system passed to the most recent call to Set.
// If there has been no call to Set, Open returns an error with text “no file system”.
func (a *atomicFS) Open(name string) (fs.File, error) {
	fsys, _ := a.v.Load().(*fs.FS)
	if fsys == nil {
		return nil, &fs.PathError{Path: name, Op: "open", Err: fmt.Errorf("no file system")}
	}
	return (*fsys).Open(name)
}

func redirectPrefix(prefix string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, strings.TrimSuffix(prefix, "/")+"/"+strings.TrimPrefix(r.URL.Path, "/"), http.StatusMovedPermanently)
	})
}