cmd/golangorg: add ability to serve tip.golang.org directly from Gerrit
Currently, tip.golang.org is deployed by a background loop
that fetches the latest Go and website repos every few minutes
and then does a deploy to a whole separate app.
Because this setup is different from the main app deploy,
it often gets broken by changes in the way the main app runs.
This CL removes the recurring source of breakage by making
the main app capable of serving tip.golang.org directly.
It does this by watching the main Go repo itself and downloading
a new copy of the file tree whenever there are changes.
The website repo is not watched: new changes to the website
repo already result in redeploys of the entire app when appropriate.
This CL does not actually enable the new tip.golang.org code.
A followup CL will do that, for easier rollback.
Change-Id: I015368c614579c90fa72a6699f6ab76202f87e7e
Reviewed-on: https://go-review.googlesource.com/c/website/+/328214
Trust: Russ Cox <rsc@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/cmd/golangorg/app.yaml b/cmd/golangorg/app.yaml
index 9d6401b3..fc1ec5c 100644
--- a/cmd/golangorg/app.yaml
+++ b/cmd/golangorg/app.yaml
@@ -20,3 +20,10 @@
- url: /.*
script: auto
secure: always
+
+# We need ~200 megabytes of RAM for tip.golang.org's GOROOT (including GC overhead).
+# Use F4 instance for 1GB RAM (4X the default size, 4X the default price).
+# Accept 4X the default maximum concurrent requests to balance out.
+instance_class: F4
+automatic_scaling:
+ max_concurrent_requests: 40
diff --git a/cmd/golangorg/server.go b/cmd/golangorg/server.go
index 81b3df1..033a9e6 100644
--- a/cmd/golangorg/server.go
+++ b/cmd/golangorg/server.go
@@ -19,7 +19,10 @@
"os"
"path/filepath"
"runtime"
+ "runtime/debug"
"strings"
+ "sync/atomic"
+ "time"
"cloud.google.com/go/datastore"
"golang.org/x/build/repos"
@@ -32,6 +35,7 @@
"golang.org/x/website/internal/codewalk"
"golang.org/x/website/internal/dl"
"golang.org/x/website/internal/env"
+ "golang.org/x/website/internal/gitfs"
"golang.org/x/website/internal/memcache"
"golang.org/x/website/internal/pkgdoc"
"golang.org/x/website/internal/proxy"
@@ -114,6 +118,8 @@
// (can be "", in which case an internal copy is used)
// and the directory or zip file of the GOROOT.
func NewHandler(contentDir, goroot string) http.Handler {
+ mux := http.NewServeMux()
+
// Serve files from _content, falling back to GOROOT.
var content fs.FS
if contentDir != "" {
@@ -132,25 +138,35 @@
} else {
gorootFS = osfs.DirFS(goroot)
}
- fsys := unionFS{content, gorootFS}
- site, err := web.NewSite(fsys)
+ site, err := newSite(mux, "", content, gorootFS)
if err != nil {
- log.Fatalf("NewSite: %v", err)
+ log.Fatalf("newSite: %v", err)
}
- mux := http.NewServeMux()
- mux.Handle("/", site)
+ // When we are ready to start serving tip.golang.org
+ // from the default golang-org app instead of a separate app,
+ // we can set serveTip = true.
+ const serveTip = false
- docs, err := pkgdoc.NewServer(fsys, site)
- if err != nil {
- log.Fatal(err)
+ if serveTip {
+ // tip.golang.org serves content from the very latest Git commit
+ // of the main Go repo, instead of the one the app is bundled with.
+ var tipGoroot atomicFS
+ if _, err := newSite(mux, "tip.golang.org", content, &tipGoroot); err != nil {
+ log.Fatalf("loading tip site: %v", err)
+ }
+
+ // TODO(rsc): Replace with redirect to tip
+ // once tip is being served by this app.
+ if _, err := newSite(mux, "beta.golang.org", content, &tipGoroot); err != nil {
+ log.Fatalf("loading beta site: %v", err)
+ }
+
+ go watchTip(&tipGoroot)
}
- mux.Handle("/cmd/", docs)
- mux.Handle("/pkg/", docs)
mux.Handle("/compile", playground.Proxy())
- mux.Handle("/doc/codewalk/", codewalk.NewServer(fsys, site))
mux.Handle("/fmt", http.HandlerFunc(fmtHandler))
mux.Handle("/x/", http.HandlerFunc(xHandler))
redirect.Register(mux)
@@ -178,6 +194,93 @@
return h
}
+// newSite creates a new site for a given content and goroot file system pair
+// and registers it in mux to handle requests for host.
+// If host is the empty string, the registrations are for the wildcard host.
+func newSite(mux *http.ServeMux, host string, content, goroot fs.FS) (*web.Site, error) {
+ fsys := unionFS{content, goroot}
+ site, err := web.NewSite(fsys)
+ if err != nil {
+ return nil, err
+ }
+ docs, err := pkgdoc.NewServer(fsys, site)
+ if err != nil {
+ return nil, err
+ }
+
+ mux.Handle(host+"/", site)
+ mux.Handle(host+"/cmd/", docs)
+ mux.Handle(host+"/pkg/", docs)
+ mux.Handle(host+"/doc/codewalk/", codewalk.NewServer(fsys, site))
+ return site, nil
+}
+
+// watchTip is a background goroutine that watches the main Go repo for updates.
+// When a new commit is available, watchTip downloads the new tree and calls
+// tipGoroot.Set to install the new file system.
+func watchTip(tipGoroot *atomicFS) {
+ for {
+ // watchTip1 runs until it panics (hopefully never).
+ // If that happens, sleep 5 minutes and try again.
+ watchTip1(tipGoroot)
+ time.Sleep(5 * time.Minute)
+ }
+}
+
+// watchTip1 does the actual work of watchTip and recovers from panics.
+func watchTip1(tipGoroot *atomicFS) {
+ defer func() {
+ if e := recover(); e != nil {
+ log.Printf("watchTip panic: %v\n%s", e, debug.Stack())
+ }
+ }()
+
+ var r *gitfs.Repo
+ for {
+ var err error
+ r, err = gitfs.NewRepo("https://go.googlesource.com/go")
+ if err != nil {
+ log.Printf("tip: %v", err)
+ time.Sleep(1 * time.Minute)
+ continue
+ }
+ break
+ }
+
+ var h gitfs.Hash
+ for {
+ var fsys fs.FS
+ var err error
+ h, fsys, err = r.Clone("HEAD")
+ if err != nil {
+ log.Printf("tip: %v", err)
+ time.Sleep(1 * time.Minute)
+ continue
+ }
+ tipGoroot.Set(fsys)
+ break
+ }
+
+ for {
+ time.Sleep(5 * time.Minute)
+ h2, err := r.Resolve("HEAD")
+ if err != nil {
+ log.Printf("tip: %v", err)
+ continue
+ }
+ if h2 != h {
+ fsys, err := r.CloneHash(h2)
+ if err != nil {
+ log.Printf("tip: %v", err)
+ time.Sleep(1 * time.Minute)
+ continue
+ }
+ tipGoroot.Set(fsys)
+ h = h2
+ }
+ }
+}
+
func appEngineSetup(site *web.Site, mux *http.ServeMux) {
site.GoogleAnalytics = os.Getenv("GOLANGORG_ANALYTICS")
@@ -231,6 +334,7 @@
"golang.org": true,
"golang.google.cn": true,
"tip.golang.org": true,
+ "beta.golang.org": true,
}
// hostEnforcerHandler redirects http://foo.golang.org/bar to https://golang.org/bar.
@@ -481,3 +585,24 @@
func (f *seekableFile) Read(b []byte) (int, error) {
return f.Reader.Read(b)
}
+
+// An atomicFS is an fs.FS value safe for reading from multiple goroutines
+// as well as updating (assigning a different fs.FS to use in future read requests).
+type atomicFS struct {
+ v atomic.Value
+}
+
+// Set sets the file system used by future calls to Open.
+func (a *atomicFS) Set(fsys fs.FS) {
+ a.v.Store(&fsys)
+}
+
+// Open returns fsys.Open(name) where fsys is the file system passed to the most recent call to Set.
+// If there has been no call to Set, Open returns an error with text “no file system”.
+func (a *atomicFS) Open(name string) (fs.File, error) {
+ fsys, _ := a.v.Load().(*fs.FS)
+ if fsys == nil {
+ return nil, &fs.PathError{Path: name, Op: "open", Err: fmt.Errorf("no file system")}
+ }
+ return (*fsys).Open(name)
+}
diff --git a/internal/gitfs/fs.go b/internal/gitfs/fs.go
index 4c0d4de..3a518bc 100644
--- a/internal/gitfs/fs.go
+++ b/internal/gitfs/fs.go
@@ -11,6 +11,7 @@
"fmt"
hashpkg "hash"
"io"
+ "runtime/debug"
"time"
"golang.org/x/website/internal/backport/io/fs"
@@ -225,7 +226,14 @@
}
// Open opens the given file or directory, implementing the fs.FS Open method.
-func (t *treeFS) Open(name string) (fs.File, error) {
+func (t *treeFS) Open(name string) (f fs.File, err error) {
+ defer func() {
+ if e := recover(); e != nil {
+ f = nil
+ err = fmt.Errorf("gitfs panic: %v\n%s", e, debug.Stack())
+ }
+ }()
+
// Process each element in the slash-separated path, producing hash identified by name.
h := t.tree
start := 0 // index of start of final path element in name
@@ -318,8 +326,14 @@
return 0, f.info.err("seek", fs.ErrInvalid)
}
-func (f *dirFile) ReadDir(n int) ([]fs.DirEntry, error) {
- var list []fs.DirEntry
+func (f *dirFile) ReadDir(n int) (list []fs.DirEntry, err error) {
+ defer func() {
+ if e := recover(); e != nil {
+ list = nil
+ err = fmt.Errorf("gitfs panic: %v\n%s", e, debug.Stack())
+ }
+ }()
+
for (n <= 0 || len(list) < n) && f.off < len(f.data) {
e, size := parseDirEntry(f.data[f.off:])
if size == 0 {
diff --git a/internal/pkgdoc/dir.go b/internal/pkgdoc/dir.go
index d71a0cc..56a2401 100644
--- a/internal/pkgdoc/dir.go
+++ b/internal/pkgdoc/dir.go
@@ -14,7 +14,6 @@
"go/token"
"log"
"path"
- "sort"
"strings"
"golang.org/x/website/internal/backport/io/fs"
@@ -107,6 +106,10 @@
return list
}
+// newDir returns a Dir describing dirpath in fsys.
+// When Go files need to be parsed, newDir uses fset.
+// If there are no package files and no subdirectories containing packages,
+// newDir returns nil.
func newDir(fsys fs.FS, fset *token.FileSet, dirpath string) *Dir {
var synopses [3]string // prioritized package documentation (0 == highest priority)
@@ -120,7 +123,6 @@
}
// determine number of subdirectories and if there are package files
- var dirchs []chan *Dir
var dirs []*Dir
for _, de := range list {
@@ -164,19 +166,6 @@
}
}
- // create subdirectory tree
- for _, ch := range dirchs {
- if d := <-ch; d != nil {
- dirs = append(dirs, d)
- }
- }
-
- // We need to sort the dirs slice because
- // it is appended again after reading from dirchs.
- sort.Slice(dirs, func(i, j int) bool {
- return dirs[i].Path < dirs[j].Path
- })
-
// if there are no package files and no subdirectories
// containing package files, ignore the directory
if !hasPkgFiles && len(dirs) == 0 {
diff --git a/internal/pkgdoc/doc.go b/internal/pkgdoc/doc.go
index b414f29..0ad9abc 100644
--- a/internal/pkgdoc/doc.go
+++ b/internal/pkgdoc/doc.go
@@ -48,10 +48,13 @@
if err != nil {
return nil, err
}
- src := newDir(fsys, token.NewFileSet(), "src")
+ var dirs []*Dir
+ if src := newDir(fsys, token.NewFileSet(), "src"); src != nil {
+ dirs = []*Dir{src}
+ }
root := &Dir{
Path: ".",
- Dirs: []*Dir{src},
+ Dirs: dirs,
}
docs := &docs{
fs: fsys,
