| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // Golangorg serves the golang.org web sites. |
| package main |
| |
| import ( |
| "bytes" |
| "context" |
| "encoding/json" |
| "flag" |
| "fmt" |
| "go/format" |
| "io" |
| "io/ioutil" |
| "log" |
| "net/http" |
| "os" |
| "path/filepath" |
| "runtime" |
| "strings" |
| |
| "cloud.google.com/go/datastore" |
| "golang.org/x/build/repos" |
| "golang.org/x/website" |
| "golang.org/x/website/internal/backport/archive/zip" |
| "golang.org/x/website/internal/backport/html/template" |
| "golang.org/x/website/internal/backport/io/fs" |
| "golang.org/x/website/internal/backport/osfs" |
| "golang.org/x/website/internal/codewalk" |
| "golang.org/x/website/internal/dl" |
| "golang.org/x/website/internal/env" |
| "golang.org/x/website/internal/memcache" |
| "golang.org/x/website/internal/proxy" |
| "golang.org/x/website/internal/redirect" |
| "golang.org/x/website/internal/short" |
| "golang.org/x/website/internal/web" |
| "golang.org/x/website/internal/webtest" |
| |
| // Registers "/compile" handler that redirects to play.golang.org/compile. |
| // If we are in prod we will register "golang.org/compile" separately, |
| // which will get used instead. |
| _ "golang.org/x/tools/playground" |
| ) |
| |
| var ( |
| httpAddr = flag.String("http", "localhost:6060", "HTTP service address") |
| verbose = flag.Bool("v", false, "verbose mode") |
| goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") |
| contentDir = flag.String("content", "", "path to _content directory") |
| |
| runningOnAppEngine = os.Getenv("PORT") != "" |
| ) |
| |
| func usage() { |
| fmt.Fprintf(os.Stderr, "usage: golangorg\n") |
| flag.PrintDefaults() |
| os.Exit(2) |
| } |
| |
| func main() { |
| if *contentDir == "" { |
| repoRoot := "../.." |
| if _, err := os.Stat("_content"); err == nil { |
| repoRoot = "." |
| } |
| *contentDir = filepath.Join(repoRoot, "_content") |
| } |
| |
| if runningOnAppEngine { |
| log.Print("golang.org server starting") |
| *goroot = "_goroot.zip" |
| log.SetFlags(log.Lshortfile | log.LstdFlags) |
| port := "8080" |
| if p := os.Getenv("PORT"); p != "" { |
| port = p |
| } |
| *httpAddr = ":" + port |
| } |
| |
| flag.Usage = usage |
| flag.Parse() |
| |
| // Check usage. |
| if flag.NArg() > 0 { |
| fmt.Fprintln(os.Stderr, "Unexpected arguments.") |
| usage() |
| } |
| if *httpAddr == "" { |
| fmt.Fprintln(os.Stderr, "-http must be set") |
| usage() |
| } |
| |
| handler := NewHandler(*contentDir, *goroot) |
| |
| handler = webtest.HandlerWithCheck(handler, "/_readycheck", |
| filepath.Join(*contentDir, "../cmd/golangorg/testdata/*.txt")) |
| |
| if *verbose { |
| log.Printf("golang.org server:") |
| log.Printf("\tversion = %s", runtime.Version()) |
| log.Printf("\taddress = %s", *httpAddr) |
| log.Printf("\tgoroot = %s", *goroot) |
| handler = loggingHandler(handler) |
| } |
| |
| // Start http server. |
| fmt.Fprintf(os.Stderr, "serving http://%s\n", *httpAddr) |
| if err := http.ListenAndServe(*httpAddr, handler); err != nil { |
| log.Fatalf("ListenAndServe %s: %v", *httpAddr, err) |
| } |
| } |
| |
| // NewHandler returns the http.Handler for the web site, |
| // given the directory where the content can be found |
| // (can be "", in which case an internal copy is used) |
| // and the directory or zip file of the GOROOT. |
| func NewHandler(contentDir, goroot string) http.Handler { |
| // Serve files from _content, falling back to GOROOT. |
| var content fs.FS |
| if contentDir != "" { |
| content = osfs.DirFS(contentDir) |
| } else { |
| content = website.Content |
| } |
| |
| var gorootFS fs.FS |
| if strings.HasSuffix(goroot, ".zip") { |
| z, err := zip.OpenReader(goroot) |
| if err != nil { |
| log.Fatal(err) |
| } |
| gorootFS = &seekableFS{z} |
| } else { |
| gorootFS = osfs.DirFS(goroot) |
| } |
| fsys := unionFS{content, gorootFS} |
| |
| site, err := web.NewSite(fsys) |
| if err != nil { |
| log.Fatalf("NewSite: %v", err) |
| } |
| |
| mux := http.NewServeMux() |
| mux.Handle("/", site) |
| mux.Handle("/doc/codewalk/", codewalk.NewServer(fsys, site)) |
| mux.Handle("/fmt", http.HandlerFunc(fmtHandler)) |
| mux.Handle("/x/", http.HandlerFunc(xHandler)) |
| redirect.Register(mux) |
| |
| mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { |
| io.WriteString(w, "User-agent: *\nDisallow: /search\n") |
| }) |
| |
| if err := redirect.LoadChangeMap(filepath.Join(contentDir, "../cmd/golangorg/hg-git-mapping.bin")); err != nil { |
| log.Fatalf("LoadChangeMap: %v", err) |
| } |
| |
| if runningOnAppEngine { |
| appEngineSetup(site, mux) |
| } else { |
| // Register a redirect handler for /dl/ to the golang.org download page. |
| mux.Handle("/dl/", http.RedirectHandler("https://golang.org/dl/", http.StatusFound)) |
| } |
| |
| var h http.Handler = mux |
| if env.EnforceHosts() { |
| h = hostEnforcerHandler(h) |
| } |
| h = hostPathHandler(h) |
| return h |
| } |
| |
| func appEngineSetup(site *web.Site, mux *http.ServeMux) { |
| site.GoogleAnalytics = os.Getenv("GOLANGORG_ANALYTICS") |
| |
| ctx := context.Background() |
| |
| datastoreClient, err := datastore.NewClient(ctx, "") |
| if err != nil { |
| if strings.Contains(err.Error(), "missing project") { |
| log.Fatalf("Missing datastore project. Set the DATASTORE_PROJECT_ID env variable. Use `gcloud beta emulators datastore` to start a local datastore.") |
| } |
| log.Fatalf("datastore.NewClient: %v.", err) |
| } |
| |
| redisAddr := os.Getenv("GOLANGORG_REDIS_ADDR") |
| if redisAddr == "" { |
| log.Fatalf("Missing redis server for golangorg in production mode. set GOLANGORG_REDIS_ADDR environment variable.") |
| } |
| memcacheClient := memcache.New(redisAddr) |
| |
| dl.RegisterHandlers(mux, site, datastoreClient, memcacheClient) |
| short.RegisterHandlers(mux, datastoreClient, memcacheClient) |
| proxy.RegisterHandlers(mux) |
| |
| log.Println("AppEngine initialization complete") |
| } |
| |
| func blogHandler(w http.ResponseWriter, r *http.Request) { |
| http.Redirect(w, r, "https://blog.golang.org"+strings.TrimPrefix(r.URL.Path, "/blog"), http.StatusFound) |
| } |
| |
| type fmtResponse struct { |
| Body string |
| Error string |
| } |
| |
| // fmtHandler takes a Go program in its "body" form value, formats it with |
| // standard gofmt formatting, and writes a fmtResponse as a JSON object. |
| func fmtHandler(w http.ResponseWriter, r *http.Request) { |
| resp := new(fmtResponse) |
| body, err := format.Source([]byte(r.FormValue("body"))) |
| if err != nil { |
| resp.Error = err.Error() |
| } else { |
| resp.Body = string(body) |
| } |
| w.Header().Set("Content-type", "application/json; charset=utf-8") |
| json.NewEncoder(w).Encode(resp) |
| } |
| |
| var validHosts = map[string]bool{ |
| "golang.org": true, |
| "golang.google.cn": true, |
| } |
| |
| // hostEnforcerHandler redirects http://foo.golang.org/bar to https://golang.org/bar. |
| // It permits golang.google.cn as an alias for golang.org, for use in China. |
| func hostEnforcerHandler(h http.Handler) http.Handler { |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| isHTTPS := r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" || r.URL.Scheme == "https" |
| isValidHost := validHosts[strings.ToLower(r.Host)] |
| |
| if !isHTTPS || !isValidHost { |
| r.URL.Scheme = "https" |
| if isValidHost { |
| r.URL.Host = r.Host |
| } else { |
| r.URL.Host = "golang.org" |
| } |
| http.Redirect(w, r, r.URL.String(), http.StatusFound) |
| return |
| } |
| w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload") |
| h.ServeHTTP(w, r) |
| }) |
| } |
| |
| // hostPathHandler infers the host from the first element of the URL path |
| // when the actual host is a testing domain (localhost or *.appspot.com). |
| // It also rewrites the output HTML in that case to link back to URLs on |
| // the test site. |
| func hostPathHandler(h http.Handler) http.Handler { |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| if r.Host != "localhost" && !strings.HasPrefix(r.Host, "localhost:") && !strings.HasSuffix(r.Host, ".appspot.com") { |
| h.ServeHTTP(w, r) |
| return |
| } |
| |
| elem, rest := strings.TrimPrefix(r.URL.Path, "/"), "" |
| if i := strings.Index(elem, "/"); i >= 0 { |
| elem, rest = elem[:i], elem[i+1:] |
| } |
| if !validHosts[elem] { |
| u := "/golang.org" + r.URL.EscapedPath() |
| if r.URL.RawQuery != "" { |
| u += "?" + r.URL.RawQuery |
| } |
| http.Redirect(w, r, u, http.StatusTemporaryRedirect) |
| return |
| } |
| |
| r.Host = elem |
| r.URL.Scheme = "https" |
| r.URL.Host = elem |
| r.URL.Path = "/" + rest |
| lw := &linkRewriter{ResponseWriter: w, host: r.Host} |
| h.ServeHTTP(lw, r) |
| lw.Flush() |
| }) |
| } |
| |
| // A linkRewriter is a ResponseWriter that rewrites links in HTML output. |
| // It rewrites relative links /foo to be /host/foo, and it rewrites any link |
| // https://h/foo, where h is in validHosts, to be /h/foo. This corrects the |
| // links to have the right form for the test server. |
| type linkRewriter struct { |
| http.ResponseWriter |
| host string |
| buf []byte |
| ct string // content-type |
| } |
| |
| func (r *linkRewriter) Write(data []byte) (int, error) { |
| if r.ct == "" { |
| ct := r.Header().Get("Content-Type") |
| if ct == "" { |
| // Note: should use first 512 bytes, but first write is fine for our purposes. |
| ct = http.DetectContentType(data) |
| } |
| r.ct = ct |
| } |
| if !strings.HasPrefix(r.ct, "text/html") { |
| return r.ResponseWriter.Write(data) |
| } |
| r.buf = append(r.buf, data...) |
| return len(data), nil |
| } |
| |
| func (r *linkRewriter) Flush() { |
| repl := []string{ |
| `href="/`, `href="/` + r.host + `/`, |
| } |
| for host := range validHosts { |
| repl = append(repl, `href="https://`+host, `href="/`+host) |
| } |
| strings.NewReplacer(repl...).WriteString(r.ResponseWriter, string(r.buf)) |
| r.buf = nil |
| } |
| |
| func loggingHandler(h http.Handler) http.Handler { |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| log.Printf("%s\t%s", r.RemoteAddr, r.URL) |
| h.ServeHTTP(w, r) |
| }) |
| } |
| |
| func xHandler(w http.ResponseWriter, r *http.Request) { |
| if !strings.HasPrefix(r.URL.Path, "/x/") { |
| // Shouldn't happen if handler is registered correctly. |
| http.Redirect(w, r, "https://pkg.go.dev/search?q=golang.org/x", http.StatusTemporaryRedirect) |
| return |
| } |
| proj, suffix := strings.TrimPrefix(r.URL.Path, "/x/"), "" |
| if i := strings.Index(proj, "/"); i != -1 { |
| proj, suffix = proj[:i], proj[i:] |
| } |
| if proj == "" { |
| http.Redirect(w, r, "https://pkg.go.dev/search?q=golang.org/x", http.StatusTemporaryRedirect) |
| return |
| } |
| repo, ok := repos.ByGerritProject[proj] |
| if !ok || !strings.HasPrefix(repo.ImportPath, "golang.org/x/") { |
| http.NotFound(w, r) |
| return |
| } |
| data := struct { |
| Proj string // Gerrit project ("net", "sys", etc) |
| Suffix string // optional "/path" for requests like /x/PROJ/path |
| }{proj, suffix} |
| if err := xTemplate.Execute(w, data); err != nil { |
| log.Println("xHandler:", err) |
| } |
| } |
| |
| var xTemplate = template.Must(template.New("x").Parse(`<!DOCTYPE html> |
| <html> |
| <head> |
| <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> |
| <meta name="go-import" content="golang.org/x/{{.Proj}} git https://go.googlesource.com/{{.Proj}}"> |
| <meta name="go-source" content="golang.org/x/{{.Proj}} https://github.com/golang/{{.Proj}}/ https://github.com/golang/{{.Proj}}/tree/master{/dir} https://github.com/golang/{{.Proj}}/blob/master{/dir}/{file}#L{line}"> |
| <meta http-equiv="refresh" content="0; url=https://pkg.go.dev/golang.org/x/{{.Proj}}{{.Suffix}}"> |
| </head> |
| <body> |
| <a href="https://pkg.go.dev/golang.org/x/{{.Proj}}{{.Suffix}}">Redirecting to documentation...</a> |
| </body> |
| </html> |
| `)) |
| |
| var _ fs.ReadDirFS = unionFS{} |
| |
| // A unionFS is an FS presenting the union of the file systems in the slice. |
| // If multiple file systems provide a particular file, Open uses the FS listed earlier in the slice. |
| // If multiple file systems provide a particular directory, ReadDir presents the |
| // concatenation of all the directories listed in the slice (with duplicates removed). |
| type unionFS []fs.FS |
| |
| func (fsys unionFS) Open(name string) (fs.File, error) { |
| var errOut error |
| for _, sub := range fsys { |
| f, err := sub.Open(name) |
| if err == nil { |
| // Note: Should technically check for directory |
| // and return a synthetic directory that merges |
| // reads from all the matching directories, |
| // but all the directory reads in internal/godoc |
| // come from fsys.ReadDir, which does that for us. |
| // So we can ignore direct f.ReadDir calls. |
| return f, nil |
| } |
| if errOut == nil { |
| errOut = err |
| } |
| } |
| return nil, errOut |
| } |
| |
| func (fsys unionFS) ReadDir(name string) ([]fs.DirEntry, error) { |
| var all []fs.DirEntry |
| var seen map[string]bool // seen[name] is true if name is listed in all; lazily initialized |
| var errOut error |
| for _, sub := range fsys { |
| list, err := fs.ReadDir(sub, name) |
| if err != nil { |
| errOut = err |
| } |
| if len(all) == 0 { |
| all = append(all, list...) |
| } else { |
| if seen == nil { |
| // Initialize seen only after we get two different directory listings. |
| seen = make(map[string]bool) |
| for _, d := range all { |
| seen[d.Name()] = true |
| } |
| } |
| for _, d := range list { |
| name := d.Name() |
| if !seen[name] { |
| seen[name] = true |
| all = append(all, d) |
| } |
| } |
| } |
| } |
| if len(all) > 0 { |
| return all, nil |
| } |
| return nil, errOut |
| } |
| |
| // A seekableFS is an FS wrapper that makes every file seekable |
| // by reading it entirely into memory when it is opened and then |
| // serving read operations (including seek) from the memory copy. |
| type seekableFS struct { |
| fs fs.FS |
| } |
| |
| func (s *seekableFS) Open(name string) (fs.File, error) { |
| f, err := s.fs.Open(name) |
| if err != nil { |
| return nil, err |
| } |
| info, err := f.Stat() |
| if err != nil { |
| f.Close() |
| return nil, err |
| } |
| if info.IsDir() { |
| return f, nil |
| } |
| data, err := ioutil.ReadAll(f) |
| if err != nil { |
| f.Close() |
| return nil, err |
| } |
| var sf seekableFile |
| sf.File = f |
| sf.Reset(data) |
| return &sf, nil |
| } |
| |
| // A seekableFile is a fs.File augmented by an in-memory copy of the file data to allow use of Seek. |
| type seekableFile struct { |
| bytes.Reader |
| fs.File |
| } |
| |
| // Read calls f.Reader.Read. |
| // Both f.Reader and f.File have Read methods - a conflict - so f inherits neither. |
| // This method calls the one we want. |
| func (f *seekableFile) Read(b []byte) (int, error) { |
| return f.Reader.Read(b) |
| } |