blob: 7665efa15971bbf55edd2b77eced5a838018d54b [file] [log] [blame]
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Golangorg serves the golang.org web sites.
package main
import (
"archive/zip"
"bytes"
"context"
"embed"
"encoding/json"
"errors"
"flag"
"fmt"
"go/format"
"html/template"
"io"
"io/fs"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"time"
"cloud.google.com/go/datastore"
"golang.org/x/build/repos"
"golang.org/x/website"
"golang.org/x/website/internal/blog"
"golang.org/x/website/internal/codewalk"
"golang.org/x/website/internal/dl"
"golang.org/x/website/internal/gitfs"
"golang.org/x/website/internal/history"
"golang.org/x/website/internal/memcache"
"golang.org/x/website/internal/pkgdoc"
"golang.org/x/website/internal/play"
"golang.org/x/website/internal/redirect"
"golang.org/x/website/internal/short"
"golang.org/x/website/internal/talks"
"golang.org/x/website/internal/tour"
"golang.org/x/website/internal/web"
"golang.org/x/website/internal/webtest"
)
var (
httpAddr = flag.String("http", "localhost:6060", "HTTP service address")
verbose = flag.Bool("v", false, "verbose mode")
goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory")
contentDir = flag.String("content", "", "path to _content directory")
runningOnAppEngine = os.Getenv("PORT") != ""
tipFlag = flag.Bool("tip", runningOnAppEngine, "load git content for tip.golang.org")
googleAnalytics string
)
func usage() {
fmt.Fprintf(os.Stderr, "usage: golangorg\n")
flag.PrintDefaults()
os.Exit(2)
}
func main() {
// Running locally, find the local _content directory,
// so that updates to those files appear on the local dev instance without restarting.
// On App Engine, leave contentDir empty, so we use the embedded copy,
// which is much faster to access than the simulated file system.
if *contentDir == "" && !runningOnAppEngine {
repoRoot := "../.."
if _, err := os.Stat("_content"); err == nil {
repoRoot = "."
}
*contentDir = filepath.Join(repoRoot, "_content")
}
if runningOnAppEngine {
log.Print("golang.org server starting")
*goroot = "_goroot.zip"
log.SetFlags(log.Lshortfile | log.LstdFlags)
port := "8080"
if p := os.Getenv("PORT"); p != "" {
port = p
}
*httpAddr = ":" + port
}
flag.Usage = usage
flag.Parse()
// Check usage.
if flag.NArg() > 0 {
fmt.Fprintln(os.Stderr, "Unexpected arguments.")
usage()
}
if *httpAddr == "" {
fmt.Fprintln(os.Stderr, "-http must be set")
usage()
}
handler := NewHandler(*contentDir, *goroot)
handler = webtest.HandlerWithCheck(handler, "/_readycheck",
testdataFS, "testdata/*.txt")
if *verbose {
log.Printf("golang.org server:")
log.Printf("\tversion = %s", runtime.Version())
log.Printf("\taddress = %s", *httpAddr)
log.Printf("\tgoroot = %s", *goroot)
handler = loggingHandler(handler)
}
// Start http server.
fmt.Fprintf(os.Stderr, "serving http://%s\n", *httpAddr)
if err := http.ListenAndServe(*httpAddr, handler); err != nil {
log.Fatalf("ListenAndServe %s: %v", *httpAddr, err)
}
}
//go:embed testdata
var testdataFS embed.FS
// NewHandler returns the http.Handler for the web site,
// given the directory where the content can be found
// (can be "", in which case an internal copy is used)
// and the directory or zip file of the GOROOT.
func NewHandler(contentDir, goroot string) http.Handler {
mux := http.NewServeMux()
// Serve files from _content, falling back to GOROOT.
// Use explicit contentDir if specified, otherwise embedded copy.
var contentFS fs.FS
if contentDir != "" {
contentFS = os.DirFS(contentDir)
} else {
contentFS = website.Content()
}
var gorootFS fs.FS
if strings.HasSuffix(goroot, ".zip") {
z, err := zip.OpenReader(goroot)
if err != nil {
log.Fatal(err)
}
gorootFS = &seekableFS{z}
} else {
gorootFS = os.DirFS(goroot)
}
// tip.golang.org serves content from the very latest Git commit
// of the main Go repo, instead of the one the app is bundled with.
var tipGoroot atomicFS
if _, err := newSite(mux, "tip.golang.org", contentFS, &tipGoroot); err != nil {
log.Fatalf("loading tip site: %v", err)
}
if *tipFlag {
go watchTip(&tipGoroot)
}
// beta.golang.org is an old name for tip.
mux.Handle("beta.golang.org/", redirectPrefix("https://tip.golang.org/"))
// By default, golang.org/foo redirects to go.dev/foo.
// All the user-facing golang.org subdomains have moved to go.dev subdirectories.
// There are some exceptions below, like for golang.org/x.
mux.Handle("golang.org/", redirectPrefix("https://go.dev/"))
mux.Handle("blog.golang.org/", redirectPrefix("https://go.dev/blog/"))
mux.Handle("learn.go.dev/", redirectPrefix("https://go.dev/learn/"))
mux.Handle("talks.golang.org/", redirectPrefix("https://go.dev/talks/"))
mux.Handle("tour.golang.org/", redirectPrefix("https://go.dev/tour/"))
// Redirect subdirectory-like domains to the actual subdirectories,
// for people whose fingers learn to type go.dev instead of golang.org
// but not the rest of the URL schema change.
// Note that these domains have to be listed in knownHosts below as well.
mux.Handle("blog.go.dev/", redirectPrefix("https://go.dev/blog/"))
mux.Handle("play.go.dev/", redirectPrefix("https://go.dev/play/"))
mux.Handle("talks.go.dev/", redirectPrefix("https://go.dev/talks/"))
mux.Handle("tour.go.dev/", redirectPrefix("https://go.dev/tour/"))
// m.golang.org is an old shortcut for golang.org mail.
// Gmail itself can serve this redirect, but only on HTTP (not HTTPS).
// Golang.org's HSTS header tells browsers to use HTTPS for all subdomains,
// which broke the redirect.
mux.Handle("m.golang.org/", http.RedirectHandler("https://mail.google.com/a/golang.org/", http.StatusMovedPermanently))
// Register a redirect handler for tip.golang.org/dl/ to the golang.org download page.
// (golang.org/dl and golang.google.cn/dl are registered separately.)
mux.Handle("tip.golang.org/dl/", http.RedirectHandler("https://go.dev/dl/", http.StatusFound))
// TODO(rsc): The unionFS is a hack until we move the files in a followup CL.
siteMux := http.NewServeMux()
godevSite, err := newSite(siteMux, "", contentFS, gorootFS)
if err != nil {
log.Fatalf("newSite go.dev: %v", err)
}
chinaSite, err := newSite(siteMux, "golang.google.cn", contentFS, gorootFS)
if err != nil {
log.Fatalf("newSite golang.google.cn: %v", err)
}
if runningOnAppEngine {
appEngineSetup(mux)
}
dl.RegisterHandlers(siteMux, godevSite, "", datastoreClient, memcacheClient)
dl.RegisterHandlers(siteMux, chinaSite, "golang.google.cn", datastoreClient, memcacheClient)
mux.Handle("/", siteMux)
play.RegisterHandlers(mux, godevSite, chinaSite)
mux.Handle("/explore/", http.StripPrefix("/explore/", redirectPrefix("https://pkg.go.dev/")))
if err := blog.RegisterFeeds(mux, "", godevSite); err != nil {
log.Fatalf("blog: %v", err)
}
// Note: Only golang.org/x/, no go.dev/x/.
mux.Handle("golang.org/x/", http.HandlerFunc(xHandler))
mux.Handle("golang.org/toolchain", http.HandlerFunc(toolchainHandler))
redirect.Register(mux)
// Note: Using godevSite (non-China) for global mux registration because there's no sharing in talks.
// Don't need the hassle of two separate registrations for different domains in siteMux.
if err := talks.RegisterHandlers(mux, godevSite, contentFS); err != nil {
log.Fatalf("talks: %v", err)
}
if err := tour.RegisterHandlers(mux); err != nil {
log.Fatalf("tour: %v", err)
}
var h http.Handler = mux
h = addCSP(mux)
h = hostEnforcerHandler(h)
h = hostPathHandler(h)
return h
}
var gorebuild = NewCachedURL("https://gorebuild.storage.googleapis.com/gorebuild.json", 5*time.Minute)
// newSite creates a new site for a given content and goroot file system pair
// and registers it in mux to handle requests for host.
// If host is the empty string, the registrations are for the wildcard host.
func newSite(mux *http.ServeMux, host string, content, goroot fs.FS) (*web.Site, error) {
fsys := unionFS{content, &hideRootMDFS{&fixSpecsFS{goroot}}}
site := web.NewSite(fsys)
site.Funcs(template.FuncMap{
"googleAnalytics": func() string { return googleAnalytics },
"googleCN": func() bool { return host == "golang.google.cn" },
"gorebuild": gorebuild.Get,
"json": jsonUnmarshal,
"newest": newest,
"now": func() time.Time { return time.Now() },
"releases": func() []*history.Major { return history.Majors },
"rfc3339": parseRFC3339,
"section": section,
"version": func() string { return runtime.Version() },
})
docs, err := pkgdoc.NewServer(fsys, site, googleCN)
if err != nil {
return nil, err
}
mux.Handle(host+"/", site)
mux.Handle(host+"/cmd/", docs)
mux.Handle(host+"/pkg/", docs)
mux.Handle(host+"/doc/codewalk/", codewalk.NewServer(fsys, site))
return site, nil
}
func parseRFC3339(s string) (time.Time, error) {
return time.Parse(time.RFC3339, s)
}
// watchTip is a background goroutine that watches the main Go repo for updates.
// When a new commit is available, watchTip downloads the new tree and calls
// tipGoroot.Set to install the new file system.
func watchTip(tipGoroot *atomicFS) {
for {
// watchTip1 runs until it panics (hopefully never).
// If that happens, sleep 5 minutes and try again.
watchTip1(tipGoroot)
time.Sleep(5 * time.Minute)
}
}
// watchTip1 does the actual work of watchTip and recovers from panics.
func watchTip1(tipGoroot *atomicFS) {
defer func() {
if e := recover(); e != nil {
log.Printf("watchTip panic: %v\n%s", e, debug.Stack())
}
}()
var r *gitfs.Repo
for {
var err error
r, err = gitfs.NewRepo("https://go.googlesource.com/go")
if err != nil {
log.Printf("tip: %v", err)
time.Sleep(1 * time.Minute)
continue
}
break
}
var h gitfs.Hash
for {
var fsys fs.FS
var err error
h, fsys, err = r.Clone("HEAD")
if err != nil {
log.Printf("tip: %v", err)
time.Sleep(1 * time.Minute)
continue
}
tipGoroot.Set(fsys)
break
}
for {
time.Sleep(5 * time.Minute)
h2, err := r.Resolve("HEAD")
if err != nil {
log.Printf("tip: %v", err)
continue
}
if h2 != h {
fsys, err := r.CloneHash(h2)
if err != nil {
log.Printf("tip: %v", err)
time.Sleep(1 * time.Minute)
continue
}
tipGoroot.Set(fsys)
h = h2
}
}
}
var datastoreClient *datastore.Client
var memcacheClient *memcache.Client
func appEngineSetup(mux *http.ServeMux) {
googleAnalytics = os.Getenv("GOLANGORG_ANALYTICS")
ctx := context.Background()
var err error
datastoreClient, err = datastore.NewClient(ctx, "")
if err != nil {
if strings.Contains(err.Error(), "missing project") {
log.Fatalf("Missing datastore project. Set the DATASTORE_PROJECT_ID env variable. Use `gcloud beta emulators datastore` to start a local datastore.")
}
log.Fatalf("datastore.NewClient: %v.", err)
}
redisAddr := os.Getenv("GOLANGORG_REDIS_ADDR")
if redisAddr == "" {
log.Fatalf("Missing redis server for golangorg in production mode. set GOLANGORG_REDIS_ADDR environment variable.")
}
memcacheClient = memcache.New(redisAddr)
short.RegisterHandlers(mux, "", datastoreClient, memcacheClient)
log.Println("AppEngine initialization complete")
}
type fmtResponse struct {
Body string
Error string
}
// fmtHandler takes a Go program in its "body" form value, formats it with
// standard gofmt formatting, and writes a fmtResponse as a JSON object.
func fmtHandler(w http.ResponseWriter, r *http.Request) {
resp := new(fmtResponse)
body, err := format.Source([]byte(r.FormValue("body")))
if err != nil {
resp.Error = err.Error()
} else {
resp.Body = string(body)
}
w.Header().Set("Content-type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(resp)
}
var validHosts = map[string]bool{
"golang.org": true,
"golang.google.cn": true,
"beta.golang.org": true,
"blog.golang.org": true,
"m.golang.org": true,
"talks.golang.org": true,
"tip.golang.org": true,
"tour.golang.org": true,
"go.dev": true,
"blog.go.dev": true,
"learn.go.dev": true,
"play.go.dev": true,
"talks.go.dev": true,
"tour.go.dev": true,
}
// hostEnforcerHandler redirects http://foo.golang.org/bar to https://golang.org/bar.
// It also forces all requests coming from China for golang.org to use golang.google.cn.
func hostEnforcerHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
isHTTPS := r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" || r.URL.Scheme == "https"
defaultHost := "go.dev"
host := strings.ToLower(r.Host)
isValidHost := validHosts[host]
if googleCN(r) && !strings.HasSuffix(host, "google.cn") {
// golang.google.cn is the only web site in China.
defaultHost = "golang.google.cn"
isValidHost = strings.ToLower(r.Host) == defaultHost
}
if !isHTTPS || !isValidHost {
r.URL.Scheme = "https"
if isValidHost {
r.URL.Host = r.Host
} else {
r.URL.Host = defaultHost
}
http.Redirect(w, r, r.URL.String(), http.StatusFound)
return
}
w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
h.ServeHTTP(w, r)
})
}
// hostPathHandler infers the host from the first element of the URL path
// when the actual host is a testing domain (localhost or *.appspot.com).
// It also rewrites the output HTML and Location headers in that case to
// link back to URLs on the test site.
func hostPathHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Host != "localhost" && !strings.HasPrefix(r.Host, "localhost:") && !strings.HasSuffix(r.Host, ".appspot.com") {
h.ServeHTTP(w, r)
return
}
elem, rest := strings.TrimPrefix(r.URL.Path, "/"), ""
if i := strings.Index(elem, "/"); i >= 0 {
if elem[:i] == "tour" {
// The Angular router serving /tour/ fails badly when it sees /go.dev/tour/.
// Just take http://localhost/tour/ as meaning /go.dev/tour/ instead of redirecting.
elem, rest = "go.dev", elem
} else {
elem, rest = elem[:i], elem[i+1:]
}
}
if !validHosts[elem] {
u := "/go.dev" + r.URL.EscapedPath()
if r.URL.RawQuery != "" {
u += "?" + r.URL.RawQuery
}
http.Redirect(w, r, u, http.StatusTemporaryRedirect)
return
}
r.Host = elem
r.URL.Scheme = "https"
r.URL.Host = elem
r.URL.Path = "/" + rest
log.Print(r.URL.String())
lw := &linkRewriter{ResponseWriter: w, host: r.Host, tour: strings.HasPrefix(r.URL.Path, "/tour/")}
h.ServeHTTP(lw, r)
lw.Flush()
})
}
// A linkRewriter is a ResponseWriter that rewrites links in HTML output.
// It rewrites relative links /foo to be /host/foo, and it rewrites any link
// https://h/foo, where h is in validHosts, to be /h/foo. This corrects the
// links to have the right form for the test server.
type linkRewriter struct {
http.ResponseWriter
host string
tour bool // is this go.dev/tour/?
buf []byte
ct string // content-type
}
func (r *linkRewriter) WriteHeader(code int) {
loc := r.Header().Get("Location")
delete(r.Header(), "Content-Length") // we might change the content
if strings.HasPrefix(loc, "/") && !strings.HasPrefix(loc, "/tour/") {
r.Header().Set("Location", "/"+r.host+loc)
} else if u, _ := url.Parse(loc); u != nil && validHosts[u.Host] {
r.Header().Set("Location", "/"+u.Host+"/"+strings.TrimPrefix(u.Path, "/")+u.RawQuery)
}
r.ResponseWriter.WriteHeader(code)
}
func (r *linkRewriter) Write(data []byte) (int, error) {
if r.ct == "" {
ct := r.Header().Get("Content-Type")
if ct == "" {
// Note: should use first 512 bytes, but first write is fine for our purposes.
ct = http.DetectContentType(data)
}
r.ct = ct
}
if !strings.HasPrefix(r.ct, "text/html") {
return r.ResponseWriter.Write(data)
}
r.buf = append(r.buf, data...)
return len(data), nil
}
func (r *linkRewriter) Flush() {
var repl []string
if !r.tour {
repl = []string{
`href="/`, `href="/` + r.host + `/`,
`src="/`, `src="/` + r.host + `/`,
}
}
for host := range validHosts {
repl = append(repl, `href="https://`+host, `href="/`+host)
repl = append(repl, `src="https://`+host, `src="/`+host)
}
strings.NewReplacer(repl...).WriteString(r.ResponseWriter, string(r.buf))
r.buf = nil
}
func loggingHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s\t%s", r.RemoteAddr, r.URL)
h.ServeHTTP(w, r)
})
}
func xHandler(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/x/") {
// Shouldn't happen if handler is registered correctly.
http.Redirect(w, r, "https://pkg.go.dev/golang.org/x", http.StatusTemporaryRedirect)
return
}
proj, suffix := strings.TrimPrefix(r.URL.Path, "/x/"), ""
if i := strings.Index(proj, "/"); i != -1 {
proj, suffix = proj[:i], proj[i:]
}
if proj == "" {
http.Redirect(w, r, "https://pkg.go.dev/golang.org/x", http.StatusTemporaryRedirect)
return
}
repo, ok := repos.ByGerritProject[proj]
if !ok || !strings.HasPrefix(repo.ImportPath, "golang.org/x/") {
http.NotFound(w, r)
return
}
if suffix == "/info/refs" && strings.HasPrefix(r.URL.Query().Get("service"), "git-") && repo.GoGerritProject != "" {
// Someone is running 'git clone https://golang.org/x/repo'.
// We want the eventual git checkout to have the right origin (go.googlesource.com)
// and not just keep hitting golang.org all the time.
// A redirect would work for this git command but not record the result.
// Instead, print a useful error for the user.
http.Error(w, fmt.Sprintf("Use 'git clone https://go.googlesource.com/%s' instead.", repo.GoGerritProject), http.StatusNotFound)
return
}
data := struct {
Proj string // Gerrit project ("net", "sys", etc)
Suffix string // optional "/path" for requests like /x/PROJ/path
}{proj, suffix}
if err := xTemplate.Execute(w, data); err != nil {
log.Println("xHandler:", err)
}
}
var xTemplate = template.Must(template.New("x").Parse(`<!DOCTYPE html>
<html lang="en">
<title>The Go Programming Language</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta name="go-import" content="golang.org/x/{{.Proj}} git https://go.googlesource.com/{{.Proj}}">
<meta http-equiv="refresh" content="0; url=https://pkg.go.dev/golang.org/x/{{.Proj}}{{.Suffix}}">
</head>
<body>
<a href="https://pkg.go.dev/golang.org/x/{{.Proj}}{{.Suffix}}">Redirecting to documentation...</a>
</body>
</html>
`))
func toolchainHandler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/toolchain" {
// Shouldn't happen if handler is registered correctly.
http.NotFound(w, r)
return
}
w.Write(toolchainPage)
}
var toolchainPage = []byte(`<!DOCTYPE html>
<html lang="en">
<head>
<title>The Go Programming Language</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta name="go-import" content="golang.org/toolchain mod https://go.dev/dl/mod">
<meta http-equiv="refresh" content="0; url=https://go.dev/dl/">
</head>
<body>
golang.org/toolchain is the module form of the Go toolchain releases.
<a href="https://go.dev/dl/">Redirecting to Go toolchain download page...</a>
</body>
</html>
`)
var _ fs.ReadDirFS = unionFS{}
// A unionFS is an FS presenting the union of the file systems in the slice.
// If multiple file systems provide a particular file, Open uses the FS listed earlier in the slice.
// If multiple file systems provide a particular directory, ReadDir presents the
// concatenation of all the directories listed in the slice (with duplicates removed).
type unionFS []fs.FS
func (fsys unionFS) Open(name string) (fs.File, error) {
var errOut error
for _, sub := range fsys {
f, err := sub.Open(name)
if err == nil {
// Note: Should technically check for directory
// and return a synthetic directory that merges
// reads from all the matching directories,
// but all the directory reads in internal/godoc
// come from fsys.ReadDir, which does that for us.
// So we can ignore direct f.ReadDir calls.
return f, nil
}
if errOut == nil {
errOut = err
}
}
return nil, errOut
}
func (fsys unionFS) ReadDir(name string) ([]fs.DirEntry, error) {
var all []fs.DirEntry
var seen map[string]bool // seen[name] is true if name is listed in all; lazily initialized
var errOut error
for _, sub := range fsys {
list, err := fs.ReadDir(sub, name)
if err != nil {
errOut = err
}
if len(all) == 0 {
all = append(all, list...)
} else {
if seen == nil {
// Initialize seen only after we get two different directory listings.
seen = make(map[string]bool)
for _, d := range all {
seen[d.Name()] = true
}
}
for _, d := range list {
name := d.Name()
if !seen[name] {
seen[name] = true
all = append(all, d)
}
}
}
}
if len(all) > 0 {
return all, nil
}
return nil, errOut
}
// A fixSpecsFS is an FS mapping /ref/mem.html and /ref/spec.html to
// /doc/go_mem.html and /doc/go_spec.html.
var _ fs.FS = &fixSpecsFS{}
type fixSpecsFS struct {
fs fs.FS
}
func (fsys fixSpecsFS) Open(name string) (fs.File, error) {
switch name {
case "ref/mem.html", "ref/spec.html":
if f, err := fsys.fs.Open(name); err == nil {
// Let Go distribution win if they move.
return f, nil
}
// Otherwise fall back to doc/go_*.html
name = "doc/go_" + strings.TrimPrefix(name, "ref/")
return fsys.fs.Open(name)
case "doc/go_mem.html", "doc/go_spec.html":
data := []byte("<!--{\n\t\"Redirect\": \"/ref/" + strings.TrimPrefix(strings.TrimSuffix(name, ".html"), "doc/go_") + "\"\n}-->\n")
return &memFile{path.Base(name), bytes.NewReader(data)}, nil
}
return fsys.fs.Open(name)
}
// A hideRootMDFS is an FS that hides *.md files in the root directory.
// We use this to hide the Go repository's CONTRIBUTING.md,
// README.md, and SECURITY.md. The last is particularly problematic
// when running locally on a Mac, because it can be opened as
// security.md, which takes priority over _content/security.html.
type hideRootMDFS struct {
fs fs.FS
}
func (fsys hideRootMDFS) Open(name string) (fs.File, error) {
if !strings.Contains(name, "/") && strings.HasSuffix(name, ".md") {
return nil, errors.New(".md file not available")
}
return fsys.fs.Open(name)
}
// A seekableFS is an FS wrapper that makes every file seekable
// by reading it entirely into memory when it is opened and then
// serving read operations (including seek) from the memory copy.
type seekableFS struct {
fs fs.FS
}
func (s *seekableFS) Open(name string) (fs.File, error) {
f, err := s.fs.Open(name)
if err != nil {
return nil, err
}
info, err := f.Stat()
if err != nil {
f.Close()
return nil, err
}
if info.IsDir() {
return f, nil
}
data, err := ioutil.ReadAll(f)
if err != nil {
f.Close()
return nil, err
}
var sf seekableFile
sf.File = f
sf.Reset(data)
return &sf, nil
}
// A seekableFile is a fs.File augmented by an in-memory copy of the file data to allow use of Seek.
type seekableFile struct {
bytes.Reader
fs.File
}
// Read calls f.Reader.Read.
// Both f.Reader and f.File have Read methods - a conflict - so f inherits neither.
// This method calls the one we want.
func (f *seekableFile) Read(b []byte) (int, error) {
return f.Reader.Read(b)
}
// A memFile is an fs.File implementation backed by in-memory data.
type memFile struct {
name string
*bytes.Reader
}
func (f *memFile) Stat() (fs.FileInfo, error) { return f, nil }
func (f *memFile) Name() string { return f.name }
func (*memFile) Mode() fs.FileMode { return 0444 }
func (*memFile) ModTime() time.Time { return time.Time{} }
func (*memFile) IsDir() bool { return false }
func (*memFile) Sys() interface{} { return nil }
func (*memFile) Close() error { return nil }
// An atomicFS is an fs.FS value safe for reading from multiple goroutines
// as well as updating (assigning a different fs.FS to use in future read requests).
type atomicFS struct {
v atomic.Value
}
// Set sets the file system used by future calls to Open.
func (a *atomicFS) Set(fsys fs.FS) {
a.v.Store(&fsys)
}
// Open returns fsys.Open(name) where fsys is the file system passed to the most recent call to Set.
// If there has been no call to Set, Open returns an error with text “no file system”.
func (a *atomicFS) Open(name string) (fs.File, error) {
fsys, _ := a.v.Load().(*fs.FS)
if fsys == nil {
return nil, &fs.PathError{Path: name, Op: "open", Err: fmt.Errorf("no file system")}
}
return (*fsys).Open(name)
}
func redirectPrefix(prefix string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := strings.TrimSuffix(prefix, "/") + "/" + strings.TrimPrefix(r.URL.Path, "/")
if r.URL.RawQuery != "" {
url += "?" + r.URL.RawQuery
}
http.Redirect(w, r, url, http.StatusMovedPermanently)
})
}
type CachedURL struct {
url string
timeout time.Duration
mu sync.Mutex
data []byte
err error
etag string
updated time.Time
}
func NewCachedURL(url string, timeout time.Duration) *CachedURL {
return &CachedURL{url: url, timeout: timeout}
}
func (c *CachedURL) Get() (data []byte, err error) {
c.mu.Lock()
defer c.mu.Unlock()
if time.Since(c.updated) < c.timeout {
return c.data, c.err
}
defer func() {
c.updated = time.Now()
c.data, c.err = data, err
}()
cli := &http.Client{Timeout: 60 * time.Second}
req, err := http.NewRequest("GET", c.url, nil)
if err != nil {
return nil, err
}
if c.etag != "" {
req.Header.Set("If-None-Match", c.etag)
}
resp, err := cli.Do(req)
if err != nil {
return nil, fmt.Errorf("loading rebuild report JSON: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotModified {
// Unmodified.
log.Printf("checked %s - unmodified", c.url)
return c.data, c.err
}
log.Printf("reloading %s", c.url)
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("loading rebuild report JSON: %v", resp.Status)
}
c.etag = resp.Header.Get("Etag")
data, err = io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("loading rebuild report JSON: %v", err)
}
return data, nil
}
func jsonUnmarshal(data []byte) (any, error) {
var x any
err := json.Unmarshal(data, &x)
return x, err
}