many: cmd/pkgsite embeds static assets
The `pkgsite` command now embeds all the static assets it needs, so it
need not be told the location of the static/ directory with a
flag. The binary is self-contained and can be placed and invoked from
anywhere.
This required embedding the static/ and third_party/
directories. Since //go:embed cannot reference files outside the
containing package's tree, we had to add trivial Go packages in
static/ and third_party/ to construct `embed.FS`s for those
directories.
Also, the frontend needed to accept `fs.FS` values where it
previously took paths, and `template.TrustedFS` values where it
previously used `template.TrustedSources`. We ended up clumsily
requiring four separate config values:
- A TrustedFS to load templates.
- Two fs.FSs, one for static and one for third_party, to load other
assets.
- The path to the static directory as a string, solely to support
dynamic loading in dev mode.
For golang/go#48410
Change-Id: I9eeb351b1c6f23444b9e65b60f2a1d3905d59ef9
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/359395
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index d13e70e..47d7e09 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -34,7 +34,7 @@
var (
queueName = config.GetEnv("GO_DISCOVERY_FRONTEND_TASK_QUEUE", "")
workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
- _ = flag.String("static", "static", "path to folder containing static files served")
+ staticFlag = flag.String("static", "static", "path to folder containing static files served")
thirdPartyPath = flag.String("third_party", "third_party", "path to folder containing third-party libraries")
devMode = flag.Bool("dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)")
disableCSP = flag.Bool("nocsp", false, "disable Content Security Policy")
@@ -111,12 +111,15 @@
if err != nil {
log.Fatalf(ctx, "vulndbc.NewClient: %v", err)
}
+ staticSource := template.TrustedSourceFromFlag(flag.Lookup("static").Value)
server, err := frontend.NewServer(frontend.ServerConfig{
DataSourceGetter: dsg,
Queue: fetchQueue,
TaskIDChangeInterval: config.TaskIDChangeIntervalFrontend,
- StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
- ThirdPartyPath: *thirdPartyPath,
+ TemplateFS: template.TrustedFSFromTrustedSource(staticSource),
+ StaticFS: os.DirFS(*staticFlag),
+ StaticPath: *staticFlag,
+ ThirdPartyFS: os.DirFS(*thirdPartyPath),
DevMode: *devMode,
AppVersionLabel: cfg.AppVersionLabel(),
GoogleTagManagerID: cfg.GoogleTagManagerID,
diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index 31e55c8..d54a9dc 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -6,29 +6,28 @@
// It runs as a web server and presents the documentation as a
// web page.
//
-// After running `go install ./cmd/pkgsite` from the pkgsite repo root, you can
-// run `pkgsite` from anywhere, but if you don't run it from the pkgsite repo
-// root you must specify the location of the static assets with -static.
+// To install, run `go install ./cmd/pkgsite` from the pkgsite repo root.
//
-// With just -static, pkgsite will serve docs for the module in the current
+// With no arguments, pkgsite will serve docs for the module in the current
// directory, which must have a go.mod file:
//
-// cd ~/repos/cue && pkgsite -static ~/repos/pkgsite/static
+// cd ~/repos/cue && pkgsite
//
// You can also serve docs from your module cache, directly from the proxy
// (it uses the GOPROXY environment variable), or both:
//
-// pkgsite -static ~/repos/pkgsite/static -cache -proxy
+// pkgsite -cache -proxy
//
-// With either -cache or -proxy, it won't look for a module in the current directory.
-// You can still provide modules on the local filesystem by listing their paths:
+// With either -cache or -proxy, pkgsite won't look for a module in the current
+// directory. You can still provide modules on the local filesystem by listing
+// their paths:
//
-// pkgsite -static ~/repos/pkgsite/static -cache -proxy ~/repos/cue some/other/module
+// pkgsite -cache -proxy ~/repos/cue some/other/module
//
// Although standard library packages will work by default, the docs can take a
// while to appear the first time because the Go repo must be cloned and
// processed. If you clone the repo yourself (https://go.googlesource.com/go),
-// provide its location with the -gorepo flag to save a little time.
+// you can provide its location with the -gorepo flag to save a little time.
package main
import (
@@ -51,12 +50,14 @@
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/source"
"golang.org/x/pkgsite/internal/stdlib"
+ "golang.org/x/pkgsite/static"
+ thirdparty "golang.org/x/pkgsite/third_party"
)
const defaultAddr = "localhost:8080" // default webserver address
var (
- _ = flag.String("static", "static", "path to folder containing static files served")
+ staticFlag = flag.String("static", "", "OBSOLETE - DO NOT USE")
gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
useCache = flag.Bool("cache", false, "fetch from the module cache")
@@ -76,6 +77,10 @@
flag.Parse()
ctx := context.Background()
+ if *staticFlag != "" {
+ fmt.Fprintf(os.Stderr, "-static is ignored. It is obsolete and may be removed in a future version.\n")
+ }
+
paths := collectPaths(flag.Args())
if len(paths) == 0 && !*useCache && !*useProxy {
paths = []string{"."}
@@ -118,7 +123,7 @@
server, err := newServer(ctx, paths, *gopathMode, modCacheDir, prox)
if err != nil {
- die("%s\nMaybe you need to provide the location of static assets with -static.", err)
+ die("%s", err)
}
router := http.NewServeMux()
server.Install(router.Handle, nil, nil)
@@ -160,7 +165,9 @@
}.New()
server, err := frontend.NewServer(frontend.ServerConfig{
DataSourceGetter: func(context.Context) internal.DataSource { return lds },
- StaticPath: template.TrustedSourceFromFlag(flag.Lookup("static").Value),
+ TemplateFS: template.TrustedFSFromEmbed(static.FS),
+ StaticFS: static.FS,
+ ThirdPartyFS: thirdparty.FS,
})
if err != nil {
return nil, err
diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go
index a938eb3..9a84ee4 100644
--- a/cmd/pkgsite/main_test.go
+++ b/cmd/pkgsite/main_test.go
@@ -6,7 +6,6 @@
import (
"context"
- "flag"
"net/http"
"net/http/httptest"
"os"
@@ -45,7 +44,6 @@
localModule := repoPath("internal/fetch/testdata/has_go_mod")
cacheDir := repoPath("internal/fetch/testdata/modcache")
- flag.Set("static", repoPath("static"))
testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
prox, teardown := proxytest.SetupTestClient(t, testModules)
defer teardown()
diff --git a/go.sum b/go.sum
index b3f3947..d40b9b9 100644
--- a/go.sum
+++ b/go.sum
@@ -288,7 +288,6 @@
github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7 h1:k+KkMRk8mGOu1xG38StS7dQ+Z6oW1i9n3dgrAVU9Q/E=
github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/safehtml v0.0.2 h1:ZOt2VXg4x24bW0m2jtzAOkhoXV0iM8vNKc0paByCZqM=
github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
github.com/google/safehtml v0.0.3-0.20211026203422-d6f0e11a5516 h1:pSEdbeokt55L2hwtWo6A2k7u5SG08rmw0LhWEyrdWgk=
github.com/google/safehtml v0.0.3-0.20211026203422-d6f0e11a5516/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go
index a5a4b3d..166d8c3 100644
--- a/internal/fetch/fetch_test.go
+++ b/internal/fetch/fetch_test.go
@@ -34,14 +34,14 @@
var testTimeout = 30 * time.Second
var (
- templateSource = template.TrustedSourceFromConstant("../../static/doc")
- testModules []*proxytest.Module
+ templateFS = template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../static"))
+ testModules []*proxytest.Module
)
type fetchFunc func(t *testing.T, withLicenseDetector bool, ctx context.Context, mod *proxytest.Module, fetchVersion string) (*FetchResult, *licenses.Detector)
func TestMain(m *testing.M) {
- dochtml.LoadTemplates(templateSource)
+ dochtml.LoadTemplates(templateFS)
testModules = proxytest.LoadTestModules("../proxy/testdata")
licenses.OmitExceptions = true
os.Exit(m.Run())
diff --git a/internal/fetchdatasource/fetchdatasource_test.go b/internal/fetchdatasource/fetchdatasource_test.go
index efa1894..e433428 100644
--- a/internal/fetchdatasource/fetchdatasource_test.go
+++ b/internal/fetchdatasource/fetchdatasource_test.go
@@ -36,7 +36,7 @@
)
func TestMain(m *testing.M) {
- dochtml.LoadTemplates(template.TrustedSourceFromConstant("../../static/doc"))
+ dochtml.LoadTemplates(template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../static")))
defaultTestModules = proxytest.LoadTestModules("../proxy/testdata")
var cleanup func()
localGetters, cleanup = buildLocalGetters()
diff --git a/internal/frontend/badge.go b/internal/frontend/badge.go
index f0d4877..d385239 100644
--- a/internal/frontend/badge.go
+++ b/internal/frontend/badge.go
@@ -5,7 +5,6 @@
package frontend
import (
- "fmt"
"net/http"
"strings"
)
@@ -23,7 +22,7 @@
func (s *Server) badgeHandler(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/badge/")
if path != "" {
- http.ServeFile(w, r, fmt.Sprintf("%s/frontend/badge/badge.svg", s.staticPath))
+ serveFileFS(w, r, s.staticFS, "frontend/badge/badge.svg")
return
}
diff --git a/internal/frontend/server.go b/internal/frontend/server.go
index e51f901..225aaff 100644
--- a/internal/frontend/server.go
+++ b/internal/frontend/server.go
@@ -14,6 +14,7 @@
"io/fs"
"net/http"
"net/url"
+ "path"
"strings"
"sync"
"time"
@@ -42,10 +43,11 @@
getDataSource func(context.Context) internal.DataSource
queue queue.Queue
taskIDChangeInterval time.Duration
- staticPath template.TrustedSource
- thirdPartyPath string
- templateDir template.TrustedSource
+ templateFS template.TrustedFS
+ staticFS fs.FS
+ thirdPartyFS fs.FS
devMode bool
+ staticPath string // used only for dynamic loading in dev mode
errorPage []byte
appVersionLabel string
googleTagManagerID string
@@ -65,9 +67,11 @@
DataSourceGetter func(context.Context) internal.DataSource
Queue queue.Queue
TaskIDChangeInterval time.Duration
- StaticPath template.TrustedSource
- ThirdPartyPath string
+ TemplateFS template.TrustedFS // for loading templates safely
+ StaticFS fs.FS // for static/ directory
+ ThirdPartyFS fs.FS // for third_party/ directory
DevMode bool
+ StaticPath string // used only for dynamic loading in dev mode
AppVersionLabel string
GoogleTagManagerID string
ServeStats bool
@@ -78,20 +82,19 @@
// NewServer creates a new Server for the given database and template directory.
func NewServer(scfg ServerConfig) (_ *Server, err error) {
defer derrors.Wrap(&err, "NewServer(...)")
- templateDir := template.TrustedSourceJoin(scfg.StaticPath)
- ts, err := parsePageTemplates(templateDir)
+ ts, err := parsePageTemplates(scfg.TemplateFS)
if err != nil {
return nil, fmt.Errorf("error parsing templates: %v", err)
}
- docTemplateDir := template.TrustedSourceJoin(templateDir, template.TrustedSourceFromConstant("doc"))
- dochtml.LoadTemplates(docTemplateDir)
+ dochtml.LoadTemplates(scfg.TemplateFS)
s := &Server{
getDataSource: scfg.DataSourceGetter,
queue: scfg.Queue,
- staticPath: scfg.StaticPath,
- thirdPartyPath: scfg.ThirdPartyPath,
- templateDir: templateDir,
+ templateFS: scfg.TemplateFS,
+ staticFS: scfg.StaticFS,
+ thirdPartyFS: scfg.ThirdPartyFS,
devMode: scfg.DevMode,
+ staticPath: scfg.StaticPath,
templates: ts,
taskIDChangeInterval: scfg.TaskIDChangeInterval,
appVersionLabel: scfg.AppVersionLabel,
@@ -134,10 +137,11 @@
log.Infof(r.Context(), "Request made to %q", r.URL.Path)
}))
handle("/static/", s.staticHandler())
- handle("/third_party/", http.StripPrefix("/third_party", http.FileServer(http.Dir(s.thirdPartyPath))))
+ handle("/third_party/", http.StripPrefix("/third_party", http.FileServer(http.FS(s.thirdPartyFS))))
handle("/favicon.ico", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- http.ServeFile(w, r, fmt.Sprintf("%s/shared/icon/favicon.ico", http.Dir(s.staticPath.String())))
+ serveFileFS(w, r, s.staticFS, "shared/icon/favicon.ico")
}))
+
handle("/sitemap/", http.StripPrefix("/sitemap/", http.FileServer(http.Dir("private/sitemap"))))
handle("/mod/", http.HandlerFunc(s.handleModuleDetailsRedirect))
handle("/pkg/", http.HandlerFunc(s.handlePackageDetailsRedirect))
@@ -539,7 +543,7 @@
s.mu.Lock()
defer s.mu.Unlock()
var err error
- s.templates, err = parsePageTemplates(s.templateDir)
+ s.templates, err = parsePageTemplates(s.templateFS)
if err != nil {
return nil, fmt.Errorf("error parsing templates: %v", err)
}
@@ -584,69 +588,73 @@
return url
}
-// parsePageTemplates parses html templates contained in the given base
-// directory in order to generate a map of Name->*template.Template.
+// parsePageTemplates parses html templates contained in the given filesystem in
+// order to generate a map of Name->*template.Template.
//
// Separate templates are used so that certain contextual functions (e.g.
// templateName) can be bound independently for each page.
//
// Templates in directories prefixed with an underscore are considered helper
// templates and parsed together with the files in each base directory.
-func parsePageTemplates(base template.TrustedSource) (map[string]*template.Template, error) {
- tsc := template.TrustedSourceFromConstant
- join := template.TrustedSourceJoin
-
+func parsePageTemplates(fsys template.TrustedFS) (map[string]*template.Template, error) {
templates := make(map[string]*template.Template)
- htmlSets := [][]template.TrustedSource{
- {tsc("badge")},
- {tsc("error")},
- {tsc("fetch")},
- {tsc("homepage")},
- {tsc("legacy_search")},
- {tsc("license-policy")},
- {tsc("search")},
- {tsc("search-help")},
- {tsc("styleguide")},
- {tsc("subrepo")},
- {tsc("unit/importedby"), tsc("unit")},
- {tsc("unit/imports"), tsc("unit")},
- {tsc("unit/licenses"), tsc("unit")},
- {tsc("unit/main"), tsc("unit")},
- {tsc("unit/versions"), tsc("unit")},
+ htmlSets := [][]string{
+ {"badge"},
+ {"error"},
+ {"fetch"},
+ {"homepage"},
+ {"legacy_search"},
+ {"license-policy"},
+ {"search"},
+ {"search-help"},
+ {"styleguide"},
+ {"subrepo"},
+ {"unit/importedby", "unit"},
+ {"unit/imports", "unit"},
+ {"unit/licenses", "unit"},
+ {"unit/main", "unit"},
+ {"unit/versions", "unit"},
}
for _, set := range htmlSets {
- t, err := template.New("frontend.tmpl").Funcs(templateFuncs).ParseGlobFromTrustedSource(join(base, tsc("frontend/*.tmpl")))
+ t, err := template.New("frontend.tmpl").Funcs(templateFuncs).ParseFS(fsys, "frontend/*.tmpl")
if err != nil {
- return nil, fmt.Errorf("ParseFilesFromTrustedSources: %v", err)
+ return nil, fmt.Errorf("ParseFS: %v", err)
}
- helperGlob := join(base, tsc("shared/*/*.tmpl"))
- if _, err := t.ParseGlobFromTrustedSource(helperGlob); err != nil {
- return nil, fmt.Errorf("ParseGlobFromTrustedSource(%q): %v", helperGlob, err)
+ helperGlob := "shared/*/*.tmpl"
+ if _, err := t.ParseFS(fsys, helperGlob); err != nil {
+ return nil, fmt.Errorf("ParseFS(%q): %v", helperGlob, err)
}
- var files []template.TrustedSource
for _, f := range set {
- if _, err := t.ParseGlobFromTrustedSource(join(base, tsc("frontend"), f, tsc("*.tmpl"))); err != nil {
- return nil, fmt.Errorf("ParseGlobFromTrustedSource(%v): %v", files, err)
+ if _, err := t.ParseFS(fsys, path.Join("frontend", f, "*.tmpl")); err != nil {
+ return nil, fmt.Errorf("ParseFS(%v): %v", f, err)
}
}
- templates[set[0].String()] = t
+ templates[set[0]] = t
}
return templates, nil
}
func (s *Server) staticHandler() http.Handler {
- staticPath := s.staticPath.String()
-
// In dev mode compile TypeScript files into minified JavaScript files
// and rebuild them on file changes.
if s.devMode {
+ if s.staticPath == "" {
+ panic("staticPath is empty in dev mode; cannot rebuild static files")
+ }
ctx := context.Background()
- _, err := static.Build(static.Config{EntryPoint: staticPath + "/frontend", Watch: true, Bundle: true})
+ _, err := static.Build(static.Config{EntryPoint: s.staticPath + "/frontend", Watch: true, Bundle: true})
if err != nil {
log.Error(ctx, err)
}
}
- return http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath)))
+ return http.StripPrefix("/static/", http.FileServer(http.FS(s.staticFS)))
+}
+
+// serveFileFS serves a file from the given filesystem.
+func serveFileFS(w http.ResponseWriter, r *http.Request, fsys fs.FS, name string) {
+ fs := http.FileServer(http.FS(fsys))
+ r.URL.Path = name
+ fs.ServeHTTP(w, r)
}
diff --git a/internal/frontend/server_test.go b/internal/frontend/server_test.go
index 02fc123..f4bd6f4 100644
--- a/internal/frontend/server_test.go
+++ b/internal/frontend/server_test.go
@@ -35,6 +35,8 @@
"golang.org/x/pkgsite/internal/testing/pagecheck"
"golang.org/x/pkgsite/internal/testing/sample"
"golang.org/x/pkgsite/internal/version"
+ "golang.org/x/pkgsite/static"
+ thirdparty "golang.org/x/pkgsite/third_party"
)
const testTimeout = 5 * time.Second
@@ -677,7 +679,7 @@
name: "static",
urlPath: "/static/",
wantStatusCode: http.StatusOK,
- want: in("", hasText("doc"), hasText("frontend"), hasText("markdown.ts"), hasText("shared"), hasText("worker")),
+ want: in("", hasText("doc"), hasText("frontend"), hasText("shared"), hasText("worker")),
},
{
name: "license policy",
@@ -1541,9 +1543,13 @@
DataSourceGetter: func(context.Context) internal.DataSource { return testDB },
Queue: q,
TaskIDChangeInterval: 10 * time.Minute,
- StaticPath: template.TrustedSourceFromConstant("../../static"),
- ThirdPartyPath: "../../third_party",
- AppVersionLabel: "",
+ TemplateFS: template.TrustedFSFromEmbed(static.FS),
+ // Use the embedded FSs here to make sure they're tested.
+ // Integration tests will use the actual directories.
+ StaticFS: static.FS,
+ ThirdPartyFS: thirdparty.FS,
+ StaticPath: "../../static",
+ AppVersionLabel: "",
})
if err != nil {
t.Fatal(err)
@@ -1568,8 +1574,8 @@
func TestCheckTemplates(t *testing.T) {
// Perform additional checks on parsed templates.
- staticPath := template.TrustedSourceFromConstant("../../static")
- templates, err := parsePageTemplates(staticPath)
+ staticFS := template.TrustedFSFromEmbed(static.FS)
+ templates, err := parsePageTemplates(staticFS)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/frontend/styleguide.go b/internal/frontend/styleguide.go
index 557b326..bfd3a97 100644
--- a/internal/frontend/styleguide.go
+++ b/internal/frontend/styleguide.go
@@ -8,9 +8,9 @@
"bytes"
"context"
"html"
- "io/ioutil"
+ "io/fs"
"net/http"
- "os"
+ "path"
"path/filepath"
"strings"
@@ -35,7 +35,7 @@
if !experiment.IsActive(ctx, internal.ExperimentStyleGuide) {
return &serverError{status: http.StatusNotFound}
}
- page, err := styleGuide(ctx, s.staticPath.String())
+ page, err := styleGuide(ctx, s.staticFS)
page.basePage = s.newBasePage(r, "")
page.AllowWideContent = true
page.UseResponsiveLayout = true
@@ -54,18 +54,18 @@
Outline []*Heading
}
-// styleGuide collects the paths to the markdown files in static/shared,
+// styleGuide collects the paths to the markdown files in staticFS,
// renders them into sections for the styleguide, and merges the document
// outlines into a single page outline.
-func styleGuide(ctx context.Context, staticPath string) (_ *styleGuidePage, err error) {
- defer derrors.WrapStack(&err, "styleGuide(%q)", staticPath)
- files, err := markdownFiles(staticPath)
+func styleGuide(ctx context.Context, staticFS fs.FS) (_ *styleGuidePage, err error) {
+ defer derrors.WrapStack(&err, "styleGuide)")
+ files, err := markdownFiles(staticFS)
if err != nil {
return nil, err
}
var sections []*StyleSection
for _, f := range files {
- doc, err := styleSection(ctx, f)
+ doc, err := styleSection(ctx, staticFS, f)
if err != nil {
return nil, err
}
@@ -99,10 +99,10 @@
// styleSection uses goldmark to parse a markdown file and render
// a section of the styleguide.
-func styleSection(ctx context.Context, filename string) (_ *StyleSection, err error) {
+func styleSection(ctx context.Context, fsys fs.FS, filename string) (_ *StyleSection, err error) {
defer derrors.WrapStack(&err, "styleSection(%q)", filename)
var buf bytes.Buffer
- source, err := ioutil.ReadFile(filename)
+ source, err := fs.ReadFile(fsys, filename)
if err != nil {
return nil, err
}
@@ -189,21 +189,16 @@
reg.Register(ast.KindFencedCodeBlock, r.renderFencedCodeBlock)
}
-// markdownFiles walks the static/shared directory and collects
+// markdownFiles walks the shared directory of fsys and collects
// the paths to markdown files.
-func markdownFiles(dir string) ([]string, error) {
+func markdownFiles(fsys fs.FS) ([]string, error) {
var matches []string
- err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ err := fs.WalkDir(fsys, "shared", func(filepath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
- if info.IsDir() {
- return nil
- }
- if matched, err := filepath.Match("*.md", filepath.Base(path)); err != nil {
- return err
- } else if matched {
- matches = append(matches, path)
+ if path.Ext(filepath) == ".md" {
+ matches = append(matches, filepath)
}
return nil
})
diff --git a/internal/godoc/dochtml/dochtml_test.go b/internal/godoc/dochtml/dochtml_test.go
index dead71a..98a3a21 100644
--- a/internal/godoc/dochtml/dochtml_test.go
+++ b/internal/godoc/dochtml/dochtml_test.go
@@ -31,7 +31,7 @@
"golang.org/x/pkgsite/internal/testing/testhelper"
)
-var templateSource = template.TrustedSourceFromConstant("../../../static/doc")
+var templateFS = template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../../static"))
var update = flag.Bool("update", false, "update goldens instead of checking against them")
@@ -42,7 +42,7 @@
}
func TestCheckTemplates(t *testing.T) {
- LoadTemplates(templateSource)
+ LoadTemplates(templateFS)
for _, tm := range []*template.Template{bodyTemplate, outlineTemplate, sidenavTemplate} {
if err := templatecheck.CheckSafe(tm, templateData{}); err != nil {
t.Fatal(err)
@@ -52,7 +52,7 @@
func TestRender(t *testing.T) {
ctx := context.Background()
- LoadTemplates(templateSource)
+ LoadTemplates(templateFS)
fset, d := mustLoadPackage("everydecl")
parts, err := Render(ctx, fset, d, testRenderOptions)
if err != nil {
@@ -114,7 +114,7 @@
}
func TestExampleRender(t *testing.T) {
- LoadTemplates(templateSource)
+ LoadTemplates(templateFS)
ctx := context.Background()
fset, d := mustLoadPackage("example_test")
diff --git a/internal/godoc/dochtml/symbol_test.go b/internal/godoc/dochtml/symbol_test.go
index 4e43cbd..6f9c558 100644
--- a/internal/godoc/dochtml/symbol_test.go
+++ b/internal/godoc/dochtml/symbol_test.go
@@ -12,7 +12,7 @@
)
func TestGetSymbols(t *testing.T) {
- LoadTemplates(templateSource)
+ LoadTemplates(templateFS)
fset,
d := mustLoadPackage("symbols")
got,
diff --git a/internal/godoc/dochtml/template.go b/internal/godoc/dochtml/template.go
index ca8c964..fcfad0c 100644
--- a/internal/godoc/dochtml/template.go
+++ b/internal/godoc/dochtml/template.go
@@ -6,6 +6,7 @@
import (
"context"
+ "path"
"reflect"
"sync"
@@ -26,29 +27,27 @@
)
// LoadTemplates reads and parses the templates used to generate documentation.
-func LoadTemplates(dir template.TrustedSource) {
+func LoadTemplates(fsys template.TrustedFS) {
+ const dir = "doc"
loadOnce.Do(func() {
- join := template.TrustedSourceJoin
- tc := template.TrustedSourceFromConstant
-
bodyTemplate = template.Must(template.New("body.tmpl").
Funcs(tmpl).
- ParseFilesFromTrustedSources(
- join(dir, tc("body.tmpl")),
- join(dir, tc("declaration.tmpl")),
- join(dir, tc("example.tmpl"))))
+ ParseFS(fsys,
+ path.Join(dir, "body.tmpl"),
+ path.Join(dir, "declaration.tmpl"),
+ path.Join(dir, "example.tmpl")))
if experiment.IsActive(context.Background(), internal.ExperimentNewUnitLayout) {
outlineTemplate = template.Must(template.New("outline.tmpl").
Funcs(tmpl).
- ParseFilesFromTrustedSources(join(dir, tc("outline.tmpl"))))
+ ParseFS(fsys, path.Join(dir, "outline.tmpl")))
} else {
outlineTemplate = template.Must(template.New("legacy-outline.tmpl").
Funcs(tmpl).
- ParseFilesFromTrustedSources(join(dir, tc("legacy-outline.tmpl"))))
+ ParseFS(fsys, path.Join(dir, "legacy-outline.tmpl")))
}
sidenavTemplate = template.Must(template.New("sidenav-mobile.tmpl").
Funcs(tmpl).
- ParseFilesFromTrustedSources(join(dir, tc("sidenav-mobile.tmpl"))))
+ ParseFS(fsys, path.Join(dir, "sidenav-mobile.tmpl")))
})
}
diff --git a/internal/godoc/render_test.go b/internal/godoc/render_test.go
index d2405e7..ea56a4e 100644
--- a/internal/godoc/render_test.go
+++ b/internal/godoc/render_test.go
@@ -18,7 +18,7 @@
"golang.org/x/pkgsite/internal/testing/htmlcheck"
)
-var templateSource = template.TrustedSourceFromConstant("../../static/doc")
+var templateFS = template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../static"))
var (
in = htmlcheck.In
@@ -26,7 +26,7 @@
)
func TestDocInfo(t *testing.T) {
- dochtml.LoadTemplates(templateSource)
+ dochtml.LoadTemplates(templateFS)
ctx := context.Background()
si := source.NewGitHubInfo("a.com/M", "", "abcde")
mi := &ModuleInfo{
@@ -85,7 +85,7 @@
}
func TestRenderParts_SinceVersion(t *testing.T) {
- dochtml.LoadTemplates(templateSource)
+ dochtml.LoadTemplates(templateFS)
ctx := context.Background()
si := source.NewGitHubInfo("a.com/M", "", "abcde")
mi := &ModuleInfo{
diff --git a/internal/testing/integration/frontend_test.go b/internal/testing/integration/frontend_test.go
index ce4eafd..3ac9a17 100644
--- a/internal/testing/integration/frontend_test.go
+++ b/internal/testing/integration/frontend_test.go
@@ -8,6 +8,7 @@
"context"
"net/http"
"net/http/httptest"
+ "os"
"testing"
"time"
@@ -30,11 +31,13 @@
// duplication
func setupFrontend(ctx context.Context, t *testing.T, q queue.Queue, rc *redis.Client) *httptest.Server {
t.Helper()
+ const staticDir = "../../../static"
s, err := frontend.NewServer(frontend.ServerConfig{
DataSourceGetter: func(context.Context) internal.DataSource { return testDB },
TaskIDChangeInterval: 10 * time.Minute,
- StaticPath: template.TrustedSourceFromConstant("../../../static"),
- ThirdPartyPath: "../../../third_party",
+ TemplateFS: template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant(staticDir)),
+ StaticFS: os.DirFS(staticDir),
+ ThirdPartyFS: os.DirFS("../../../third_party"),
AppVersionLabel: "",
Queue: q,
ServeStats: true,
diff --git a/internal/testing/integration/integration_test.go b/internal/testing/integration/integration_test.go
index 4a5be8f..b414227 100644
--- a/internal/testing/integration/integration_test.go
+++ b/internal/testing/integration/integration_test.go
@@ -32,7 +32,7 @@
)
func TestMain(m *testing.M) {
- dochtml.LoadTemplates(template.TrustedSourceFromConstant("../../../static/doc"))
+ dochtml.LoadTemplates(template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../../static")))
testModules = proxytest.LoadTestModules("../../proxy/testdata")
postgres.RunDBTests("discovery_integration_test", m, &testDB)
}
diff --git a/internal/worker/server.go b/internal/worker/server.go
index 8051007..354b8ed 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -87,7 +87,9 @@
if err != nil {
return nil, err
}
- dochtml.LoadTemplates(template.TrustedSourceJoin(scfg.StaticPath, template.TrustedSourceFromConstant("doc")))
+ ts := template.TrustedSourceJoin(scfg.StaticPath, template.TrustedSourceFromConstant("doc"))
+ tfs := template.TrustedFSFromTrustedSource(ts)
+ dochtml.LoadTemplates(tfs)
templates := map[string]*template.Template{
indexTemplate: t1,
versionsTemplate: t2,
diff --git a/internal/worker/server_test.go b/internal/worker/server_test.go
index 6708806..96a01b3 100644
--- a/internal/worker/server_test.go
+++ b/internal/worker/server_test.go
@@ -40,7 +40,7 @@
func TestMain(m *testing.M) {
httpClient = &http.Client{Transport: fakeTransport{}}
- dochtml.LoadTemplates(template.TrustedSourceFromConstant("../../static/doc"))
+ dochtml.LoadTemplates(template.TrustedFSFromTrustedSource(template.TrustedSourceFromConstant("../../static")))
testModules = proxytest.LoadTestModules("../proxy/testdata")
postgres.RunDBTests("discovery_worker_test", m, &testDB)
}
diff --git a/static/fs.go b/static/fs.go
new file mode 100644
index 0000000..72e6a87
--- /dev/null
+++ b/static/fs.go
@@ -0,0 +1,14 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package static
+
+import "embed"
+
+// Until https://golang.org/issue/43854 is implemented, all directories
+// containing files beginning with an underscore must be specified explicitly.
+
+//go:embed doc/* frontend/* shared/* worker/*
+//go:embed frontend/unit/* frontend/unit/main/* frontend/legacy_search/*
+var FS embed.FS
diff --git a/static/fs_test.go b/static/fs_test.go
new file mode 100644
index 0000000..0bed26c
--- /dev/null
+++ b/static/fs_test.go
@@ -0,0 +1,22 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package static
+
+import "testing"
+
+func TestFS(t *testing.T) {
+ for _, f := range []string{
+ "shared/reset.css",
+ "frontend/_modals.tmpl",
+ "frontend/homepage/homepage.tmpl",
+ "shared/logo/go-blue.svg",
+ "frontend/unit/unit.css",
+ "frontend/unit/_header.tmpl",
+ } {
+ if _, err := FS.Open(f); err != nil {
+ t.Errorf("%s: %v", f, err)
+ }
+ }
+}
diff --git a/third_party/README.md b/third_party/README.md
new file mode 100644
index 0000000..7938213
--- /dev/null
+++ b/third_party/README.md
@@ -0,0 +1,7 @@
+This directory holds non-Go libraries that were not developed
+as part of the pkgsite project.
+
+Some of the libraries are used by the frontend UI; others are for testing.
+
+To add a library here, place it in a subdirectory. If the frontend UI needs it,
+then add the subdirectory to the `//go:embed` line in `fs.go`.
diff --git a/third_party/fs.go b/third_party/fs.go
new file mode 100644
index 0000000..57dfbc8
--- /dev/null
+++ b/third_party/fs.go
@@ -0,0 +1,10 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package thirdparty
+
+import "embed"
+
+//go:embed autoComplete.js/* dialog-polyfill/*
+var FS embed.FS
diff --git a/third_party/fs_test.go b/third_party/fs_test.go
new file mode 100644
index 0000000..21de480
--- /dev/null
+++ b/third_party/fs_test.go
@@ -0,0 +1,18 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package thirdparty
+
+import "testing"
+
+func TestFS(t *testing.T) {
+ for _, f := range []string{
+ "autoComplete.js/autoComplete.min.js",
+ "dialog-polyfill/dialog-polyfill.js",
+ } {
+ if _, err := FS.Open(f); err != nil {
+ t.Errorf("%s: %v", f, err)
+ }
+ }
+}
