| // Copyright 2022 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // Package vcweb serves version control repos for testing the go command. |
| // |
| // It is loosely derived from golang.org/x/build/vcs-test/vcweb, |
| // which ran as a service hosted at vcs-test.golang.org. |
| // |
| // When a repository URL is first requested, the vcweb [Server] dynamically |
| // regenerates the repository using a script interpreted by a [script.Engine]. |
| // The script produces the server's contents for a corresponding root URL and |
| // all subdirectories of that URL, which are then cached: subsequent requests |
| // for any URL generated by the script will serve the script's previous output |
| // until the script is modified. |
| // |
| // The script engine includes all of the engine's default commands and |
| // conditions, as well as commands for each supported VCS binary (bzr, fossil, |
| // git, hg, and svn), a "handle" command that informs the script which protocol |
| // or handler to use to serve the request, and utilities "at" (which sets |
| // environment variables for Git timestamps) and "unquote" (which unquotes its |
| // argument as if it were a Go string literal). |
| // |
| // The server's "/" endpoint provides a summary of the available scripts, |
| // and "/help" provides documentation for the script environment. |
| // |
| // To run a standalone server based on the vcweb engine, use: |
| // |
| // go test cmd/go/internal/vcweb/vcstest -v --port=0 |
| package vcweb |
| |
| import ( |
| "bufio" |
| "cmd/go/internal/script" |
| "context" |
| "crypto/sha256" |
| "errors" |
| "fmt" |
| "io" |
| "io/fs" |
| "log" |
| "net/http" |
| "os" |
| "os/exec" |
| "path" |
| "path/filepath" |
| "runtime/debug" |
| "strings" |
| "sync" |
| "text/tabwriter" |
| "time" |
| ) |
| |
| // A Server serves cached, dynamically-generated version control repositories. |
| type Server struct { |
| env []string |
| logger *log.Logger |
| |
| scriptDir string |
| workDir string |
| homeDir string // $workdir/home |
| engine *script.Engine |
| |
| scriptCache sync.Map // script path → *scriptResult |
| |
| vcsHandlers map[string]vcsHandler |
| } |
| |
| // A vcsHandler serves repositories over HTTP for a known version-control tool. |
| type vcsHandler interface { |
| Available() bool |
| Handler(dir string, env []string, logger *log.Logger) (http.Handler, error) |
| } |
| |
| // A scriptResult describes the cached result of executing a vcweb script. |
| type scriptResult struct { |
| mu sync.RWMutex |
| |
| hash [sha256.Size]byte // hash of the script file, for cache invalidation |
| hashTime time.Time // timestamp at which the script was run, for diagnostics |
| |
| handler http.Handler // HTTP handler configured by the script |
| err error // error from executing the script, if any |
| } |
| |
| // NewServer returns a Server that generates and serves repositories in workDir |
| // using the scripts found in scriptDir and its subdirectories. |
| // |
| // A request for the path /foo/bar/baz will be handled by the first script along |
| // that path that exists: $scriptDir/foo.txt, $scriptDir/foo/bar.txt, or |
| // $scriptDir/foo/bar/baz.txt. |
| func NewServer(scriptDir, workDir string, logger *log.Logger) (*Server, error) { |
| if scriptDir == "" { |
| panic("vcweb.NewServer: scriptDir is required") |
| } |
| var err error |
| scriptDir, err = filepath.Abs(scriptDir) |
| if err != nil { |
| return nil, err |
| } |
| |
| if workDir == "" { |
| workDir, err = os.MkdirTemp("", "vcweb-*") |
| if err != nil { |
| return nil, err |
| } |
| logger.Printf("vcweb work directory: %s", workDir) |
| } else { |
| workDir, err = filepath.Abs(workDir) |
| if err != nil { |
| return nil, err |
| } |
| } |
| |
| homeDir := filepath.Join(workDir, "home") |
| if err := os.MkdirAll(homeDir, 0755); err != nil { |
| return nil, err |
| } |
| |
| env := scriptEnviron(homeDir) |
| |
| s := &Server{ |
| env: env, |
| logger: logger, |
| scriptDir: scriptDir, |
| workDir: workDir, |
| homeDir: homeDir, |
| engine: newScriptEngine(), |
| vcsHandlers: map[string]vcsHandler{ |
| "auth": new(authHandler), |
| "dir": new(dirHandler), |
| "bzr": new(bzrHandler), |
| "fossil": new(fossilHandler), |
| "git": new(gitHandler), |
| "hg": new(hgHandler), |
| "insecure": new(insecureHandler), |
| "svn": &svnHandler{svnRoot: workDir, logger: logger}, |
| }, |
| } |
| |
| if err := os.WriteFile(filepath.Join(s.homeDir, ".gitconfig"), []byte(gitConfig), 0644); err != nil { |
| return nil, err |
| } |
| gitConfigDir := filepath.Join(s.homeDir, ".config", "git") |
| if err := os.MkdirAll(gitConfigDir, 0755); err != nil { |
| return nil, err |
| } |
| if err := os.WriteFile(filepath.Join(gitConfigDir, "ignore"), []byte(""), 0644); err != nil { |
| return nil, err |
| } |
| |
| if err := os.WriteFile(filepath.Join(s.homeDir, ".hgrc"), []byte(hgrc), 0644); err != nil { |
| return nil, err |
| } |
| |
| return s, nil |
| } |
| |
| func (s *Server) Close() error { |
| var firstErr error |
| for _, h := range s.vcsHandlers { |
| if c, ok := h.(io.Closer); ok { |
| if closeErr := c.Close(); firstErr == nil { |
| firstErr = closeErr |
| } |
| } |
| } |
| return firstErr |
| } |
| |
| // gitConfig contains a ~/.gitconfg file that attempts to provide |
| // deterministic, platform-agnostic behavior for the 'git' command. |
| var gitConfig = ` |
| [user] |
| name = Go Gopher |
| email = gopher@golang.org |
| [init] |
| defaultBranch = main |
| [core] |
| eol = lf |
| [gui] |
| encoding = utf-8 |
| `[1:] |
| |
| // hgrc contains a ~/.hgrc file that attempts to provide |
| // deterministic, platform-agnostic behavior for the 'hg' command. |
| var hgrc = ` |
| [ui] |
| username=Go Gopher <gopher@golang.org> |
| [phases] |
| new-commit=public |
| [extensions] |
| convert= |
| `[1:] |
| |
| // ServeHTTP implements [http.Handler] for version-control repositories. |
| func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { |
| s.logger.Printf("serving %s", req.URL) |
| |
| defer func() { |
| if v := recover(); v != nil { |
| debug.PrintStack() |
| s.logger.Fatal(v) |
| } |
| }() |
| |
| urlPath := req.URL.Path |
| if !strings.HasPrefix(urlPath, "/") { |
| urlPath = "/" + urlPath |
| } |
| clean := path.Clean(urlPath)[1:] |
| if clean == "" { |
| s.overview(w, req) |
| return |
| } |
| if clean == "help" { |
| s.help(w, req) |
| return |
| } |
| |
| // Locate the script that generates the requested path. |
| // We follow directories all the way to the end, then look for a ".txt" file |
| // matching the first component that doesn't exist. That guarantees |
| // uniqueness: if a path exists as a directory, then it cannot exist as a |
| // ".txt" script (because the search would ignore that file). |
| scriptPath := "." |
| for _, part := range strings.Split(clean, "/") { |
| scriptPath = filepath.Join(scriptPath, part) |
| dir := filepath.Join(s.scriptDir, scriptPath) |
| if _, err := os.Stat(dir); err != nil { |
| if !os.IsNotExist(err) { |
| http.Error(w, err.Error(), http.StatusInternalServerError) |
| return |
| } |
| // scriptPath does not exist as a directory, so it either is the script |
| // location or the script doesn't exist. |
| break |
| } |
| } |
| scriptPath += ".txt" |
| |
| err := s.HandleScript(scriptPath, s.logger, func(handler http.Handler) { |
| handler.ServeHTTP(w, req) |
| }) |
| if err != nil { |
| s.logger.Print(err) |
| if notFound := (ScriptNotFoundError{}); errors.As(err, ¬Found) { |
| http.NotFound(w, req) |
| } else if notInstalled := (ServerNotInstalledError{}); errors.As(err, ¬Installed) || errors.Is(err, exec.ErrNotFound) { |
| http.Error(w, err.Error(), http.StatusNotImplemented) |
| } else { |
| http.Error(w, err.Error(), http.StatusInternalServerError) |
| } |
| } |
| } |
| |
| // A ScriptNotFoundError indicates that the requested script file does not exist. |
| // (It typically wraps a "stat" error for the script file.) |
| type ScriptNotFoundError struct{ err error } |
| |
| func (e ScriptNotFoundError) Error() string { return e.err.Error() } |
| func (e ScriptNotFoundError) Unwrap() error { return e.err } |
| |
| // A ServerNotInstalledError indicates that the server binary required for the |
| // indicated VCS does not exist. |
| type ServerNotInstalledError struct{ name string } |
| |
| func (v ServerNotInstalledError) Error() string { |
| return fmt.Sprintf("server for %#q VCS is not installed", v.name) |
| } |
| |
| // HandleScript ensures that the script at scriptRelPath has been evaluated |
| // with its current contents. |
| // |
| // If the script completed successfully, HandleScript invokes f on the handler |
| // with the script's result still read-locked, and waits for it to return. (That |
| // ensures that cache invalidation does not race with an in-flight handler.) |
| // |
| // Otherwise, HandleScript returns the (cached) error from executing the script. |
| func (s *Server) HandleScript(scriptRelPath string, logger *log.Logger, f func(http.Handler)) error { |
| ri, ok := s.scriptCache.Load(scriptRelPath) |
| if !ok { |
| ri, _ = s.scriptCache.LoadOrStore(scriptRelPath, new(scriptResult)) |
| } |
| r := ri.(*scriptResult) |
| |
| relDir := strings.TrimSuffix(scriptRelPath, filepath.Ext(scriptRelPath)) |
| workDir := filepath.Join(s.workDir, relDir) |
| prefix := path.Join("/", filepath.ToSlash(relDir)) |
| |
| r.mu.RLock() |
| defer r.mu.RUnlock() |
| for { |
| // For efficiency, we cache the script's output (in the work directory) |
| // across invocations. However, to allow for rapid iteration, we hash the |
| // script's contents and regenerate its output if the contents change. |
| // |
| // That way, one can use 'go run main.go' in this directory to stand up a |
| // server and see the output of the test script in order to fine-tune it. |
| content, err := os.ReadFile(filepath.Join(s.scriptDir, scriptRelPath)) |
| if err != nil { |
| if !os.IsNotExist(err) { |
| return err |
| } |
| return ScriptNotFoundError{err} |
| } |
| |
| hash := sha256.Sum256(content) |
| if prevHash := r.hash; prevHash != hash { |
| // The script's hash has changed, so regenerate its output. |
| func() { |
| r.mu.RUnlock() |
| r.mu.Lock() |
| defer func() { |
| r.mu.Unlock() |
| r.mu.RLock() |
| }() |
| if r.hash != prevHash { |
| // The cached result changed while we were waiting on the lock. |
| // It may have been updated to our hash or something even newer, |
| // so don't overwrite it. |
| return |
| } |
| |
| r.hash = hash |
| r.hashTime = time.Now() |
| r.handler, r.err = nil, nil |
| |
| if err := os.RemoveAll(workDir); err != nil { |
| r.err = err |
| return |
| } |
| |
| // Note: we use context.Background here instead of req.Context() so that we |
| // don't cache a spurious error (and lose work) if the request is canceled |
| // while the script is still running. |
| scriptHandler, err := s.loadScript(context.Background(), logger, scriptRelPath, content, workDir) |
| if err != nil { |
| r.err = err |
| return |
| } |
| r.handler = http.StripPrefix(prefix, scriptHandler) |
| }() |
| } |
| |
| if r.hash != hash { |
| continue // Raced with an update from another handler; try again. |
| } |
| |
| if r.err != nil { |
| return r.err |
| } |
| f(r.handler) |
| return nil |
| } |
| } |
| |
| // overview serves an HTML summary of the status of the scripts in the server's |
| // script directory. |
| func (s *Server) overview(w http.ResponseWriter, r *http.Request) { |
| fmt.Fprintf(w, "<html>\n") |
| fmt.Fprintf(w, "<title>vcweb</title>\n<pre>\n") |
| fmt.Fprintf(w, "<b>vcweb</b>\n\n") |
| fmt.Fprintf(w, "This server serves various version control repos for testing the go command.\n\n") |
| fmt.Fprintf(w, "For an overview of the script language, see <a href=\"/help\">/help</a>.\n\n") |
| |
| fmt.Fprintf(w, "<b>cache</b>\n") |
| |
| tw := tabwriter.NewWriter(w, 1, 8, 1, '\t', 0) |
| err := filepath.WalkDir(s.scriptDir, func(path string, d fs.DirEntry, err error) error { |
| if err != nil { |
| return err |
| } |
| if filepath.Ext(path) != ".txt" { |
| return nil |
| } |
| |
| rel, err := filepath.Rel(s.scriptDir, path) |
| if err != nil { |
| return err |
| } |
| hashTime := "(not loaded)" |
| status := "" |
| if ri, ok := s.scriptCache.Load(rel); ok { |
| r := ri.(*scriptResult) |
| r.mu.RLock() |
| defer r.mu.RUnlock() |
| |
| if !r.hashTime.IsZero() { |
| hashTime = r.hashTime.Format(time.RFC3339) |
| } |
| if r.err == nil { |
| status = "ok" |
| } else { |
| status = r.err.Error() |
| } |
| } |
| fmt.Fprintf(tw, "%s\t%s\t%s\n", rel, hashTime, status) |
| return nil |
| }) |
| tw.Flush() |
| |
| if err != nil { |
| fmt.Fprintln(w, err) |
| } |
| } |
| |
| // help serves a plain-text summary of the server's supported script language. |
| func (s *Server) help(w http.ResponseWriter, req *http.Request) { |
| st, err := s.newState(req.Context(), s.workDir) |
| if err != nil { |
| http.Error(w, err.Error(), http.StatusInternalServerError) |
| return |
| } |
| |
| scriptLog := new(strings.Builder) |
| err = s.engine.Execute(st, "help", bufio.NewReader(strings.NewReader("help")), scriptLog) |
| if err != nil { |
| http.Error(w, err.Error(), http.StatusInternalServerError) |
| return |
| } |
| |
| w.Header().Set("Content-Type", "text/plain; charset=UTF-8") |
| io.WriteString(w, scriptLog.String()) |
| } |
