sumdb: client and server protocol for checksum database
Copied from golang.org/x/exp/sumdb/internal/sumweb
with types cleaned up:
Conn -> Client
Client -> ClientOps
Handler -> Server
Server -> ServerOps
For golang/go#31761.
Change-Id: If0e004e6c9cab69c82de428810d67aba074aa843
Reviewed-on: https://go-review.googlesource.com/c/mod/+/176466
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Hyang-Ah Hana Kim <hyangah@gmail.com>
diff --git a/sumdb/cache.go b/sumdb/cache.go
new file mode 100644
index 0000000..629e591
--- /dev/null
+++ b/sumdb/cache.go
@@ -0,0 +1,59 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Parallel cache.
+// This file is copied from cmd/go/internal/par.
+
+package sumdb
+
+import (
+ "sync"
+ "sync/atomic"
+)
+
+// parCache runs an action once per key and caches the result.
+type parCache struct {
+ m sync.Map
+}
+
+type cacheEntry struct {
+ done uint32
+ mu sync.Mutex
+ result interface{}
+}
+
+// Do calls the function f if and only if Do is being called for the first time with this key.
+// No call to Do with a given key returns until the one call to f returns.
+// Do returns the value returned by the one call to f.
+func (c *parCache) Do(key interface{}, f func() interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry))
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.mu.Lock()
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.result = f()
+ atomic.StoreUint32(&e.done, 1)
+ }
+ e.mu.Unlock()
+ }
+ return e.result
+}
+
+// Get returns the cached result associated with key.
+// It returns nil if there is no such result.
+// If the result for key is being computed, Get does not wait for the computation to finish.
+func (c *parCache) Get(key interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ return nil
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ return nil
+ }
+ return e.result
+}
diff --git a/sumdb/client.go b/sumdb/client.go
new file mode 100644
index 0000000..70dd56f
--- /dev/null
+++ b/sumdb/client.go
@@ -0,0 +1,671 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "path"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// A ClientOps provides the external operations
+// (file caching, HTTP fetches, and so on) needed by the Client.
+// The methods must be safe for concurrent use by multiple goroutines.
+type ClientOps interface {
+ // ReadRemote reads and returns the content served at the given path
+ // on the remote database server. The path begins with "/lookup" or "/tile/",
+ // and there is no need to parse the path in any way.
+ // It is the implementation's responsibility to turn that path into a full URL
+ // and make the HTTP request. ReadRemote should return an error for
+ // any non-200 HTTP response status.
+ ReadRemote(path string) ([]byte, error)
+
+ // ReadConfig reads and returns the content of the named configuration file.
+ // There are only a fixed set of configuration files.
+ //
+ // "key" returns a file containing the verifier key for the server.
+ //
+ // serverName + "/latest" returns a file containing the latest known
+ // signed tree from the server.
+ // To signal that the client wishes to start with an "empty" signed tree,
+ // ReadConfig can return a successful empty result (0 bytes of data).
+ ReadConfig(file string) ([]byte, error)
+
+ // WriteConfig updates the content of the named configuration file,
+ // changing it from the old []byte to the new []byte.
+ // If the old []byte does not match the stored configuration,
+ // WriteConfig must return ErrWriteConflict.
+ // Otherwise, WriteConfig should atomically replace old with new.
+ // The "key" configuration file is never written using WriteConfig.
+ WriteConfig(file string, old, new []byte) error
+
+ // ReadCache reads and returns the content of the named cache file.
+ // Any returned error will be treated as equivalent to the file not existing.
+ // There can be arbitrarily many cache files, such as:
+ // serverName/lookup/pkg@version
+ // serverName/tile/8/1/x123/456
+ ReadCache(file string) ([]byte, error)
+
+ // WriteCache writes the named cache file.
+ WriteCache(file string, data []byte)
+
+ // Log prints the given log message (such as with log.Print)
+ Log(msg string)
+
+ // SecurityError prints the given security error log message.
+ // The Client returns ErrSecurity from any operation that invokes SecurityError,
+ // but the return value is mainly for testing. In a real program,
+ // SecurityError should typically print the message and call log.Fatal or os.Exit.
+ SecurityError(msg string)
+}
+
+// ErrWriteConflict signals a write conflict during Client.WriteConfig.
+var ErrWriteConflict = errors.New("write conflict")
+
+// ErrSecurity is returned by Client operations that invoke Client.SecurityError.
+var ErrSecurity = errors.New("security error: misbehaving server")
+
+// A Client is a client connection to a checksum database.
+// All the methods are safe for simultaneous use by multiple goroutines.
+type Client struct {
+ ops ClientOps // access to operations in the external world
+
+ didLookup uint32
+
+ // one-time initialized data
+ initOnce sync.Once
+ initErr error // init error, if any
+ name string // name of accepted verifier
+ verifiers note.Verifiers // accepted verifiers (just one, but Verifiers for note.Open)
+ tileReader tileReader
+ tileHeight int
+ nosumdb string
+
+ record parCache // cache of record lookup, keyed by path@vers
+ tileCache parCache // cache of c.readTile, keyed by tile
+
+ latestMu sync.Mutex
+ latest tlog.Tree // latest known tree head
+ latestMsg []byte // encoded signed note for latest
+
+ tileSavedMu sync.Mutex
+ tileSaved map[tlog.Tile]bool // which tiles have been saved using c.ops.WriteCache already
+}
+
+// NewClient returns a new Client using the given Client.
+func NewClient(ops ClientOps) *Client {
+ return &Client{
+ ops: ops,
+ }
+}
+
+// init initiailzes the client (if not already initialized)
+// and returns any initialization error.
+func (c *Client) init() error {
+ c.initOnce.Do(c.initWork)
+ return c.initErr
+}
+
+// initWork does the actual initialization work.
+func (c *Client) initWork() {
+ defer func() {
+ if c.initErr != nil {
+ c.initErr = fmt.Errorf("initializing sumdb.Client: %v", c.initErr)
+ }
+ }()
+
+ c.tileReader.c = c
+ if c.tileHeight == 0 {
+ c.tileHeight = 8
+ }
+ c.tileSaved = make(map[tlog.Tile]bool)
+
+ vkey, err := c.ops.ReadConfig("key")
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ verifier, err := note.NewVerifier(strings.TrimSpace(string(vkey)))
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ c.verifiers = note.VerifierList(verifier)
+ c.name = verifier.Name()
+
+ data, err := c.ops.ReadConfig(c.name + "/latest")
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ if err := c.mergeLatest(data); err != nil {
+ c.initErr = err
+ return
+ }
+}
+
+// SetTileHeight sets the tile height for the Client.
+// Any call to SetTileHeight must happen before the first call to Lookup.
+// If SetTileHeight is not called, the Client defaults to tile height 8.
+// SetTileHeight can be called at most once,
+// and if so it must be called before the first call to Lookup.
+func (c *Client) SetTileHeight(height int) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetTileHeight used after Lookup")
+ }
+ if height <= 0 {
+ panic("invalid call to SetTileHeight")
+ }
+ if c.tileHeight != 0 {
+ panic("multiple calls to SetTileHeight")
+ }
+ c.tileHeight = height
+}
+
+// SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Client.
+// For any module path matching one of the patterns,
+// Lookup will return ErrGONOSUMDB.
+// SetGONOSUMDB can be called at most once,
+// and if so it must be called before the first call to Lookup.
+func (c *Client) SetGONOSUMDB(list string) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetGONOSUMDB used after Lookup")
+ }
+ if c.nosumdb != "" {
+ panic("multiple calls to SetGONOSUMDB")
+ }
+ c.nosumdb = list
+}
+
+// ErrGONOSUMDB is returned by Lookup for paths that match
+// a pattern listed in the GONOSUMDB list (set by SetGONOSUMDB,
+// usually from the environment variable).
+var ErrGONOSUMDB = errors.New("skipped (listed in GONOSUMDB)")
+
+func (c *Client) skip(target string) bool {
+ return globsMatchPath(c.nosumdb, target)
+}
+
+// globsMatchPath reports whether any path prefix of target
+// matches one of the glob patterns (as defined by path.Match)
+// in the comma-separated globs list.
+// It ignores any empty or malformed patterns in the list.
+func globsMatchPath(globs, target string) bool {
+ for globs != "" {
+ // Extract next non-empty glob in comma-separated list.
+ var glob string
+ if i := strings.Index(globs, ","); i >= 0 {
+ glob, globs = globs[:i], globs[i+1:]
+ } else {
+ glob, globs = globs, ""
+ }
+ if glob == "" {
+ continue
+ }
+
+ // A glob with N+1 path elements (N slashes) needs to be matched
+ // against the first N+1 path elements of target,
+ // which end just before the N+1'th slash.
+ n := strings.Count(glob, "/")
+ prefix := target
+ // Walk target, counting slashes, truncating at the N+1'th slash.
+ for i := 0; i < len(target); i++ {
+ if target[i] == '/' {
+ if n == 0 {
+ prefix = target[:i]
+ break
+ }
+ n--
+ }
+ }
+ if n > 0 {
+ // Not enough prefix elements.
+ continue
+ }
+ matched, _ := path.Match(glob, prefix)
+ if matched {
+ return true
+ }
+ }
+ return false
+}
+
+// Lookup returns the go.sum lines for the given module path and version.
+// The version may end in a /go.mod suffix, in which case Lookup returns
+// the go.sum lines for the module's go.mod-only hash.
+func (c *Client) Lookup(path, vers string) (lines []string, err error) {
+ atomic.StoreUint32(&c.didLookup, 1)
+
+ if c.skip(path) {
+ return nil, ErrGONOSUMDB
+ }
+
+ defer func() {
+ if err != nil {
+ err = fmt.Errorf("%s@%s: %v", path, vers, err)
+ }
+ }()
+
+ if err := c.init(); err != nil {
+ return nil, err
+ }
+
+ // Prepare encoded cache filename / URL.
+ epath, err := module.EscapePath(path)
+ if err != nil {
+ return nil, err
+ }
+ evers, err := module.EscapeVersion(strings.TrimSuffix(vers, "/go.mod"))
+ if err != nil {
+ return nil, err
+ }
+ remotePath := "/lookup/" + epath + "@" + evers
+ file := c.name + remotePath
+
+ // Fetch the data.
+ // The lookupCache avoids redundant ReadCache/GetURL operations
+ // (especially since go.sum lines tend to come in pairs for a given
+ // path and version) and also avoids having multiple of the same
+ // request in flight at once.
+ type cached struct {
+ data []byte
+ err error
+ }
+ result := c.record.Do(file, func() interface{} {
+ // Try the on-disk cache, or else get from web.
+ writeCache := false
+ data, err := c.ops.ReadCache(file)
+ if err != nil {
+ data, err = c.ops.ReadRemote(remotePath)
+ if err != nil {
+ return cached{nil, err}
+ }
+ writeCache = true
+ }
+
+ // Validate the record before using it for anything.
+ id, text, treeMsg, err := tlog.ParseRecord(data)
+ if err != nil {
+ return cached{nil, err}
+ }
+ if err := c.mergeLatest(treeMsg); err != nil {
+ return cached{nil, err}
+ }
+ if err := c.checkRecord(id, text); err != nil {
+ return cached{nil, err}
+ }
+
+ // Now that we've validated the record,
+ // save it to the on-disk cache (unless that's where it came from).
+ if writeCache {
+ c.ops.WriteCache(file, data)
+ }
+
+ return cached{data, nil}
+ }).(cached)
+ if result.err != nil {
+ return nil, result.err
+ }
+
+ // Extract the lines for the specific version we want
+ // (with or without /go.mod).
+ prefix := path + " " + vers + " "
+ var hashes []string
+ for _, line := range strings.Split(string(result.data), "\n") {
+ if strings.HasPrefix(line, prefix) {
+ hashes = append(hashes, line)
+ }
+ }
+ return hashes, nil
+}
+
+// mergeLatest merges the tree head in msg
+// with the Client's current latest tree head,
+// ensuring the result is a consistent timeline.
+// If the result is inconsistent, mergeLatest calls c.ops.SecurityError
+// with a detailed security error message and then
+// (only if c.ops.SecurityError does not exit the program) returns ErrSecurity.
+// If the Client's current latest tree head moves forward,
+// mergeLatest updates the underlying configuration file as well,
+// taking care to merge any independent updates to that configuration.
+func (c *Client) mergeLatest(msg []byte) error {
+ // Merge msg into our in-memory copy of the latest tree head.
+ when, err := c.mergeLatestMem(msg)
+ if err != nil {
+ return err
+ }
+ if when != msgFuture {
+ // msg matched our present or was in the past.
+ // No change to our present, so no update of config file.
+ return nil
+ }
+
+ // Flush our extended timeline back out to the configuration file.
+ // If the configuration file has been updated in the interim,
+ // we need to merge any updates made there as well.
+ // Note that writeConfig is an atomic compare-and-swap.
+ for {
+ msg, err := c.ops.ReadConfig(c.name + "/latest")
+ if err != nil {
+ return err
+ }
+ when, err := c.mergeLatestMem(msg)
+ if err != nil {
+ return err
+ }
+ if when != msgPast {
+ // msg matched our present or was from the future,
+ // and now our in-memory copy matches.
+ return nil
+ }
+
+ // msg (== config) is in the past, so we need to update it.
+ c.latestMu.Lock()
+ latestMsg := c.latestMsg
+ c.latestMu.Unlock()
+ if err := c.ops.WriteConfig(c.name+"/latest", msg, latestMsg); err != ErrWriteConflict {
+ // Success or a non-write-conflict error.
+ return err
+ }
+ }
+}
+
+const (
+ msgPast = 1 + iota
+ msgNow
+ msgFuture
+)
+
+// mergeLatestMem is like mergeLatest but is only concerned with
+// updating the in-memory copy of the latest tree head (c.latest)
+// not the configuration file.
+// The when result explains when msg happened relative to our
+// previous idea of c.latest:
+// msgPast means msg was from before c.latest,
+// msgNow means msg was exactly c.latest, and
+// msgFuture means msg was from after c.latest, which has now been updated.
+func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
+ if len(msg) == 0 {
+ // Accept empty msg as the unsigned, empty timeline.
+ c.latestMu.Lock()
+ latest := c.latest
+ c.latestMu.Unlock()
+ if latest.N == 0 {
+ return msgNow, nil
+ }
+ return msgPast, nil
+ }
+
+ note, err := note.Open(msg, c.verifiers)
+ if err != nil {
+ return 0, fmt.Errorf("reading tree note: %v\nnote:\n%s", err, msg)
+ }
+ tree, err := tlog.ParseTree([]byte(note.Text))
+ if err != nil {
+ return 0, fmt.Errorf("reading tree: %v\ntree:\n%s", err, note.Text)
+ }
+
+ // Other lookups may be calling mergeLatest with other heads,
+ // so c.latest is changing underfoot. We don't want to hold the
+ // c.mu lock during tile fetches, so loop trying to update c.latest.
+ c.latestMu.Lock()
+ latest := c.latest
+ latestMsg := c.latestMsg
+ c.latestMu.Unlock()
+
+ for {
+ // If the tree head looks old, check that it is on our timeline.
+ if tree.N <= latest.N {
+ if err := c.checkTrees(tree, msg, latest, latestMsg); err != nil {
+ return 0, err
+ }
+ if tree.N < latest.N {
+ return msgPast, nil
+ }
+ return msgNow, nil
+ }
+
+ // The tree head looks new. Check that we are on its timeline and try to move our timeline forward.
+ if err := c.checkTrees(latest, latestMsg, tree, msg); err != nil {
+ return 0, err
+ }
+
+ // Install our msg if possible.
+ // Otherwise we will go around again.
+ c.latestMu.Lock()
+ installed := false
+ if c.latest == latest {
+ installed = true
+ c.latest = tree
+ c.latestMsg = msg
+ } else {
+ latest = c.latest
+ latestMsg = c.latestMsg
+ }
+ c.latestMu.Unlock()
+
+ if installed {
+ return msgFuture, nil
+ }
+ }
+}
+
+// checkTrees checks that older (from olderNote) is contained in newer (from newerNote).
+// If an error occurs, such as malformed data or a network problem, checkTrees returns that error.
+// If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed
+// message and calls log.Fatal.
+func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error {
+ thr := tlog.TileHashReader(newer, &c.tileReader)
+ h, err := tlog.TreeHash(older.N, thr)
+ if err != nil {
+ if older.N == newer.N {
+ return fmt.Errorf("checking tree#%d: %v", older.N, err)
+ }
+ return fmt.Errorf("checking tree#%d against tree#%d: %v", older.N, newer.N, err)
+ }
+ if h == older.Hash {
+ return nil
+ }
+
+ // Detected a fork in the tree timeline.
+ // Start by reporting the inconsistent signed tree notes.
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "SECURITY ERROR\n")
+ fmt.Fprintf(&buf, "go.sum database server misbehavior detected!\n\n")
+ indent := func(b []byte) []byte {
+ return bytes.Replace(b, []byte("\n"), []byte("\n\t"), -1)
+ }
+ fmt.Fprintf(&buf, "old database:\n\t%s\n", indent(olderNote))
+ fmt.Fprintf(&buf, "new database:\n\t%s\n", indent(newerNote))
+
+ // The notes alone are not enough to prove the inconsistency.
+ // We also need to show that the newer note's tree hash for older.N
+ // does not match older.Hash. The consumer of this report could
+ // of course consult the server to try to verify the inconsistency,
+ // but we are holding all the bits we need to prove it right now,
+ // so we might as well print them and make the report not depend
+ // on the continued availability of the misbehaving server.
+ // Preparing this data only reuses the tiled hashes needed for
+ // tlog.TreeHash(older.N, thr) above, so assuming thr is caching tiles,
+ // there are no new access to the server here, and these operations cannot fail.
+ fmt.Fprintf(&buf, "proof of misbehavior:\n\t%v", h)
+ if p, err := tlog.ProveTree(newer.N, older.N, thr); err != nil {
+ fmt.Fprintf(&buf, "\tinternal error: %v\n", err)
+ } else if err := tlog.CheckTree(p, newer.N, newer.Hash, older.N, h); err != nil {
+ fmt.Fprintf(&buf, "\tinternal error: generated inconsistent proof\n")
+ } else {
+ for _, h := range p {
+ fmt.Fprintf(&buf, "\n\t%v", h)
+ }
+ }
+ c.ops.SecurityError(buf.String())
+ return ErrSecurity
+}
+
+// checkRecord checks that record #id's hash matches data.
+func (c *Client) checkRecord(id int64, data []byte) error {
+ c.latestMu.Lock()
+ latest := c.latest
+ c.latestMu.Unlock()
+
+ if id >= latest.N {
+ return fmt.Errorf("cannot validate record %d in tree of size %d", id, latest.N)
+ }
+ hashes, err := tlog.TileHashReader(latest, &c.tileReader).ReadHashes([]int64{tlog.StoredHashIndex(0, id)})
+ if err != nil {
+ return err
+ }
+ if hashes[0] == tlog.RecordHash(data) {
+ return nil
+ }
+ return fmt.Errorf("cannot authenticate record data in server response")
+}
+
+// tileReader is a *Client wrapper that implements tlog.TileReader.
+// The separate type avoids exposing the ReadTiles and SaveTiles
+// methods on Client itself.
+type tileReader struct {
+ c *Client
+}
+
+func (r *tileReader) Height() int {
+ return r.c.tileHeight
+}
+
+// ReadTiles reads and returns the requested tiles,
+// either from the on-disk cache or the server.
+func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
+ // Read all the tiles in parallel.
+ data := make([][]byte, len(tiles))
+ errs := make([]error, len(tiles))
+ var wg sync.WaitGroup
+ for i, tile := range tiles {
+ wg.Add(1)
+ go func(i int, tile tlog.Tile) {
+ defer wg.Done()
+ data[i], errs[i] = r.c.readTile(tile)
+ }(i, tile)
+ }
+ wg.Wait()
+
+ for _, err := range errs {
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return data, nil
+}
+
+// tileCacheKey returns the cache key for the tile.
+func (c *Client) tileCacheKey(tile tlog.Tile) string {
+ return c.name + "/" + tile.Path()
+}
+
+// tileRemotePath returns the remote path for the tile.
+func (c *Client) tileRemotePath(tile tlog.Tile) string {
+ return "/" + tile.Path()
+}
+
+// readTile reads a single tile, either from the on-disk cache or the server.
+func (c *Client) readTile(tile tlog.Tile) ([]byte, error) {
+ type cached struct {
+ data []byte
+ err error
+ }
+
+ result := c.tileCache.Do(tile, func() interface{} {
+ // Try the requested tile in on-disk cache.
+ data, err := c.ops.ReadCache(c.tileCacheKey(tile))
+ if err == nil {
+ c.markTileSaved(tile)
+ return cached{data, nil}
+ }
+
+ // Try the full tile in on-disk cache (if requested tile not already full).
+ // We only save authenticated tiles to the on-disk cache,
+ // so the recreated prefix is equally authenticated.
+ full := tile
+ full.W = 1 << uint(tile.H)
+ if tile != full {
+ data, err := c.ops.ReadCache(c.tileCacheKey(full))
+ if err == nil {
+ c.markTileSaved(tile) // don't save tile later; we already have full
+ return cached{data[:len(data)/full.W*tile.W], nil}
+ }
+ }
+
+ // Try requested tile from server.
+ data, err = c.ops.ReadRemote(c.tileRemotePath(tile))
+ if err == nil {
+ return cached{data, nil}
+ }
+
+ // Try full tile on server.
+ // If the partial tile does not exist, it should be because
+ // the tile has been completed and only the complete one
+ // is available.
+ if tile != full {
+ data, err := c.ops.ReadRemote(c.tileRemotePath(full))
+ if err == nil {
+ // Note: We could save the full tile in the on-disk cache here,
+ // but we don't know if it is valid yet, and we will only find out
+ // about the partial data, not the full data. So let SaveTiles
+ // save the partial tile, and we'll just refetch the full tile later
+ // once we can validate more (or all) of it.
+ return cached{data[:len(data)/full.W*tile.W], nil}
+ }
+ }
+
+ // Nothing worked.
+ // Return the error from the server fetch for the requested (not full) tile.
+ return cached{nil, err}
+ }).(cached)
+
+ return result.data, result.err
+}
+
+// markTileSaved records that tile is already present in the on-disk cache,
+// so that a future SaveTiles for that tile can be ignored.
+func (c *Client) markTileSaved(tile tlog.Tile) {
+ c.tileSavedMu.Lock()
+ c.tileSaved[tile] = true
+ c.tileSavedMu.Unlock()
+}
+
+// SaveTiles saves the now validated tiles.
+func (r *tileReader) SaveTiles(tiles []tlog.Tile, data [][]byte) {
+ c := r.c
+
+ // Determine which tiles need saving.
+ // (Tiles that came from the cache need not be saved back.)
+ save := make([]bool, len(tiles))
+ c.tileSavedMu.Lock()
+ for i, tile := range tiles {
+ if !c.tileSaved[tile] {
+ save[i] = true
+ c.tileSaved[tile] = true
+ }
+ }
+ c.tileSavedMu.Unlock()
+
+ for i, tile := range tiles {
+ if save[i] {
+ // If WriteCache fails here (out of disk space? i/o error?),
+ // c.tileSaved[tile] is still true and we will not try to write it again.
+ // Next time we run maybe we'll redownload it again and be
+ // more successful.
+ c.ops.WriteCache(c.name+"/"+tile.Path(), data[i])
+ }
+ }
+}
diff --git a/sumdb/client_test.go b/sumdb/client_test.go
new file mode 100644
index 0000000..0f3c481
--- /dev/null
+++ b/sumdb/client_test.go
@@ -0,0 +1,460 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "bytes"
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+const (
+ testName = "localhost.localdev/sumdb"
+ testVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6"
+ testSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
+)
+
+func TestClientLookup(t *testing.T) {
+ tc := newTestClient(t)
+ tc.mustHaveLatest(1)
+
+ // Basic lookup.
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+ tc.mustHaveLatest(3)
+
+ // Everything should now be cached, both for the original package and its /go.mod.
+ tc.getOK = false
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+ tc.mustLookup("rsc.io/sampler", "v1.3.0/go.mod", "rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=")
+ tc.mustHaveLatest(3)
+ tc.getOK = true
+ tc.getTileOK = false // the cache has what we need
+
+ // Lookup with multiple returned lines.
+ tc.mustLookup("rsc.io/quote", "v1.5.2", "rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=\nrsc.io/quote v1.5.2 h2:xyzzy")
+ tc.mustHaveLatest(3)
+
+ // Lookup with need for !-encoding.
+ // rsc.io/Quote is the only record written after rsc.io/samper,
+ // so it is the only one that should need more tiles.
+ tc.getTileOK = true
+ tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
+ tc.mustHaveLatest(4)
+}
+
+func TestClientBadTiles(t *testing.T) {
+ tc := newTestClient(t)
+
+ flipBits := func() {
+ for url, data := range tc.remote {
+ if strings.Contains(url, "/tile/") {
+ for i := range data {
+ data[i] ^= 0x80
+ }
+ }
+ }
+ }
+
+ // Bad tiles in initial download.
+ tc.mustHaveLatest(1)
+ flipBits()
+ _, err := tc.client.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile")
+ flipBits()
+ tc.newClient()
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+
+ // Bad tiles after initial download.
+ flipBits()
+ _, err = tc.client.Lookup("rsc.io/Quote", "v1.5.2")
+ tc.mustError(err, "rsc.io/Quote@v1.5.2: checking tree#3 against tree#4: downloaded inconsistent tile")
+ flipBits()
+ tc.newClient()
+ tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
+
+ // Bad starting tree hash looks like bad tiles.
+ tc.newClient()
+ text := tlog.FormatTree(tlog.Tree{N: 1, Hash: tlog.Hash{}})
+ data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.config[testName+"/latest"] = data
+ _, err = tc.client.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile")
+}
+
+func TestClientFork(t *testing.T) {
+ tc := newTestClient(t)
+ tc2 := tc.fork()
+
+ tc.addRecord("rsc.io/pkg1@v1.5.2", `rsc.io/pkg1 v1.5.2 h1:hash!=
+`)
+ tc.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!=
+`)
+ tc.mustLookup("rsc.io/pkg1", "v1.5.2", "rsc.io/pkg1 v1.5.2 h1:hash!=")
+
+ tc2.addRecord("rsc.io/pkg1@v1.5.3", `rsc.io/pkg1 v1.5.3 h1:hash!=
+`)
+ tc2.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!=
+`)
+ tc2.mustLookup("rsc.io/pkg1", "v1.5.4", "rsc.io/pkg1 v1.5.4 h1:hash!=")
+
+ key := "/lookup/rsc.io/pkg1@v1.5.2"
+ tc2.remote[key] = tc.remote[key]
+ _, err := tc2.client.Lookup("rsc.io/pkg1", "v1.5.2")
+ tc2.mustError(err, ErrSecurity.Error())
+
+ /*
+ SECURITY ERROR
+ go.sum database server misbehavior detected!
+
+ old database:
+ go.sum database tree!
+ 5
+ nWzN20+pwMt62p7jbv1/NlN95ePTlHijabv5zO/s36w=
+
+ — localhost.localdev/sumdb AAAMZ5/2FVAdMH58kmnz/0h299pwyskEbzDzoa2/YaPdhvLya4YWDFQQxu2TQb5GpwAH4NdWnTwuhILafisyf3CNbgg=
+
+ new database:
+ go.sum database tree
+ 6
+ wc4SkQt52o5W2nQ8To2ARs+mWuUJjss+sdleoiqxMmM=
+
+ — localhost.localdev/sumdb AAAMZ6oRNswlEZ6ZZhxrCvgl1MBy+nusq4JU+TG6Fe2NihWLqOzb+y2c2kzRLoCr4tvw9o36ucQEnhc20e4nA4Qc/wc=
+
+ proof of misbehavior:
+ T7i+H/8ER4nXOiw4Bj0koZOkGjkxoNvlI34GpvhHhQg=
+ Nsuejv72de9hYNM5bqFv8rv3gm3zJQwv/DT/WNbLDLA=
+ mOmqqZ1aI/lzS94oq/JSbj7pD8Rv9S+xDyi12BtVSHo=
+ /7Aw5jVSMM9sFjQhaMg+iiDYPMk6decH7QLOGrL9Lx0=
+ */
+
+ wants := []string{
+ "SECURITY ERROR",
+ "go.sum database server misbehavior detected!",
+ "old database:\n\tgo.sum database tree\n\t5\n",
+ "— localhost.localdev/sumdb AAAMZ5/2FVAd",
+ "new database:\n\tgo.sum database tree\n\t6\n",
+ "— localhost.localdev/sumdb AAAMZ6oRNswl",
+ "proof of misbehavior:\n\tT7i+H/8ER4nXOiw4Bj0k",
+ }
+ text := tc2.security.String()
+ for _, want := range wants {
+ if !strings.Contains(text, want) {
+ t.Fatalf("cannot find %q in security text:\n%s", want, text)
+ }
+ }
+}
+
+func TestClientGONOSUMDB(t *testing.T) {
+ tc := newTestClient(t)
+ tc.client.SetGONOSUMDB("p,*/q")
+ tc.client.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network
+ tc.getOK = false
+
+ ok := []string{
+ "abc",
+ "a/p",
+ "pq",
+ "q",
+ "n/o/p/q",
+ }
+ skip := []string{
+ "p",
+ "p/x",
+ "x/q",
+ "x/q/z",
+ }
+
+ for _, path := range ok {
+ _, err := tc.client.Lookup(path, "v1.0.0")
+ if err == ErrGONOSUMDB {
+ t.Errorf("Lookup(%q): ErrGONOSUMDB, wanted failed actual lookup", path)
+ }
+ }
+ for _, path := range skip {
+ _, err := tc.client.Lookup(path, "v1.0.0")
+ if err != ErrGONOSUMDB {
+ t.Errorf("Lookup(%q): %v, wanted ErrGONOSUMDB", path, err)
+ }
+ }
+}
+
+// A testClient is a self-contained client-side testing environment.
+type testClient struct {
+ t *testing.T // active test
+ client *Client // client being tested
+ tileHeight int // tile height to use (default 2)
+ getOK bool // should tc.GetURL succeed?
+ getTileOK bool // should tc.GetURL of tiles succeed?
+ treeSize int64
+ hashes []tlog.Hash
+ remote map[string][]byte
+ signer note.Signer
+
+ // mu protects config, cache, log, security
+ // during concurrent use of the exported methods
+ // by the client itself (testClient is the Client's ClientOps,
+ // and the Client methods can both read and write these fields).
+ // Unexported methods invoked directly by the test
+ // (for example, addRecord) need not hold the mutex:
+ // for proper test execution those methods should only
+ // be called when the Client is idle and not using its ClientOps.
+ // Not holding the mutex in those methods ensures
+ // that if a mistake is made, go test -race will report it.
+ // (Holding the mutex would eliminate the race report but
+ // not the underlying problem.)
+ // Similarly, the get map is not protected by the mutex,
+ // because the Client methods only read it.
+ mu sync.Mutex // prot
+ config map[string][]byte
+ cache map[string][]byte
+ security bytes.Buffer
+}
+
+// newTestClient returns a new testClient that will call t.Fatal on error
+// and has a few records already available on the remote server.
+func newTestClient(t *testing.T) *testClient {
+ tc := &testClient{
+ t: t,
+ tileHeight: 2,
+ getOK: true,
+ getTileOK: true,
+ config: make(map[string][]byte),
+ cache: make(map[string][]byte),
+ remote: make(map[string][]byte),
+ }
+
+ tc.config["key"] = []byte(testVerifierKey + "\n")
+ var err error
+ tc.signer, err = note.NewSigner(testSignerKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tc.newClient()
+
+ tc.addRecord("rsc.io/quote@v1.5.2", `rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/quote v1.5.2 h2:xyzzy
+`)
+
+ tc.addRecord("golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c", `golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+`)
+ tc.addRecord("rsc.io/sampler@v1.3.0", `rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+`)
+ tc.config[testName+"/latest"] = tc.signTree(1)
+
+ tc.addRecord("rsc.io/!quote@v1.5.2", `rsc.io/Quote v1.5.2 h1:uppercase!=
+`)
+ return tc
+}
+
+// newClient resets the Client associated with tc.
+// This clears any in-memory cache from the Client
+// but not tc's on-disk cache.
+func (tc *testClient) newClient() {
+ tc.client = NewClient(tc)
+ tc.client.SetTileHeight(tc.tileHeight)
+}
+
+// mustLookup does a lookup for path@vers and checks that the lines that come back match want.
+func (tc *testClient) mustLookup(path, vers, want string) {
+ tc.t.Helper()
+ lines, err := tc.client.Lookup(path, vers)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ if strings.Join(lines, "\n") != want {
+ tc.t.Fatalf("Lookup(%q, %q):\n\t%s\nwant:\n\t%s", path, vers, strings.Join(lines, "\n\t"), strings.Replace(want, "\n", "\n\t", -1))
+ }
+}
+
+// mustHaveLatest checks that the on-disk configuration
+// for latest is a tree of size n.
+func (tc *testClient) mustHaveLatest(n int64) {
+ tc.t.Helper()
+
+ latest := tc.config[testName+"/latest"]
+ lines := strings.Split(string(latest), "\n")
+ if len(lines) < 2 || lines[1] != fmt.Sprint(n) {
+ tc.t.Fatalf("/latest should have tree %d, but has:\n%s", n, latest)
+ }
+}
+
+// mustError checks that err's error string contains the text.
+func (tc *testClient) mustError(err error, text string) {
+ tc.t.Helper()
+ if err == nil || !strings.Contains(err.Error(), text) {
+ tc.t.Fatalf("err = %v, want %q", err, text)
+ }
+}
+
+// fork returns a copy of tc.
+// Changes made to the new copy or to tc are not reflected in the other.
+func (tc *testClient) fork() *testClient {
+ tc2 := &testClient{
+ t: tc.t,
+ getOK: tc.getOK,
+ getTileOK: tc.getTileOK,
+ tileHeight: tc.tileHeight,
+ treeSize: tc.treeSize,
+ hashes: append([]tlog.Hash{}, tc.hashes...),
+ signer: tc.signer,
+ config: copyMap(tc.config),
+ cache: copyMap(tc.cache),
+ remote: copyMap(tc.remote),
+ }
+ tc2.newClient()
+ return tc2
+}
+
+func copyMap(m map[string][]byte) map[string][]byte {
+ m2 := make(map[string][]byte)
+ for k, v := range m {
+ m2[k] = v
+ }
+ return m2
+}
+
+// ReadHashes is tc's implementation of tlog.HashReader, for use with
+// tlog.TreeHash and so on.
+func (tc *testClient) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
+ var list []tlog.Hash
+ for _, id := range indexes {
+ list = append(list, tc.hashes[id])
+ }
+ return list, nil
+}
+
+// addRecord adds a log record using the given (!-encoded) key and data.
+func (tc *testClient) addRecord(key, data string) {
+ tc.t.Helper()
+
+ // Create record, add hashes to log tree.
+ id := tc.treeSize
+ tc.treeSize++
+ rec, err := tlog.FormatRecord(id, []byte(data))
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.hashes = append(tc.hashes, hashes...)
+
+ // Create lookup result.
+ tc.remote["/lookup/"+key] = append(rec, tc.signTree(tc.treeSize)...)
+
+ // Create new tiles.
+ tiles := tlog.NewTiles(tc.tileHeight, id, tc.treeSize)
+ for _, tile := range tiles {
+ data, err := tlog.ReadTileData(tile, tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.remote["/"+tile.Path()] = data
+ // TODO delete old partial tiles
+ }
+}
+
+// signTree returns the signed head for the tree of the given size.
+func (tc *testClient) signTree(size int64) []byte {
+ h, err := tlog.TreeHash(size, tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
+ data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ return data
+}
+
+// ReadRemote is for tc's implementation of Client.
+func (tc *testClient) ReadRemote(path string) ([]byte, error) {
+ // No mutex here because only the Client should be running
+ // and the Client cannot change tc.get.
+ if !tc.getOK {
+ return nil, fmt.Errorf("disallowed remote read %s", path)
+ }
+ if strings.Contains(path, "/tile/") && !tc.getTileOK {
+ return nil, fmt.Errorf("disallowed remote tile read %s", path)
+ }
+
+ data, ok := tc.remote[path]
+ if !ok {
+ return nil, fmt.Errorf("no remote path %s", path)
+ }
+ return data, nil
+}
+
+// ReadConfig is for tc's implementation of Client.
+func (tc *testClient) ReadConfig(file string) ([]byte, error) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data, ok := tc.config[file]
+ if !ok {
+ return nil, fmt.Errorf("no config %s", file)
+ }
+ return data, nil
+}
+
+// WriteConfig is for tc's implementation of Client.
+func (tc *testClient) WriteConfig(file string, old, new []byte) error {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data := tc.config[file]
+ if !bytes.Equal(old, data) {
+ return ErrWriteConflict
+ }
+ tc.config[file] = new
+ return nil
+}
+
+// ReadCache is for tc's implementation of Client.
+func (tc *testClient) ReadCache(file string) ([]byte, error) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data, ok := tc.cache[file]
+ if !ok {
+ return nil, fmt.Errorf("no cache %s", file)
+ }
+ return data, nil
+}
+
+// WriteCache is for tc's implementation of Client.
+func (tc *testClient) WriteCache(file string, data []byte) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ tc.cache[file] = data
+}
+
+// Log is for tc's implementation of Client.
+func (tc *testClient) Log(msg string) {
+ tc.t.Log(msg)
+}
+
+// SecurityError is for tc's implementation of Client.
+func (tc *testClient) SecurityError(msg string) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ fmt.Fprintf(&tc.security, "%s\n", strings.TrimRight(msg, "\n"))
+}
diff --git a/sumdb/server.go b/sumdb/server.go
new file mode 100644
index 0000000..acaf720
--- /dev/null
+++ b/sumdb/server.go
@@ -0,0 +1,181 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package sumdb implements the HTTP protocols for serving or accessing a module checksum database.
+package sumdb
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "regexp"
+ "strings"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// A ServerOps provides the external operations
+// (underlying database access and so on) needed by the Server.
+type ServerOps interface {
+ // Signed returns the signed hash of the latest tree.
+ Signed(ctx context.Context) ([]byte, error)
+
+ // ReadRecords returns the content for the n records id through id+n-1.
+ ReadRecords(ctx context.Context, id, n int64) ([][]byte, error)
+
+ // Lookup looks up a record for the given module,
+ // returning the record ID.
+ Lookup(ctx context.Context, m module.Version) (int64, error)
+
+ // ReadTileData reads the content of tile t.
+ // It is only invoked for hash tiles (t.L ≥ 0).
+ ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error)
+}
+
+// A Server is the checksum database HTTP server,
+// which implements http.Handler and should be invoked
+// to serve the paths listed in ServerPaths.
+type Server struct {
+ ops ServerOps
+}
+
+// NewServer returns a new Server using the given operations.
+func NewServer(ops ServerOps) *Server {
+ return &Server{ops: ops}
+}
+
+// ServerPaths are the URL paths the Server can (and should) serve.
+//
+// Typically a server will do:
+//
+// srv := sumdb.NewServer(ops)
+// for _, path := range sumdb.ServerPaths {
+// http.Handle(path, srv)
+// }
+//
+var ServerPaths = []string{
+ "/lookup/",
+ "/latest",
+ "/tile/",
+}
+
+var modVerRE = regexp.MustCompile(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?(\+incompatible)?$`)
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
+
+ switch {
+ default:
+ http.NotFound(w, r)
+
+ case strings.HasPrefix(r.URL.Path, "/lookup/"):
+ mod := strings.TrimPrefix(r.URL.Path, "/lookup/")
+ if !modVerRE.MatchString(mod) {
+ http.Error(w, "invalid module@version syntax", http.StatusBadRequest)
+ return
+ }
+ i := strings.Index(mod, "@")
+ escPath, escVers := mod[:i], mod[i+1:]
+ path, err := module.UnescapePath(escPath)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ vers, err := module.UnescapeVersion(escVers)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ id, err := s.ops.Lookup(ctx, module.Version{Path: path, Version: vers})
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ records, err := s.ops.ReadRecords(ctx, id, 1)
+ if err != nil {
+ // This should never happen - the lookup says the record exists.
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if len(records) != 1 {
+ http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
+ return
+ }
+ msg, err := tlog.FormatRecord(id, records[0])
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ signed, err := s.ops.Signed(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(msg)
+ w.Write(signed)
+
+ case r.URL.Path == "/latest":
+ data, err := s.ops.Signed(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(data)
+
+ case strings.HasPrefix(r.URL.Path, "/tile/"):
+ t, err := tlog.ParseTilePath(r.URL.Path[1:])
+ if err != nil {
+ http.Error(w, "invalid tile syntax", http.StatusBadRequest)
+ return
+ }
+ if t.L == -1 {
+ // Record data.
+ start := t.N << uint(t.H)
+ records, err := s.ops.ReadRecords(ctx, start, int64(t.W))
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ if len(records) != t.W {
+ http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
+ return
+ }
+ var data []byte
+ for i, text := range records {
+ msg, err := tlog.FormatRecord(start+int64(i), text)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ data = append(data, msg...)
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(data)
+ return
+ }
+
+ data, err := s.ops.ReadTileData(ctx, t)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Write(data)
+ }
+}
+
+// reportError reports err to w.
+// If it's a not-found, the reported error is 404.
+// Otherwise it is an internal server error.
+// The caller must only call reportError in contexts where
+// a not-found err should be reported as 404.
+func reportError(w http.ResponseWriter, r *http.Request, err error) {
+ if os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ }
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+}
diff --git a/sumdb/test.go b/sumdb/test.go
new file mode 100644
index 0000000..534ca3e
--- /dev/null
+++ b/sumdb/test.go
@@ -0,0 +1,128 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumdb
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "sync"
+
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/mod/sumdb/tlog"
+)
+
+// NewTestServer constructs a new TestServer
+// that will sign its tree with the given signer key
+// (see golang.org/x/mod/sumdb/note)
+// and fetch new records as needed by calling gosum.
+func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) *TestServer {
+ return &TestServer{signer: signer, gosum: gosum}
+}
+
+// A TestServer is an in-memory implementation of Server for testing.
+type TestServer struct {
+ signer string
+ gosum func(path, vers string) ([]byte, error)
+
+ mu sync.Mutex
+ hashes testHashes
+ records [][]byte
+ lookup map[string]int64
+}
+
+// testHashes implements tlog.HashReader, reading from a slice.
+type testHashes []tlog.Hash
+
+func (h testHashes) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
+ var list []tlog.Hash
+ for _, id := range indexes {
+ list = append(list, h[id])
+ }
+ return list, nil
+}
+
+func (s *TestServer) Signed(ctx context.Context) ([]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ size := int64(len(s.records))
+ h, err := tlog.TreeHash(size, s.hashes)
+ if err != nil {
+ return nil, err
+ }
+ text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
+ signer, err := note.NewSigner(s.signer)
+ if err != nil {
+ return nil, err
+ }
+ return note.Sign(¬e.Note{Text: string(text)}, signer)
+}
+
+func (s *TestServer) ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ var list [][]byte
+ for i := int64(0); i < n; i++ {
+ if id+i >= int64(len(s.records)) {
+ return nil, fmt.Errorf("missing records")
+ }
+ list = append(list, s.records[id+i])
+ }
+ return list, nil
+}
+
+func (s *TestServer) Lookup(ctx context.Context, key string) (int64, error) {
+ s.mu.Lock()
+ id, ok := s.lookup[key]
+ s.mu.Unlock()
+ if ok {
+ return id, nil
+ }
+
+ // Look up module and compute go.sum lines.
+ i := strings.Index(key, "@")
+ if i < 0 {
+ return 0, fmt.Errorf("invalid lookup key %q", key)
+ }
+ path, vers := key[:i], key[i+1:]
+ data, err := s.gosum(path, vers)
+ if err != nil {
+ return 0, err
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ // We ran the fetch without the lock.
+ // If another fetch happened and committed, use it instead.
+ id, ok = s.lookup[key]
+ if ok {
+ return id, nil
+ }
+
+ // Add record.
+ id = int64(len(s.records))
+ s.records = append(s.records, data)
+ if s.lookup == nil {
+ s.lookup = make(map[string]int64)
+ }
+ s.lookup[key] = id
+ hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), s.hashes)
+ if err != nil {
+ panic(err)
+ }
+ s.hashes = append(s.hashes, hashes...)
+
+ return id, nil
+}
+
+func (s *TestServer) ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ return tlog.ReadTileData(t, s.hashes)
+}
