client: delete
The client package is deleted. x/vuln/client is replacing this package.
Change-Id: I93376d9f0f9ce151fdde3ee3c3c0223570d713cd
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/361204
Trust: Julie Qiu <julie@golang.org>
Run-TryBot: Julie Qiu <julie@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/client/cache.go b/client/cache.go
deleted file mode 100644
index a9a540f..0000000
--- a/client/cache.go
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package client
-
-import (
- "time"
-
- "golang.org/x/vulndb/osv"
-)
-
-// Cache interface for vuln DB caching.
-type Cache interface {
- ReadIndex(string) (osv.DBIndex, time.Time, error)
- WriteIndex(string, osv.DBIndex, time.Time) error
- ReadEntries(string, string) ([]*osv.Entry, error)
- WriteEntries(string, string, []*osv.Entry) error
-}
diff --git a/client/client.go b/client/client.go
deleted file mode 100644
index 7cd18c8..0000000
--- a/client/client.go
+++ /dev/null
@@ -1,389 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package client provides an interface for accessing vulnerability
-// databases, via either HTTP or local filesystem access.
-//
-// The expected database layout is the same for both HTTP and local
-// databases. The database index is located at the root of the
-// database, and contains a list of all of the vulnerable modules
-// documented in the databse and the time the most recent vulnerability
-// was added. The index file is called indx.json, and has the
-// following format:
-//
-// map[string]time.Time (osv.DBIndex)
-//
-// Each vulnerable module is represented by an individual JSON file
-// which contains all of the vulnerabilities in that module. The path
-// for each module file is simply the import path of the module,
-// i.e. vulnerabilities in golang.org/x/crypto are contained in the
-// golang.org/x/crypto.json file. The per-module JSON files have
-// the following format:
-//
-// []osv.Entry
-//
-// A single client.Client can be used to access multiple vulnerability
-// databases. When looking up vulnerable module each database is
-// consulted, and results are merged together.
-//
-// TODO: allow filtering private module, possibly at a database level?
-// (e.g. I may want to use multiple databases, but only lookup a specific
-// module in a subset of them)
-package client
-
-import (
- "encoding/json"
- "fmt"
- "io/ioutil"
- "net/http"
- "net/url"
- "os"
- "path/filepath"
- "sort"
- "strings"
- "time"
-
- "golang.org/x/vulndb/internal"
- "golang.org/x/vulndb/internal/derrors"
- "golang.org/x/vulndb/osv"
-)
-
-// Client interface for fetching vulnerabilities based on module path or ID.
-type Client interface {
- // GetByModule returns the entries that affect the given module path.
- // It returns (nil, nil) if there are none.
- GetByModule(string) ([]*osv.Entry, error)
-
- // GetByID returns the entry with the given ID, or (nil, nil) if there isn't
- // one.
- GetByID(string) (*osv.Entry, error)
-
- // ListIDs returns the IDs of all entries in the database.
- ListIDs() ([]string, error)
-
- unexported() // ensures that adding a method won't break users
-}
-
-type source interface {
- Client
- Index() (osv.DBIndex, error)
-}
-
-type localSource struct {
- dir string
-}
-
-func (*localSource) unexported() {}
-
-func (ls *localSource) GetByModule(module string) (_ []*osv.Entry, err error) {
- defer derrors.Wrap(&err, "GetByModule(%q)", module)
- content, err := ioutil.ReadFile(filepath.Join(ls.dir, module+".json"))
- if os.IsNotExist(err) {
- return nil, nil
- } else if err != nil {
- return nil, err
- }
- var e []*osv.Entry
- if err = json.Unmarshal(content, &e); err != nil {
- return nil, err
- }
- return e, nil
-}
-
-func (ls *localSource) GetByID(id string) (_ *osv.Entry, err error) {
- defer derrors.Wrap(&err, "GetByID(%q)", id)
- content, err := ioutil.ReadFile(filepath.Join(ls.dir, internal.IDDirectory, id+".json"))
- if os.IsNotExist(err) {
- return nil, nil
- } else if err != nil {
- return nil, err
- }
- var e osv.Entry
- if err = json.Unmarshal(content, &e); err != nil {
- return nil, err
- }
- return &e, nil
-}
-
-func (ls *localSource) ListIDs() (_ []string, err error) {
- defer derrors.Wrap(&err, "ListIDs()")
- content, err := ioutil.ReadFile(filepath.Join(ls.dir, internal.IDDirectory, "index.json"))
- if err != nil {
- return nil, err
- }
- var ids []string
- if err := json.Unmarshal(content, &ids); err != nil {
- return nil, err
- }
- return ids, nil
-}
-
-func (ls *localSource) Index() (_ osv.DBIndex, err error) {
- defer derrors.Wrap(&err, "Index()")
- var index osv.DBIndex
- b, err := ioutil.ReadFile(filepath.Join(ls.dir, "index.json"))
- if err != nil {
- return nil, err
- }
- if err = json.Unmarshal(b, &index); err != nil {
- return nil, err
- }
- return index, nil
-}
-
-type httpSource struct {
- url string // the base URI of the source (without trailing "/"). e.g. https://vuln.golang.org
- c *http.Client
- cache Cache
- dbName string
-}
-
-func (hs *httpSource) Index() (_ osv.DBIndex, err error) {
- defer derrors.Wrap(&err, "Index()")
-
- var cachedIndex osv.DBIndex
- var cachedIndexRetrieved *time.Time
-
- if hs.cache != nil {
- index, retrieved, err := hs.cache.ReadIndex(hs.dbName)
- if err != nil {
- return nil, err
- }
-
- cachedIndex = index
- if cachedIndex != nil {
- if time.Since(retrieved) < time.Hour*2 {
- return cachedIndex, nil
- }
-
- cachedIndexRetrieved = &retrieved
- }
- }
-
- req, err := http.NewRequest("GET", fmt.Sprintf("%s/index.json", hs.url), nil)
- if err != nil {
- return nil, err
- }
- if cachedIndexRetrieved != nil {
- req.Header.Add("If-Modified-Since", cachedIndexRetrieved.Format(http.TimeFormat))
- }
- resp, err := hs.c.Do(req)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if cachedIndexRetrieved != nil && resp.StatusCode == http.StatusNotModified {
- // If status has not been modified, this is equivalent to returning the
- // same index. We update the timestamp so the next cache index read does
- // not require a roundtrip to the server.
- if err = hs.cache.WriteIndex(hs.dbName, cachedIndex, time.Now()); err != nil {
- return nil, err
- }
- return cachedIndex, nil
- }
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
- }
- b, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
- var index osv.DBIndex
- if err = json.Unmarshal(b, &index); err != nil {
- return nil, err
- }
-
- if hs.cache != nil {
- if err = hs.cache.WriteIndex(hs.dbName, index, time.Now()); err != nil {
- return nil, err
- }
- }
-
- return index, nil
-}
-
-func (*httpSource) unexported() {}
-
-func (hs *httpSource) GetByModule(module string) (_ []*osv.Entry, err error) {
- defer derrors.Wrap(&err, "GetByModule(%q)", module)
-
- index, err := hs.Index()
- if err != nil {
- return nil, err
- }
-
- lastModified, present := index[module]
- if !present {
- return nil, nil
- }
-
- if hs.cache != nil {
- if cached, err := hs.cache.ReadEntries(hs.dbName, module); err != nil {
- return nil, err
- } else if len(cached) != 0 {
- var stale bool
- for _, c := range cached {
- if c.Modified.Before(lastModified) {
- stale = true
- break
- }
- }
- if !stale {
- return cached, nil
- }
- }
- }
-
- content, err := hs.readBody(fmt.Sprintf("%s/%s.json", hs.url, module))
- if err != nil || content == nil {
- return nil, err
- }
- var e []*osv.Entry
- // TODO: we may want to check that the returned entries actually match
- // the module we asked about, so that the cache cannot be poisoned
- if err = json.Unmarshal(content, &e); err != nil {
- return nil, err
- }
-
- if hs.cache != nil {
- if err := hs.cache.WriteEntries(hs.dbName, module, e); err != nil {
- return nil, err
- }
- }
- return e, nil
-}
-
-func (hs *httpSource) GetByID(id string) (_ *osv.Entry, err error) {
- defer derrors.Wrap(&err, "GetByID(%q)", id)
-
- content, err := hs.readBody(fmt.Sprintf("%s/%s/%s.json", hs.url, internal.IDDirectory, id))
- if err != nil || content == nil {
- return nil, err
- }
- var e osv.Entry
- if err := json.Unmarshal(content, &e); err != nil {
- return nil, err
- }
- return &e, nil
-}
-
-func (hs *httpSource) ListIDs() (_ []string, err error) {
- defer derrors.Wrap(&err, "ListIDs()")
-
- content, err := hs.readBody(fmt.Sprintf("%s/%s/index.json", hs.url, internal.IDDirectory))
- if err != nil {
- return nil, err
- }
- var ids []string
- if err := json.Unmarshal(content, &ids); err != nil {
- return nil, err
- }
- return ids, nil
-}
-
-func (hs *httpSource) readBody(url string) ([]byte, error) {
- resp, err := hs.c.Get(url)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if resp.StatusCode == http.StatusNotFound {
- return nil, nil
- }
- // might want this to be a LimitedReader
- return ioutil.ReadAll(resp.Body)
-}
-
-type client struct {
- sources []source
-}
-
-type Options struct {
- HTTPClient *http.Client
- HTTPCache Cache
-}
-
-func NewClient(sources []string, opts Options) (_ Client, err error) {
- defer derrors.Wrap(&err, "NewClient(%v, opts)", sources)
- c := &client{}
- for _, uri := range sources {
- uri = strings.TrimRight(uri, "/")
- // should parse the URI out here instead of in there
- switch {
- case strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://"):
- hs := &httpSource{url: uri}
- url, err := url.Parse(uri)
- if err != nil {
- return nil, err
- }
- hs.dbName = url.Hostname()
- if opts.HTTPCache != nil {
- hs.cache = opts.HTTPCache
- }
- if opts.HTTPClient != nil {
- hs.c = opts.HTTPClient
- } else {
- hs.c = new(http.Client)
- }
- c.sources = append(c.sources, hs)
- case strings.HasPrefix(uri, "file://"):
- c.sources = append(c.sources, &localSource{dir: strings.TrimPrefix(uri, "file://")})
- default:
- return nil, fmt.Errorf("source %q has unsupported scheme", uri)
- }
- }
- return c, nil
-}
-
-func (*client) unexported() {}
-
-func (c *client) GetByModule(module string) (_ []*osv.Entry, err error) {
- defer derrors.Wrap(&err, "GetByModule(%q)", module)
- var entries []*osv.Entry
- // probably should be parallelized
- for _, s := range c.sources {
- e, err := s.GetByModule(module)
- if err != nil {
- return nil, err // be failure tolerant?
- }
- entries = append(entries, e...)
- }
- return entries, nil
-}
-
-func (c *client) GetByID(id string) (_ *osv.Entry, err error) {
- defer derrors.Wrap(&err, "GetByID(%q)", id)
- for _, s := range c.sources {
- entry, err := s.GetByID(id)
- if err != nil {
- return nil, err // be failure tolerant?
- }
- if entry != nil {
- return entry, nil
- }
- }
- return nil, nil
-}
-
-// ListIDs returns the union of the IDs from all sources,
-// sorted lexically.
-func (c *client) ListIDs() (_ []string, err error) {
- defer derrors.Wrap(&err, "ListIDs()")
- idSet := map[string]bool{}
- for _, s := range c.sources {
- ids, err := s.ListIDs()
- if err != nil {
- return nil, err
- }
- for _, id := range ids {
- idSet[id] = true
- }
- }
- var ids []string
- for id := range idSet {
- ids = append(ids, id)
- }
- sort.Strings(ids)
- return ids, nil
-}
diff --git a/client/client_test.go b/client/client_test.go
deleted file mode 100644
index f4f0cb0..0000000
--- a/client/client_test.go
+++ /dev/null
@@ -1,360 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package client
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "net/http/httptest"
- "net/url"
- "os"
- "path"
- "reflect"
- "runtime"
- "testing"
- "time"
-
- "github.com/google/go-cmp/cmp"
- "golang.org/x/vulndb/internal"
- "golang.org/x/vulndb/osv"
-)
-
-var (
- testVuln = `
- {"ID":"ID","Package":{"Name":"golang.org/example/one","Ecosystem":"go"}, "Summary":"",
- "Severity":2,"Affects":{"Ranges":[{"Type":"SEMVER","Introduced":"","Fixed":"v2.2.0"}]},
- "ecosystem_specific":{"Symbols":["some_symbol_1"]
- }}`
-
- testVulns = "[" + testVuln + "]"
-)
-
-var (
- // index containing timestamps for package in testVuln.
- index string = `{
- "golang.org/example/one": "2020-03-09T10:00:00.81362141-07:00"
- }`
- // index of IDs
- idIndex string = `["ID"]`
-)
-
-// testCache for testing purposes
-type testCache struct {
- indexMap map[string]osv.DBIndex
- indexStamp map[string]time.Time
- vulnMap map[string]map[string][]*osv.Entry
-}
-
-func freshTestCache() *testCache {
- return &testCache{
- indexMap: make(map[string]osv.DBIndex),
- indexStamp: make(map[string]time.Time),
- vulnMap: make(map[string]map[string][]*osv.Entry),
- }
-}
-
-func (tc *testCache) ReadIndex(db string) (osv.DBIndex, time.Time, error) {
- index, ok := tc.indexMap[db]
- if !ok {
- return nil, time.Time{}, nil
- }
- stamp, ok := tc.indexStamp[db]
- if !ok {
- return nil, time.Time{}, nil
- }
- return index, stamp, nil
-}
-
-func (tc *testCache) WriteIndex(db string, index osv.DBIndex, stamp time.Time) error {
- tc.indexMap[db] = index
- tc.indexStamp[db] = stamp
- return nil
-}
-
-func (tc *testCache) ReadEntries(db, module string) ([]*osv.Entry, error) {
- mMap, ok := tc.vulnMap[db]
- if !ok {
- return nil, nil
- }
- return mMap[module], nil
-}
-
-func (tc *testCache) WriteEntries(db, module string, entries []*osv.Entry) error {
- mMap, ok := tc.vulnMap[db]
- if !ok {
- mMap = make(map[string][]*osv.Entry)
- tc.vulnMap[db] = mMap
- }
- mMap[module] = append(mMap[module], entries...)
- return nil
-}
-
-// cachedTestVuln returns a function creating a local cache
-// for db with `dbName` with a version of testVuln where
-// Summary="cached" and LastModified happened after entry
-// in the `index` for the same pkg.
-func cachedTestVuln(dbName string) Cache {
- c := freshTestCache()
- e := &osv.Entry{
- ID: "ID1",
- Details: "cached",
- Modified: time.Now(),
- }
- c.WriteEntries(dbName, "golang.org/example/one", []*osv.Entry{e})
- return c
-}
-
-// createDirAndFile creates a directory `dir` if such directory does
-// not exist and creates a `file` with `content` in the directory.
-func createDirAndFile(dir, file, content string) error {
- if err := os.MkdirAll(dir, 0755); err != nil {
- return err
- }
- return ioutil.WriteFile(path.Join(dir, file), []byte(content), 0644)
-}
-
-// localDB creates a local db with testVulns and index as contents.
-func localDB(t *testing.T) (string, error) {
- dbName := t.TempDir()
- for _, f := range []struct {
- dir, filename, content string
- }{
- {"golang.org/example", "one.json", testVulns},
- {"", "index.json", index},
- {internal.IDDirectory, "ID.json", testVuln},
- {internal.IDDirectory, "index.json", idIndex},
- } {
- if err := createDirAndFile(path.Join(dbName, f.dir), f.filename, f.content); err != nil {
- return "", err
- }
- }
- return dbName, nil
-}
-
-func newTestServer() *httptest.Server {
- dataHandler := func(data string) http.HandlerFunc {
- return func(w http.ResponseWriter, _ *http.Request) {
- io.WriteString(w, data)
- }
- }
-
- mux := http.NewServeMux()
- mux.HandleFunc("/golang.org/example/one.json", dataHandler(testVulns))
- mux.HandleFunc("/index.json", dataHandler(index))
- mux.HandleFunc(fmt.Sprintf("/%s/ID.json", internal.IDDirectory), dataHandler(testVuln))
- mux.HandleFunc("/ID/index.json", func(w http.ResponseWriter, r *http.Request) {
- io.WriteString(w, idIndex)
- })
- return httptest.NewServer(mux)
-}
-
-func TestClient(t *testing.T) {
- if runtime.GOOS == "js" {
- t.Skip("skipping test: no network on js")
- }
-
- // Create a local http database.
- srv := newTestServer()
- defer srv.Close()
- u, err := url.Parse(srv.URL)
- if err != nil {
- t.Fatal(err)
- }
- dbName := u.Hostname()
-
- // Create a local file database.
- localDBName, err := localDB(t)
- if err != nil {
- t.Fatal(err)
- }
- defer os.RemoveAll(localDBName)
-
- for _, test := range []struct {
- name string
- source string
- cache Cache
- // cache summary for testVuln
- summary string
- }{
- // Test the http client without any cache.
- {name: "http-no-cache", source: srv.URL, cache: nil, summary: ""},
- // Test the http client with empty cache.
- {name: "http-empty-cache", source: srv.URL, cache: freshTestCache(), summary: ""},
- // Test the client with non-stale cache containing a version of testVuln2 where Summary="cached".
- {name: "http-cache", source: srv.URL, cache: cachedTestVuln(dbName), summary: "cached"},
- // Repeat the same for local file client.
- {name: "file-no-cache", source: "file://" + localDBName, cache: nil, summary: ""},
- {name: "file-empty-cache", source: "file://" + localDBName, cache: freshTestCache(), summary: ""},
- // Cache does not play a role in local file databases.
- {name: "file-cache", source: "file://" + localDBName, cache: cachedTestVuln(localDBName), summary: ""},
- } {
- client, err := NewClient([]string{test.source}, Options{HTTPCache: test.cache})
- if err != nil {
- t.Fatal(err)
- }
-
- vulns, err := client.GetByModule("golang.org/example/one")
- if err != nil {
- t.Fatal(err)
- }
-
- if len(vulns) != 1 {
- t.Errorf("%s: want 1 vuln for golang.org/example/one; got %v", test.name, len(vulns))
- }
-
- if v := vulns[0]; v.Details != test.summary {
- t.Errorf("%s: want '%s' summary for testVuln; got '%s'", test.name, test.summary, v.Details)
- }
- }
-}
-
-func TestCorrectFetchesNoCache(t *testing.T) {
- fetches := map[string]int{}
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- fetches[r.URL.Path]++
- if r.URL.Path == "/index.json" {
- j, _ := json.Marshal(osv.DBIndex{
- "a": time.Now(),
- "b": time.Now(),
- })
- w.Write(j)
- } else {
- w.Write([]byte("[]"))
- }
- }))
- defer ts.Close()
-
- hs := &httpSource{url: ts.URL, c: new(http.Client)}
- for _, module := range []string{"a", "b", "c"} {
- if _, err := hs.GetByModule(module); err != nil {
- t.Fatalf("unexpected error: %s", err)
- }
- }
- expectedFetches := map[string]int{"/index.json": 3, "/a.json": 1, "/b.json": 1}
- if !reflect.DeepEqual(fetches, expectedFetches) {
- t.Errorf("unexpected fetches, got %v, want %v", fetches, expectedFetches)
- }
-}
-
-// Make sure that a cached index is used in the case it is stale
-// but there were no changes to it at the server side.
-func TestCorrectFetchesNoChangeIndex(t *testing.T) {
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path == "/index.json" {
- w.WriteHeader(http.StatusNotModified)
- }
- }))
- defer ts.Close()
- url, _ := url.Parse(ts.URL)
-
- // set timestamp so that cached index is stale,
- // i.e., more than two hours old.
- timeStamp := time.Now().Add(time.Hour * (-3))
- index := osv.DBIndex{"a": timeStamp}
- cache := freshTestCache()
- cache.WriteIndex(url.Hostname(), index, timeStamp)
-
- e := &osv.Entry{
- ID: "ID1",
- Modified: timeStamp,
- }
- cache.WriteEntries(url.Hostname(), "a", []*osv.Entry{e})
-
- client, err := NewClient([]string{ts.URL}, Options{HTTPCache: cache})
- if err != nil {
- t.Fatal(err)
- }
- vulns, err := client.GetByModule("a")
- if err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(vulns, []*osv.Entry{e}) {
- t.Errorf("want %v vuln; got %v", e, vulns)
- }
-}
-
-func TestClientByID(t *testing.T) {
- if runtime.GOOS == "js" {
- t.Skip("skipping test: no network on js")
- }
-
- srv := newTestServer()
- defer srv.Close()
-
- // Create a local file database.
- localDBName, err := localDB(t)
- if err != nil {
- t.Fatal(err)
- }
- defer os.RemoveAll(localDBName)
-
- var want osv.Entry
- if err := json.Unmarshal([]byte(testVuln), &want); err != nil {
- t.Fatal(err)
- }
- for _, test := range []struct {
- name string
- source string
- }{
- {name: "http", source: srv.URL},
- {name: "file", source: "file://" + localDBName},
- } {
- t.Run(test.name, func(t *testing.T) {
- client, err := NewClient([]string{test.source}, Options{})
- if err != nil {
- t.Fatal(err)
- }
- got, err := client.GetByID("ID")
- if err != nil {
- t.Fatal(err)
- }
- if !cmp.Equal(got, &want) {
- t.Errorf("got\n%+v\nwant\n%+v", got, &want)
- }
- })
- }
-}
-
-func TestListIDs(t *testing.T) {
- if runtime.GOOS == "js" {
- t.Skip("skipping test: no network on js")
- }
-
- srv := newTestServer()
- defer srv.Close()
-
- // Create a local file database.
- localDBName, err := localDB(t)
- if err != nil {
- t.Fatal(err)
- }
- defer os.RemoveAll(localDBName)
-
- want := []string{"ID"}
- for _, test := range []struct {
- name string
- source string
- }{
- {name: "http", source: srv.URL},
- {name: "file", source: "file://" + localDBName},
- } {
- t.Run(test.name, func(t *testing.T) {
- client, err := NewClient([]string{test.source}, Options{})
- if err != nil {
- t.Fatal(err)
- }
- got, err := client.ListIDs()
- if err != nil {
- t.Fatal(err)
- }
- if !cmp.Equal(got, want) {
- t.Errorf("got\n%+v\nwant\n%+v", got, want)
- }
- })
- }
-}
