vulndb/govulncheck: move cache from osv client to govulncheck
This CL copies cache implementation from osv client package to
govulncheck. This CL will be followed by a CL that removes cache
implementation from osv client package.
Change-Id: I2b7b53f08a9c2c6ceb5e9f1a66825e4cc65b6281
Reviewed-on: https://go-review.googlesource.com/c/exp/+/347169
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Guodong Li <guodongli@google.com>
Reviewed-by: Tim King <taking@google.com>
Trust: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/vulndb/govulncheck/cache.go b/vulndb/govulncheck/cache.go
new file mode 100644
index 0000000..7021d7c
--- /dev/null
+++ b/vulndb/govulncheck/cache.go
@@ -0,0 +1,96 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "encoding/json"
+ "go/build"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "time"
+
+ "golang.org/x/vulndb/osv"
+)
+
+// fsCache is file-system cache implementing osv.Cache
+type fsCache struct {
+ rootDir string
+}
+
+// use cfg.GOMODCACHE available in cmd/go/internal?
+var defaultCacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vulndb")
+
+func defaultCache() *fsCache {
+ return &fsCache{rootDir: defaultCacheRoot}
+}
+
+type cachedIndex struct {
+ Retrieved time.Time
+ Index osv.DBIndex
+}
+
+func (c *fsCache) ReadIndex(dbName string) (osv.DBIndex, time.Time, error) {
+ b, err := ioutil.ReadFile(filepath.Join(c.rootDir, dbName, "index.json"))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, time.Time{}, nil
+ }
+ return nil, time.Time{}, err
+ }
+ var index cachedIndex
+ if err := json.Unmarshal(b, &index); err != nil {
+ return nil, time.Time{}, err
+ }
+ return index.Index, index.Retrieved, nil
+}
+
+func (c *fsCache) WriteIndex(dbName string, index osv.DBIndex, retrieved time.Time) error {
+ path := filepath.Join(c.rootDir, dbName)
+ if err := os.MkdirAll(path, 0755); err != nil {
+ return err
+ }
+ j, err := json.Marshal(cachedIndex{
+ Index: index,
+ Retrieved: retrieved,
+ })
+ if err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(path, "index.json"), j, 0666); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
+ b, err := ioutil.ReadFile(filepath.Join(c.rootDir, dbName, p, "vulns.json"))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ var entries []*osv.Entry
+ if err := json.Unmarshal(b, &entries); err != nil {
+ return nil, err
+ }
+ return entries, nil
+}
+
+func (c *fsCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error {
+ path := filepath.Join(c.rootDir, dbName, p)
+ if err := os.MkdirAll(path, 0777); err != nil {
+ return err
+ }
+ j, err := json.Marshal(entries)
+ if err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(path, "vulns.json"), j, 0666); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/vulndb/govulncheck/cache_test.go b/vulndb/govulncheck/cache_test.go
new file mode 100644
index 0000000..1d3d4a2
--- /dev/null
+++ b/vulndb/govulncheck/cache_test.go
@@ -0,0 +1,77 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+
+ "golang.org/x/vulndb/osv"
+)
+
+func TestCache(t *testing.T) {
+ tmpDir := t.TempDir()
+
+ cache := &fsCache{rootDir: tmpDir}
+ dbName := "vulndb.golang.org"
+
+ _, _, err := cache.ReadIndex(dbName)
+ if err != nil {
+ t.Fatalf("ReadIndex failed for non-existent database: %v", err)
+ }
+
+ if err = os.Mkdir(filepath.Join(tmpDir, dbName), 0777); err != nil {
+ t.Fatalf("os.Mkdir failed: %v", err)
+ }
+ _, _, err = cache.ReadIndex(dbName)
+ if err != nil {
+ t.Fatalf("ReadIndex failed for database without cached index: %v", err)
+ }
+
+ now := time.Now()
+ expectedIdx := osv.DBIndex{
+ "a.vuln.example.com": time.Time{}.Add(time.Hour),
+ "b.vuln.example.com": time.Time{}.Add(time.Hour * 2),
+ "c.vuln.example.com": time.Time{}.Add(time.Hour * 3),
+ }
+ if err = cache.WriteIndex(dbName, expectedIdx, now); err != nil {
+ t.Fatalf("WriteIndex failed to write index: %v", err)
+ }
+
+ idx, retrieved, err := cache.ReadIndex(dbName)
+ if err != nil {
+ t.Fatalf("ReadIndex failed for database with cached index: %v", err)
+ }
+ if !reflect.DeepEqual(idx, expectedIdx) {
+ t.Errorf("ReadIndex returned unexpected index, got:\n%s\nwant:\n%s", idx, expectedIdx)
+ }
+ if !retrieved.Equal(now) {
+ t.Errorf("ReadIndex returned unexpected retrieved: got %s, want %s", retrieved, now)
+ }
+
+ if _, err = cache.ReadEntries(dbName, "vuln.example.com"); err != nil {
+ t.Fatalf("ReadEntires failed for non-existent package: %v", err)
+ }
+
+ expectedEntries := []*osv.Entry{
+ &osv.Entry{ID: "001"},
+ &osv.Entry{ID: "002"},
+ &osv.Entry{ID: "003"},
+ }
+ if err := cache.WriteEntries(dbName, "vuln.example.com", expectedEntries); err != nil {
+ t.Fatalf("WriteEntries failed: %v", err)
+ }
+
+ entries, err := cache.ReadEntries(dbName, "vuln.example.com")
+ if err != nil {
+ t.Fatalf("ReadEntries failed for cached package: %v", err)
+ }
+ if !reflect.DeepEqual(entries, expectedEntries) {
+ t.Errorf("ReadEntries returned unexpected entries, got:\n%v\nwant:\n%v", entries, expectedEntries)
+ }
+}
diff --git a/vulndb/govulncheck/main.go b/vulndb/govulncheck/main.go
index 78d35f0..a50e426 100644
--- a/vulndb/govulncheck/main.go
+++ b/vulndb/govulncheck/main.go
@@ -11,8 +11,7 @@
// WARNING WARNING WARNING
//
// govulncheck is still experimental and neither its output or the vulnerability
-// database should be relied on to be stable or comprehensive. It also performs no
-// caching of vulnerability database entries.
+// database should be relied on to be stable or comprehensive.
package main
import (
@@ -94,7 +93,7 @@
if GOVULNDB := os.Getenv("GOVULNDB"); GOVULNDB != "" {
dbs = strings.Split(GOVULNDB, ",")
}
- dbClient, err := client.NewClient(dbs, client.Options{HTTPCache: client.NewFsCache()})
+ dbClient, err := client.NewClient(dbs, client.Options{HTTPCache: defaultCache()})
if err != nil {
fmt.Fprintf(os.Stderr, "govulncheck: %s\n", err)
os.Exit(1)
diff --git a/vulndb/govulncheck/main_test.go b/vulndb/govulncheck/main_test.go
index 6cb3987..0fe79cd 100644
--- a/vulndb/govulncheck/main_test.go
+++ b/vulndb/govulncheck/main_test.go
@@ -171,10 +171,6 @@
t.Logf("failed to get %s: %s", hashiVaultOkta+"@v1.6.3", out)
t.Fatal(err)
}
- // if out, err := execCmd(e.Config.Dir, env, "go", "mod", "tidy"); err != nil {
- // t.Logf("failed to mod tidy: %s", out)
- // t.Fatal(err)
- // }
// run goaudit.
cfg := &packages.Config{
@@ -198,6 +194,8 @@
// list of packages whose vulns should be addded to source
toAdd []string
want []finding
+ // "" indicates no cache should be used
+ cacheRoot string
}{
// test local db without yaml, which should result in no findings.
{source: "file://" + dbPath, want: nil,
@@ -215,6 +213,12 @@
{"github.com/go-yaml/yaml.decoder.unmarshal", 6},
{"github.com/go-yaml/yaml.yaml_parser_fetch_more_tokens", 12}},
},
+ // repeat the last test but with cache
+ {source: "http://localhost:8080", cacheRoot: filepath.Join(e.Config.Dir, "/pkg/mod/cache/download/vulndb"),
+ want: []finding{
+ {"github.com/go-yaml/yaml.decoder.unmarshal", 6},
+ {"github.com/go-yaml/yaml.yaml_parser_fetch_more_tokens", 12}},
+ },
} {
for _, add := range test.toAdd {
if strings.HasPrefix(test.source, "file://") {
@@ -224,8 +228,11 @@
}
}
- // TODO: add caching
- dbClient, err := client.NewClient([]string{test.source}, client.Options{})
+ var opts client.Options
+ if test.cacheRoot != "" {
+ opts.HTTPCache = &fsCache{rootDir: test.cacheRoot}
+ }
+ dbClient, err := client.NewClient([]string{test.source}, opts)
if err != nil {
t.Error(err)
}
@@ -351,6 +358,8 @@
// list of packages whose vulns should be addded to source
toAdd []string
want []finding
+ // "" indicates no cache should be used
+ cacheRoot string
}{
// test local db with only apiserver vuln, which should result in a single finding.
{source: "file://" + dbPath, toAdd: []string{"github.com/go-yaml/yaml.json", "k8s.io/apiextensions-apiserver/pkg/apiserver.json"},
@@ -372,6 +381,14 @@
{"golang.org/x/crypto/ssh.NewPublicKey", 4},
{"golang.org/x/crypto/ssh.parseED25519", 9},
}},
+ //repeat the last test but with a cache
+ {source: "http://localhost:8080", cacheRoot: filepath.Join(e.Config.Dir, "/pkg/mod/cache/download/vulndb"),
+ want: []finding{
+ {"golang.org/x/crypto/ssh.NewPublicKey", 1},
+ {"k8s.io/apiextensions-apiserver/pkg/apiserver.NewCustomResourceDefinitionHandler", 3},
+ {"golang.org/x/crypto/ssh.NewPublicKey", 4},
+ {"golang.org/x/crypto/ssh.parseED25519", 9},
+ }},
} {
for _, add := range test.toAdd {
if strings.HasPrefix(test.source, "file://") {
@@ -381,8 +398,11 @@
}
}
- // TODO: add caching
- dbClient, err := client.NewClient([]string{test.source}, client.Options{})
+ var opts client.Options
+ if test.cacheRoot != "" {
+ opts.HTTPCache = &fsCache{rootDir: test.cacheRoot}
+ }
+ dbClient, err := client.NewClient([]string{test.source}, opts)
if err != nil {
t.Error(err)
}
