vulndb/govulncheck: move cache from osv client to govulncheck This CL copies cache implementation from osv client package to govulncheck. This CL will be followed by a CL that removes cache implementation from osv client package. Change-Id: I2b7b53f08a9c2c6ceb5e9f1a66825e4cc65b6281 Reviewed-on: https://go-review.googlesource.com/c/exp/+/347169 Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Guodong Li <guodongli@google.com> Reviewed-by: Tim King <taking@google.com> Trust: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/vulndb/govulncheck/cache.go b/vulndb/govulncheck/cache.go new file mode 100644 index 0000000..7021d7c --- /dev/null +++ b/vulndb/govulncheck/cache.go
@@ -0,0 +1,96 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "go/build" + "io/ioutil" + "os" + "path/filepath" + "time" + + "golang.org/x/vulndb/osv" +) + +// fsCache is file-system cache implementing osv.Cache +type fsCache struct { + rootDir string +} + +// use cfg.GOMODCACHE available in cmd/go/internal? +var defaultCacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vulndb") + +func defaultCache() *fsCache { + return &fsCache{rootDir: defaultCacheRoot} +} + +type cachedIndex struct { + Retrieved time.Time + Index osv.DBIndex +} + +func (c *fsCache) ReadIndex(dbName string) (osv.DBIndex, time.Time, error) { + b, err := ioutil.ReadFile(filepath.Join(c.rootDir, dbName, "index.json")) + if err != nil { + if os.IsNotExist(err) { + return nil, time.Time{}, nil + } + return nil, time.Time{}, err + } + var index cachedIndex + if err := json.Unmarshal(b, &index); err != nil { + return nil, time.Time{}, err + } + return index.Index, index.Retrieved, nil +} + +func (c *fsCache) WriteIndex(dbName string, index osv.DBIndex, retrieved time.Time) error { + path := filepath.Join(c.rootDir, dbName) + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + j, err := json.Marshal(cachedIndex{ + Index: index, + Retrieved: retrieved, + }) + if err != nil { + return err + } + if err := ioutil.WriteFile(filepath.Join(path, "index.json"), j, 0666); err != nil { + return err + } + return nil +} + +func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) { + b, err := ioutil.ReadFile(filepath.Join(c.rootDir, dbName, p, "vulns.json")) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var entries []*osv.Entry + if err := json.Unmarshal(b, &entries); err != nil { + return nil, err + } + return entries, nil +} + +func (c *fsCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error { + path := filepath.Join(c.rootDir, dbName, p) + if err := os.MkdirAll(path, 0777); err != nil { + return err + } + j, err := json.Marshal(entries) + if err != nil { + return err + } + if err := ioutil.WriteFile(filepath.Join(path, "vulns.json"), j, 0666); err != nil { + return err + } + return nil +}
diff --git a/vulndb/govulncheck/cache_test.go b/vulndb/govulncheck/cache_test.go new file mode 100644 index 0000000..1d3d4a2 --- /dev/null +++ b/vulndb/govulncheck/cache_test.go
@@ -0,0 +1,77 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "golang.org/x/vulndb/osv" +) + +func TestCache(t *testing.T) { + tmpDir := t.TempDir() + + cache := &fsCache{rootDir: tmpDir} + dbName := "vulndb.golang.org" + + _, _, err := cache.ReadIndex(dbName) + if err != nil { + t.Fatalf("ReadIndex failed for non-existent database: %v", err) + } + + if err = os.Mkdir(filepath.Join(tmpDir, dbName), 0777); err != nil { + t.Fatalf("os.Mkdir failed: %v", err) + } + _, _, err = cache.ReadIndex(dbName) + if err != nil { + t.Fatalf("ReadIndex failed for database without cached index: %v", err) + } + + now := time.Now() + expectedIdx := osv.DBIndex{ + "a.vuln.example.com": time.Time{}.Add(time.Hour), + "b.vuln.example.com": time.Time{}.Add(time.Hour * 2), + "c.vuln.example.com": time.Time{}.Add(time.Hour * 3), + } + if err = cache.WriteIndex(dbName, expectedIdx, now); err != nil { + t.Fatalf("WriteIndex failed to write index: %v", err) + } + + idx, retrieved, err := cache.ReadIndex(dbName) + if err != nil { + t.Fatalf("ReadIndex failed for database with cached index: %v", err) + } + if !reflect.DeepEqual(idx, expectedIdx) { + t.Errorf("ReadIndex returned unexpected index, got:\n%s\nwant:\n%s", idx, expectedIdx) + } + if !retrieved.Equal(now) { + t.Errorf("ReadIndex returned unexpected retrieved: got %s, want %s", retrieved, now) + } + + if _, err = cache.ReadEntries(dbName, "vuln.example.com"); err != nil { + t.Fatalf("ReadEntires failed for non-existent package: %v", err) + } + + expectedEntries := []*osv.Entry{ + &osv.Entry{ID: "001"}, + &osv.Entry{ID: "002"}, + &osv.Entry{ID: "003"}, + } + if err := cache.WriteEntries(dbName, "vuln.example.com", expectedEntries); err != nil { + t.Fatalf("WriteEntries failed: %v", err) + } + + entries, err := cache.ReadEntries(dbName, "vuln.example.com") + if err != nil { + t.Fatalf("ReadEntries failed for cached package: %v", err) + } + if !reflect.DeepEqual(entries, expectedEntries) { + t.Errorf("ReadEntries returned unexpected entries, got:\n%v\nwant:\n%v", entries, expectedEntries) + } +}
diff --git a/vulndb/govulncheck/main.go b/vulndb/govulncheck/main.go index 78d35f0..a50e426 100644 --- a/vulndb/govulncheck/main.go +++ b/vulndb/govulncheck/main.go
@@ -11,8 +11,7 @@ // WARNING WARNING WARNING // // govulncheck is still experimental and neither its output or the vulnerability -// database should be relied on to be stable or comprehensive. It also performs no -// caching of vulnerability database entries. +// database should be relied on to be stable or comprehensive. package main import ( @@ -94,7 +93,7 @@ if GOVULNDB := os.Getenv("GOVULNDB"); GOVULNDB != "" { dbs = strings.Split(GOVULNDB, ",") } - dbClient, err := client.NewClient(dbs, client.Options{HTTPCache: client.NewFsCache()}) + dbClient, err := client.NewClient(dbs, client.Options{HTTPCache: defaultCache()}) if err != nil { fmt.Fprintf(os.Stderr, "govulncheck: %s\n", err) os.Exit(1)
diff --git a/vulndb/govulncheck/main_test.go b/vulndb/govulncheck/main_test.go index 6cb3987..0fe79cd 100644 --- a/vulndb/govulncheck/main_test.go +++ b/vulndb/govulncheck/main_test.go
@@ -171,10 +171,6 @@ t.Logf("failed to get %s: %s", hashiVaultOkta+"@v1.6.3", out) t.Fatal(err) } - // if out, err := execCmd(e.Config.Dir, env, "go", "mod", "tidy"); err != nil { - // t.Logf("failed to mod tidy: %s", out) - // t.Fatal(err) - // } // run goaudit. cfg := &packages.Config{ @@ -198,6 +194,8 @@ // list of packages whose vulns should be addded to source toAdd []string want []finding + // "" indicates no cache should be used + cacheRoot string }{ // test local db without yaml, which should result in no findings. {source: "file://" + dbPath, want: nil, @@ -215,6 +213,12 @@ {"github.com/go-yaml/yaml.decoder.unmarshal", 6}, {"github.com/go-yaml/yaml.yaml_parser_fetch_more_tokens", 12}}, }, + // repeat the last test but with cache + {source: "http://localhost:8080", cacheRoot: filepath.Join(e.Config.Dir, "/pkg/mod/cache/download/vulndb"), + want: []finding{ + {"github.com/go-yaml/yaml.decoder.unmarshal", 6}, + {"github.com/go-yaml/yaml.yaml_parser_fetch_more_tokens", 12}}, + }, } { for _, add := range test.toAdd { if strings.HasPrefix(test.source, "file://") { @@ -224,8 +228,11 @@ } } - // TODO: add caching - dbClient, err := client.NewClient([]string{test.source}, client.Options{}) + var opts client.Options + if test.cacheRoot != "" { + opts.HTTPCache = &fsCache{rootDir: test.cacheRoot} + } + dbClient, err := client.NewClient([]string{test.source}, opts) if err != nil { t.Error(err) } @@ -351,6 +358,8 @@ // list of packages whose vulns should be addded to source toAdd []string want []finding + // "" indicates no cache should be used + cacheRoot string }{ // test local db with only apiserver vuln, which should result in a single finding. {source: "file://" + dbPath, toAdd: []string{"github.com/go-yaml/yaml.json", "k8s.io/apiextensions-apiserver/pkg/apiserver.json"}, @@ -372,6 +381,14 @@ {"golang.org/x/crypto/ssh.NewPublicKey", 4}, {"golang.org/x/crypto/ssh.parseED25519", 9}, }}, + //repeat the last test but with a cache + {source: "http://localhost:8080", cacheRoot: filepath.Join(e.Config.Dir, "/pkg/mod/cache/download/vulndb"), + want: []finding{ + {"golang.org/x/crypto/ssh.NewPublicKey", 1}, + {"k8s.io/apiextensions-apiserver/pkg/apiserver.NewCustomResourceDefinitionHandler", 3}, + {"golang.org/x/crypto/ssh.NewPublicKey", 4}, + {"golang.org/x/crypto/ssh.parseED25519", 9}, + }}, } { for _, add := range test.toAdd { if strings.HasPrefix(test.source, "file://") { @@ -381,8 +398,11 @@ } } - // TODO: add caching - dbClient, err := client.NewClient([]string{test.source}, client.Options{}) + var opts client.Options + if test.cacheRoot != "" { + opts.HTTPCache = &fsCache{rootDir: test.cacheRoot} + } + dbClient, err := client.NewClient([]string{test.source}, opts) if err != nil { t.Error(err) }