all: use consistent index type across packages
All reports still need to have their published fields set.
Change-Id: I64feda32742bb5f85e310211f8da270e4346ad6b
Reviewed-on: https://team-review.git.corp.google.com/c/golang/vulndb/+/1036000
Reviewed-by: Roland Shoemaker <bracewell@google.com>
diff --git a/client/cache.go b/client/cache.go
index 3ef8d8a..05d031c 100644
--- a/client/cache.go
+++ b/client/cache.go
@@ -25,7 +25,7 @@
// $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/indexes/index.json
// {
// Retrieved time.Time
-// Index map[string]time.Time
+// Index osv.DBIndex
// }
//
// Each package also has a JSON file which contains the array of vulnerability
@@ -35,8 +35,8 @@
// []*osv.Entry
type Cache interface {
- ReadIndex(string) (map[string]time.Time, time.Time, error)
- WriteIndex(string, map[string]time.Time, time.Time) error
+ ReadIndex(string) (osv.DBIndex, time.Time, error)
+ WriteIndex(string, osv.DBIndex, time.Time) error
ReadEntries(string, string) ([]*osv.Entry, error)
WriteEntries(string, string, []*osv.Entry) error
}
@@ -54,10 +54,10 @@
type cachedIndex struct {
Retrieved time.Time
- Index map[string]time.Time
+ Index osv.DBIndex
}
-func (c *fsCache) ReadIndex(dbName string) (map[string]time.Time, time.Time, error) {
+func (c *fsCache) ReadIndex(dbName string) (osv.DBIndex, time.Time, error) {
b, err := os.ReadFile(filepath.Join(cacheRoot, dbName, "index.json"))
if err != nil {
if os.IsNotExist(err) {
@@ -72,7 +72,7 @@
return index.Index, index.Retrieved, nil
}
-func (c *fsCache) WriteIndex(dbName string, index map[string]time.Time, retrieved time.Time) error {
+func (c *fsCache) WriteIndex(dbName string, index osv.DBIndex, retrieved time.Time) error {
path := filepath.Join(cacheRoot, dbName)
if err := os.MkdirAll(path, 0777); err != nil {
return err
diff --git a/client/cache_test.go b/client/cache_test.go
index bd5e7f2..9c474fe 100644
--- a/client/cache_test.go
+++ b/client/cache_test.go
@@ -38,7 +38,7 @@
}
now := time.Now()
- expectedIdx := map[string]time.Time{
+ expectedIdx := osv.DBIndex{
"a.vuln.example.com": time.Time{}.Add(time.Hour),
"b.vuln.example.com": time.Time{}.Add(time.Hour * 2),
"c.vuln.example.com": time.Time{}.Add(time.Hour * 3),
diff --git a/client/client.go b/client/client.go
index 63694eb..c42ac2f 100644
--- a/client/client.go
+++ b/client/client.go
@@ -19,7 +19,7 @@
type source interface {
Get([]string) ([]*osv.Entry, error)
- Index() (map[string]time.Time, error)
+ Index() (osv.DBIndex, error)
}
type localSource struct {
@@ -44,8 +44,8 @@
return entries, nil
}
-func (ls *localSource) Index() (map[string]time.Time, error) {
- var index map[string]time.Time
+func (ls *localSource) Index() (osv.DBIndex, error) {
+ var index osv.DBIndex
b, err := os.ReadFile(filepath.Join(ls.dir, "index.json"))
if err != nil {
return nil, err
@@ -63,8 +63,8 @@
dbName string
}
-func (hs *httpSource) Index() (map[string]time.Time, error) {
- var cachedIndex map[string]time.Time
+func (hs *httpSource) Index() (osv.DBIndex, error) {
+ var cachedIndex osv.DBIndex
var cachedIndexRetrieved *time.Time
if hs.cache != nil {
@@ -104,7 +104,7 @@
if err != nil {
return nil, err
}
- var index map[string]time.Time
+ var index osv.DBIndex
if err = json.Unmarshal(b, &index); err != nil {
return nil, err
}
diff --git a/cmd/gendb/main.go b/cmd/gendb/main.go
index f2526b0..b024817 100644
--- a/cmd/gendb/main.go
+++ b/cmd/gendb/main.go
@@ -9,18 +9,12 @@
"path/filepath"
"reflect"
"strings"
- "time"
"github.com/BurntSushi/toml"
"golang.org/x/vulndb/osv"
"golang.org/x/vulndb/report"
)
-type IndexEntry struct {
- LastModified time.Time
- LastNewFinding time.Time
-}
-
func fail(why string) {
fmt.Fprintln(os.Stderr, why)
os.Exit(1)
@@ -77,46 +71,26 @@
}
}
- index := map[string]*IndexEntry{}
- if content, err := ioutil.ReadFile(filepath.Join(*jsonDir, "index.json")); err == nil {
- err = json.Unmarshal(content, &index)
- if err != nil {
- fail(fmt.Sprintf("failed to parse index: %s", err))
- }
- } else if err != nil && !os.IsNotExist(err) {
- fail(fmt.Sprintf("failed to read index %q: %s", filepath.Join(*jsonDir, "index.json"), err))
- }
-
- // TODO(bracewell): I'm pretty sure the freshness stuff is basically
- // completely broken at the moment.
- now := time.Now()
- for path, v := range jsonVulns {
+ index := make(osv.DBIndex, len(jsonVulns))
+ for path, vulns := range jsonVulns {
outPath := filepath.Join(*jsonDir, path)
- content, err := json.Marshal(v)
+ content, err := json.Marshal(vulns)
if err != nil {
fail(fmt.Sprintf("failed to marshal json: %s", err))
}
- // fmt.Println("making", filepath.Dir(outPath))
err = os.MkdirAll(filepath.Dir(outPath), 0700)
if err != nil {
fail(fmt.Sprintf("failed to create directory %q: %s", filepath.Dir(outPath), err))
}
- // if there is already an index entry, only update the file
- // if the set of vulns differ from what is already on disk
- if _, ok := index[path]; ok && matchesCurrent(outPath, v) {
- // fmt.Println("skipping", outPath)
- continue
- }
- // fmt.Println("writing", outPath, string(content))
err = ioutil.WriteFile(outPath+".json", content, 0644)
if err != nil {
fail(fmt.Sprintf("failed to write %q: %s", outPath+".json", err))
}
- if index[path] == nil {
- index[path] = &IndexEntry{}
+ for _, v := range vulns {
+ if v.LastModified.After(index[path]) {
+ index[path] = v.LastModified
+ }
}
- index[path].LastModified = now
- // also need to set the LastNewFinding, somewhat more complicated...
}
indexJSON, err := json.Marshal(index)
diff --git a/osv/json.go b/osv/json.go
index acf5eb5..19c249c 100644
--- a/osv/json.go
+++ b/osv/json.go
@@ -7,6 +7,13 @@
"golang.org/x/vulndb/report"
)
+// DBIndex contains a mapping of vulnerable packages to the
+// last time a new vulnerability was added to the database.
+// TODO: this is probably not the correct place to put this
+// type, since it's not really an OSV/CVF thing, but rather
+// vulndb implementatiion detail.
+type DBIndex map[string]time.Time
+
type Severity int
const (
@@ -166,7 +173,11 @@
// It would be better if this was just a recursive thing probably
for _, additional := range r.AdditionalPackages {
entryCopy := entry
- entryCopy.Package.Name = additional.Package
+ additionalImportPath := additional.Module
+ if additional.Package != "" {
+ additionalImportPath = additional.Package
+ }
+ entryCopy.Package.Name = additionalImportPath
entryCopy.EcosystemSpecific.Symbols = additional.Symbols
entryCopy.Affects = generateAffects(additional.Versions)
diff --git a/report/report.go b/report/report.go
index aa39392..2b94a85 100644
--- a/report/report.go
+++ b/report/report.go
@@ -1,5 +1,7 @@
package report
+import "time"
+
type VersionRange struct {
Introduced string
Fixed string
@@ -26,15 +28,17 @@
Symbols []string
Versions []VersionRange
} `toml:"additional_packages"`
- Versions []VersionRange
- Description string
- Severity string
- CVE string
- Credit string
- Symbols []string
- OS []string
- Arch []string
- Links struct {
+ Versions []VersionRange
+ Description string
+ Published time.Time
+ LastModified time.Time `toml:"last_modified"`
+ Severity string
+ CVE string
+ Credit string
+ Symbols []string
+ OS []string
+ Arch []string
+ Links struct {
PR string
Commit string
Context []string
