blob: 06feef5f9147bd91dafc327877b38bcee6483305 [file] [log] [blame]
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package database
import (
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"golang.org/x/exp/slices"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/osv"
"golang.org/x/vulndb/internal/osvutils"
"golang.org/x/vulndb/internal/report"
)
// Load loads a database assuming that path contains a full, valid
// database following the v1 specification.
//
// It errors if:
// - any required files are missing or invalid
// - any unexpected files are found in the index/ or ID/ folders
// (with the exception that ID/index.json, from the legacy spec, is ignored)
//
// Any files present in the top level directory are ignored.
func Load(path string) (_ *Database, err error) {
defer derrors.Wrap(&err, "Load(%q)", path)
db, err := RawLoad(filepath.Join(path, idDir))
if err != nil {
return nil, err
}
requireGzip := true
if err := db.validateIndex(filepath.Join(path, indexDir), requireGzip); err != nil {
return nil, err
}
if err := db.validateEntries(filepath.Join(path, idDir), requireGzip); err != nil {
return nil, err
}
return db, nil
}
// RawLoad loads a database assuming that vulnsPath contains ".json" files
// representing OSV entries.
// It errors if any of the files cannot be unmarshaled into osv.Entry.
// It does not require any database indexes or gzipped files to be present,
// Directories and non-JSON files are ignored.
// Also, to accommodate the legacy spec, the file "index.json" is ignored
// if present.
func RawLoad(vulnsPath string) (_ *Database, err error) {
defer derrors.Wrap(&err, "RawLoad(%q)", vulnsPath)
db, err := New()
if err != nil {
return nil, err
}
if err := filepath.WalkDir(vulnsPath, func(path string, f fs.DirEntry, err error) error {
if err != nil {
return err
}
fname := f.Name()
if f.IsDir() ||
fname == "index.json" ||
filepath.Ext(fname) != ".json" {
return nil
}
var entry osv.Entry
if err = report.UnmarshalFromFile(path, &entry); err != nil {
return fmt.Errorf("could not unmarshal %q: %v", path, err)
}
return db.Add(entry)
}); err != nil {
return nil, err
}
return db, nil
}
func (db *Database) validateIndex(indexPath string, requireGzip bool) (err error) {
defer derrors.Wrap(&err, "validateIndex(%q)", indexPath)
// Check that the index files are present and have the correct
// contents.
dbPath := filepath.Join(indexPath, dbEndpoint)
if err := checkFiles(dbPath, db.DB, requireGzip); err != nil {
return err
}
modulesPath := filepath.Join(indexPath, modulesEndpoint)
if err := checkFiles(modulesPath, db.Modules, requireGzip); err != nil {
return err
}
vulnsPath := filepath.Join(indexPath, vulnsEndpoint)
if err := checkFiles(vulnsPath, db.Vulns, requireGzip); err != nil {
return err
}
// Check for unexpected files in the index folder.
expected := []string{
indexDir,
dbEndpoint, dbEndpoint + ".gz",
modulesEndpoint, modulesEndpoint + ".gz",
vulnsEndpoint, vulnsEndpoint + ".gz",
}
return checkNoUnexpectedFiles(indexPath, expected)
}
func (db *Database) validateEntries(idPath string, requireGzip bool) (err error) {
defer derrors.Wrap(&err, "validateEntries(%q)", idPath)
expected := []string{
idDir,
"index.json", // index.json is OK to accommodate legacy spec
}
for _, entry := range db.Entries {
if err = osvutils.Validate(&entry); err != nil {
return err
}
path := filepath.Join(idPath, entry.ID+".json")
if err = checkFiles(path, entry, requireGzip); err != nil {
return err
}
expected = append(expected, entry.ID+".json", entry.ID+".json.gz")
}
return checkNoUnexpectedFiles(idPath, expected)
}
func checkNoUnexpectedFiles(path string, expected []string) error {
if err := filepath.WalkDir(path, func(path string, f fs.DirEntry, err error) error {
if err != nil {
return err
}
if !slices.Contains(expected, f.Name()) {
return fmt.Errorf("unexpected file %s", f.Name())
}
return nil
}); err != nil {
return err
}
return nil
}
// checkFiles ensures that filepath and filepath+".gz" exist and
// have contents consistent with v.
// Returns an error if:
// - any expected files don't exist or v cannot be marshaled
// - the contents of filepath do not match the result
// of marshaling v
// - the uncompressed contents of filepath+".gz" do not match the
// contents of filepath
func checkFiles(filepath string, v any, requireGzip bool) (err error) {
defer derrors.Wrap(&err, "checkFiles(%q)", filepath)
contents, err := os.ReadFile(filepath)
if err != nil {
return err
}
marshaled, err := json.Marshal(v)
if err != nil {
return err
}
if c, m := string(contents), string(marshaled); c != m {
return fmt.Errorf("%s: contents do not match marshaled bytes of value:\ncontents:\n%s\nvalue (marshaled):\n%s", filepath, c, m)
}
if requireGzip {
gzipped, err := readGzipped(filepath + ".gz")
if err != nil {
return err
}
if c, g := string(contents), string(gzipped); c != g {
return fmt.Errorf("%[1]s: contents of uncompressed file do not match contents of compressed file:\ncontents of %[1]s:\n%[2]s\ncontents of %[1]s.gz:\n%[3]s", filepath, c, g)
}
}
return nil
}