internal/vulndb: add schema and vulndb endpoint
Change-Id: I57a14295220377853ff8e360c196911205847b28
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/488176
Reviewed-by: Julie Qiu <julieqiu@google.com>
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/go.mod b/go.mod
index f088985..8a40c46 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@
golang.org/x/exp/event v0.0.0-20220218215828-6cf2b201936e
golang.org/x/mod v0.7.0
golang.org/x/net v0.7.0
+ golang.org/x/sync v0.1.0
golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2
golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1
google.golang.org/api v0.110.0
@@ -57,7 +58,6 @@
go.opentelemetry.io/otel/trace v1.11.2 // indirect
golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
golang.org/x/oauth2 v0.5.0 // indirect
- golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
diff --git a/internal/vulndb/vulndb.go b/internal/vulndb/vulndb.go
new file mode 100644
index 0000000..6fa62fd
--- /dev/null
+++ b/internal/vulndb/vulndb.go
@@ -0,0 +1,109 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package vulndb provides functionality for manipulating
+// inputs and outputs of vulndb endpoint.
+package vulndb
+
+import (
+ "time"
+
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/vuln/osv"
+)
+
+// Definitions for BigQuery.
+
+// SchemaVersion changes whenever the BigQuery vulndb schema changes.
+var SchemaVersion string
+
+func init() {
+ s, err := bigquery.InferSchema(Entry{})
+ if err != nil {
+ panic(err)
+ }
+ SchemaVersion = bigquery.SchemaVersion(s)
+ bigquery.AddTable(TableName, s)
+}
+
+const (
+ // Vuln DB requests live in their own dataset that doesn't vary.
+ // This is the same database the vulnbreqs endpoint uses.
+ DatasetName = "vulndb"
+ TableName = "vulndb"
+)
+
+// Entry is a row stored in a table. It follows the core
+// structure of osv.Entry.
+type Entry struct {
+ CreatedAt time.Time `bigquery:"created_at"`
+
+ ModifiedTime time.Time `bigquery:"modified_time"`
+ PublishedTime time.Time `bigquery:"published_time"`
+ WithdrawnTime time.Time `bigquery:"withdrawn_time"`
+
+ ID string `bigquery:"id"`
+
+ // Modules can in principle have multiple entries
+ // with the same path.
+ Modules []Module `bigquery:"modules"`
+}
+
+func (e *Entry) SetUploadTime(t time.Time) { e.CreatedAt = t }
+
+// Module plays the role of osv.Affected. The latter also has
+// a Module field (among others), but we merge them into one
+// type to avoid nesting which can make the queries more complex.
+type Module struct {
+ Path string `bigquery:"path"`
+ // Ranges field plays the role of osv.Range type
+ // where “SEMVER” range kind is assumed.
+ Ranges []Range `bigquery:"ranges"`
+}
+
+// Range plays the role of osv.RangeEvent. That is, it is
+// a list of versions representing the ranges in which the
+// module is vulnerable. The events should be sorted, and
+// MUST represent non-overlapping ranges.
+type Range struct {
+ Introduced string `bigquery:"introduced"`
+ Fixed string `bigquery:"fixed"`
+}
+
+func Convert(oe *osv.Entry) *Entry {
+ e := &Entry{
+ ID: oe.ID,
+ ModifiedTime: oe.Modified,
+ PublishedTime: oe.Published,
+ Modules: modules(oe),
+ }
+ if oe.Withdrawn != nil {
+ e.WithdrawnTime = *oe.Withdrawn
+ }
+ return e
+}
+
+func modules(oe *osv.Entry) []Module {
+ var modules []Module
+ for _, a := range oe.Affected {
+ modules = append(modules, Module{
+ Path: a.Package.Name,
+ Ranges: ranges(a),
+ })
+ }
+ return modules
+}
+
+func ranges(a osv.Affected) []Range {
+ var rs []Range
+ for _, r := range a.Ranges {
+ for _, e := range r.Events {
+ rs = append(rs, Range{
+ Introduced: e.Introduced,
+ Fixed: e.Fixed,
+ })
+ }
+ }
+ return rs
+}
diff --git a/internal/vulndb/vulndb_test.go b/internal/vulndb/vulndb_test.go
new file mode 100644
index 0000000..6c86f73
--- /dev/null
+++ b/internal/vulndb/vulndb_test.go
@@ -0,0 +1,38 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vulndb
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/vuln/osv"
+)
+
+func TestConvert(t *testing.T) {
+ oe := &osv.Entry{
+ ID: "a",
+ Affected: []osv.Affected{
+ {Package: osv.Package{Name: "example.mod/a"}, Ranges: []osv.AffectsRange{{Events: []osv.RangeEvent{{Introduced: "0"}, {Fixed: "0.9.0"}}}}},
+ {Package: osv.Package{Name: "a.example.mod/a"}, Ranges: []osv.AffectsRange{{Events: []osv.RangeEvent{{Introduced: "1.0.0"}, {Fixed: "2.0.0"}}}}},
+ }}
+ want := &Entry{
+ ID: "a",
+ Modules: []Module{
+ {
+ Path: "example.mod/a",
+ Ranges: []Range{{Introduced: "0"}, {Fixed: "0.9.0"}},
+ },
+ {
+ Path: "a.example.mod/a",
+ Ranges: []Range{{Introduced: "1.0.0"}, {Fixed: "2.0.0"}},
+ },
+ },
+ }
+ got := Convert(oe)
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Fatalf("mismatch (-want, +got):\n%s", diff)
+ }
+}
diff --git a/internal/worker/server.go b/internal/worker/server.go
index d577d78..1c18039 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -16,7 +16,6 @@
"time"
"cloud.google.com/go/errorreporting"
- "golang.org/x/pkgsite-metrics/internal"
"golang.org/x/pkgsite-metrics/internal/analysis"
"golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/config"
@@ -26,7 +25,6 @@
"golang.org/x/pkgsite-metrics/internal/observe"
"golang.org/x/pkgsite-metrics/internal/proxy"
"golang.org/x/pkgsite-metrics/internal/queue"
- "golang.org/x/pkgsite-metrics/internal/vulndbreqs"
vulnc "golang.org/x/vuln/client"
)
@@ -119,6 +117,9 @@
if err := s.registerAnalysisHandlers(ctx); err != nil {
return nil, err
}
+
+ // compute vulndb entries
+ s.handle("/vulndb", s.handleVulnDB)
// compute missing vuln.go.dev request counts
s.handle("/compute-requests", s.handleComputeRequests)
return s, nil
@@ -196,27 +197,6 @@
return nil
}
-func (s *Server) handleComputeRequests(w http.ResponseWriter, r *http.Request) error {
- ctx := r.Context()
- // Don't use the Server's BigQuery client: it's for the wrong
- // dataset.
- vClient, err := bigquery.NewClientCreate(ctx, s.cfg.ProjectID, vulndbreqs.DatasetName)
- if err != nil {
- return err
- }
- keyName := "projects/" + s.cfg.ProjectID + "/secrets/vulndb-hmac-key"
- hmacKey, err := internal.GetSecret(ctx, keyName)
- if err != nil {
- return err
- }
- err = vulndbreqs.ComputeAndStore(ctx, s.cfg.VulnDBBucketProjectID, vClient, []byte(hmacKey))
- if err != nil {
- return err
- }
- fmt.Fprintf(w, "Successfully computed and stored request counts.\n")
- return nil
-}
-
type serverError struct {
status int // HTTP status code
err error // wrapped error
diff --git a/internal/worker/vulndb.go b/internal/worker/vulndb.go
index f681c6d..47a1b01 100644
--- a/internal/worker/vulndb.go
+++ b/internal/worker/vulndb.go
@@ -7,15 +7,85 @@
import (
"context"
"encoding/json"
+ "errors"
"os"
"path/filepath"
"strings"
+ "fmt"
+ "net/http"
+
"cloud.google.com/go/storage"
"golang.org/x/vuln/osv"
"google.golang.org/api/iterator"
+
+ "golang.org/x/pkgsite-metrics/internal"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/vulndb"
+ "golang.org/x/pkgsite-metrics/internal/vulndbreqs"
)
+func (s *Server) handleComputeRequests(w http.ResponseWriter, r *http.Request) (err error) {
+ defer derrors.Wrap(&err, "handleComputeRequests")
+
+ ctx := r.Context()
+ // Don't use the Server's BigQuery client: it's for the wrong
+ // dataset.
+ vClient, err := bigquery.NewClientCreate(ctx, s.cfg.ProjectID, vulndbreqs.DatasetName)
+ if err != nil {
+ return err
+ }
+ keyName := "projects/" + s.cfg.ProjectID + "/secrets/vulndb-hmac-key"
+ hmacKey, err := internal.GetSecret(ctx, keyName)
+ if err != nil {
+ return err
+ }
+ err = vulndbreqs.ComputeAndStore(ctx, s.cfg.VulnDBBucketProjectID, vClient, []byte(hmacKey))
+ if err != nil {
+ return err
+ }
+ fmt.Fprintf(w, "Successfully computed and stored request counts.\n")
+ return nil
+}
+
+func (s *Server) handleVulnDB(w http.ResponseWriter, r *http.Request) (err error) {
+ defer derrors.Wrap(&err, "handleVulnDB")
+
+ ctx := r.Context()
+ dbClient, err := bigquery.NewClientCreate(ctx, s.cfg.ProjectID, vulndb.DatasetName)
+ if err != nil {
+ return err
+ }
+
+ c, err := storage.NewClient(ctx)
+ if err != nil {
+ return err
+ }
+ bucket := c.Bucket("go-vulndb")
+ if bucket == nil {
+ return errors.New("failed to create go-vulndb bucket")
+ }
+ entries, err := vulndbEntries(ctx, bucket)
+ if err != nil {
+ return err
+ }
+
+ return bigquery.UploadMany(ctx, dbClient, vulndb.TableName, entries, 10000)
+}
+
+func vulndbEntries(ctx context.Context, bucket *storage.BucketHandle) ([]*vulndb.Entry, error) {
+ osvEntries, err := allVulnerabilities(ctx, bucket)
+ if err != nil {
+ return nil, err
+ }
+ var entries []*vulndb.Entry
+ for _, oe := range osvEntries {
+ entries = append(entries, vulndb.Convert(oe))
+ }
+ return entries, nil
+}
+
// gcsOSVPrefix is the directory under which .json
// files with OSV entries are located.
const gcsOSVPrefix = "ID"
diff --git a/internal/worker/vulndb_test.go b/internal/worker/vulndb_test.go
index c6c2b3f..98c2538 100644
--- a/internal/worker/vulndb_test.go
+++ b/internal/worker/vulndb_test.go
@@ -1,4 +1,4 @@
-// Copyright 2022 The Go Authors. All rights reserved.
+// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
