internal/vulncheck: vulncheck data structures
This CL follows the
https://go-review.git.corp.google.com/c/pkgsite-metrics/+/472657.
There are many modifications, but they are just pushing things around.
This will also help when simplifying vulncheck support in the future.
Change-Id: I3a58d4de8f4ee459ee708f077a45c829217153ab
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/473835
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/internal/bigquery/bigquery.go b/internal/bigquery/bigquery.go
index e02d269..c51248b 100644
--- a/internal/bigquery/bigquery.go
+++ b/internal/bigquery/bigquery.go
@@ -91,6 +91,11 @@
return gerr.Code == code
}
+// Dataset returns the underlying client dataset.
+func (c *Client) Dataset() *bq.Dataset {
+ return c.dataset
+}
+
// Table returns a handle for the given tableID in the client's dataset.
func (c *Client) Table(tableID string) *bq.Table {
return c.dataset.Table(tableID)
@@ -106,7 +111,7 @@
// CreateTable creates a table with the given name if it doesn't exist.
func (c *Client) CreateTable(ctx context.Context, tableID string) (err error) {
defer derrors.Wrap(&err, "CreateTable(%q)", tableID)
- schema := tableSchema(tableID)
+ schema := TableSchema(tableID)
if schema == nil {
return fmt.Errorf("no schema registered for table %q", tableID)
}
@@ -128,7 +133,7 @@
}
return true, c.CreateTable(ctx, tableID)
}
- schema := tableSchema(tableID)
+ schema := TableSchema(tableID)
if schema == nil {
return false, fmt.Errorf("no schema registered for table %q", tableID)
}
@@ -243,12 +248,12 @@
// SchemaVersion computes a relatively short string from a schema, such that
// different schemas result in different strings with high probability.
func SchemaVersion(schema bq.Schema) string {
- hash := sha256.Sum256([]byte(schemaString(schema)))
+ hash := sha256.Sum256([]byte(SchemaString(schema)))
return hex.EncodeToString(hash[:])
}
-// schemaString returns a long, human-readable string summarizing schema.
-func schemaString(schema bq.Schema) string {
+// SchemaString returns a long, human-readable string summarizing schema.
+func SchemaString(schema bq.Schema) string {
var b strings.Builder
for i, field := range schema {
if i > 0 {
@@ -263,7 +268,7 @@
}
b.WriteByte(':')
if field.Type == bq.RecordFieldType {
- fmt.Fprintf(&b, "(%s)", schemaString(field.Schema))
+ fmt.Fprintf(&b, "(%s)", SchemaString(field.Schema))
} else {
b.WriteString(string(field.Type))
}
@@ -282,9 +287,9 @@
tables[tableID] = s
}
-// tableSchema returns the schema associated with the given table,
+// TableSchema returns the schema associated with the given table,
// or nil if there is none.
-func tableSchema(tableID string) bq.Schema {
+func TableSchema(tableID string) bq.Schema {
tableMu.Lock()
defer tableMu.Unlock()
return tables[tableID]
diff --git a/internal/bigquery/bigquery_test.go b/internal/bigquery/bigquery_test.go
index 908ac14..559b45e 100644
--- a/internal/bigquery/bigquery_test.go
+++ b/internal/bigquery/bigquery_test.go
@@ -8,7 +8,6 @@
"context"
"flag"
"fmt"
- "sort"
"testing"
"time"
@@ -17,8 +16,6 @@
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/exp/slices"
- "golang.org/x/pkgsite-metrics/internal/version"
- "google.golang.org/api/iterator"
)
var integration = flag.Bool("integration", false, "test against actual service")
@@ -48,155 +45,6 @@
must(client.dataset.Delete(ctx))
}()
- if _, err := client.CreateOrUpdateTable(ctx, VulncheckTableName); err != nil {
- t.Fatal(err)
- }
- defer func() { must(client.Table(VulncheckTableName).Delete(ctx)) }()
-
- tm := time.Date(2022, 7, 21, 0, 0, 0, 0, time.UTC)
- row := &VulnResult{
- ModulePath: "m",
- Version: "v",
- SortVersion: "sv",
- ImportedBy: 10,
- VulncheckWorkVersion: VulncheckWorkVersion{
- WorkerVersion: "1",
- SchemaVersion: "s",
- VulnVersion: "2",
- VulnDBLastModified: tm,
- },
- }
-
- t.Run("upload", func(t *testing.T) {
- must(client.Upload(ctx, VulncheckTableName, row))
- // Round, strip monotonic data and convert to UTC.
- // Discrepancies of a few microseconds have been seen, so round to seconds
- // just to be safe.
- row.CreatedAt = row.CreatedAt.Round(time.Second).UTC()
- gots, err := readTable[VulnResult](ctx, client.Table(VulncheckTableName), nil)
- if err != nil {
- t.Fatal(err)
- }
- if g, w := len(gots), 1; g != w {
- t.Fatalf("got %d, rows, wanted %d", g, w)
- }
- got := gots[0]
- got.CreatedAt = got.CreatedAt.Round(time.Second)
- if diff := cmp.Diff(row, got); diff != "" {
- t.Errorf("mismatch (-want, +got):\n%s", diff)
- }
- })
- t.Run("work versions", func(t *testing.T) {
- wv, err := ReadVulncheckWorkVersions(ctx, client)
- if err != nil {
- t.Fatal(err)
- }
- wgot := wv[[2]string{"m", "v"}]
- if wgot == nil {
- t.Fatal("got nil, wanted work version")
- }
- if want := &row.VulncheckWorkVersion; !wgot.Equal(want) {
- t.Errorf("got %+v, want %+v", wgot, want)
- }
-
- if got := wv[[2]string{"m", "v2"}]; got != nil {
- t.Errorf("got %v; want nil", got)
- }
- })
-
- t.Run("latest", func(t *testing.T) {
- latestTableID := VulncheckTableName + "-latest"
- AddTable(latestTableID, tableSchema(VulncheckTableName))
- must(client.CreateTable(ctx, latestTableID))
- defer func() { must(client.Table(latestTableID).Delete(ctx)) }()
-
- var want []*VulnResult
- // Module "a": same work version, should get the latest module version.
- a1 := &VulnResult{
- ModulePath: "a",
- Version: "v1.0.0",
- ScanMode: "M1",
- VulncheckWorkVersion: VulncheckWorkVersion{
- WorkerVersion: "1",
- SchemaVersion: "s",
- VulnVersion: "2",
- VulnDBLastModified: tm,
- },
- }
- a2 := *a1
- a2.Version = "v1.1.0"
- want = append(want, &a2)
-
- // Different scan mode: should get this one too.
- a3 := a2
- a3.ScanMode = "M2"
- want = append(want, &a3)
-
- // Module "b": same module version, should get the latest work version.
- b1 := &VulnResult{
- ModulePath: "b",
- Version: "v1.0.0",
- VulncheckWorkVersion: VulncheckWorkVersion{
- WorkerVersion: "1",
- SchemaVersion: "s",
- VulnVersion: "2",
- VulnDBLastModified: tm,
- },
- }
- b2 := *b1
- b2.WorkerVersion = "0"
- want = append(want, b1)
-
- vrs := []*VulnResult{
- a1, &a2, &a3,
- b1, &b2,
- }
- for _, vr := range vrs {
- vr.SortVersion = version.ForSorting(vr.Version)
- }
- must(UploadMany(ctx, client, latestTableID, vrs, 20))
-
- got, err := fetchVulncheckResults(ctx, client, latestTableID)
- if err != nil {
- t.Fatal(err)
- }
- sort.Slice(got, func(i, j int) bool { return got[i].ModulePath < got[j].ModulePath })
- if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(VulnResult{}, "CreatedAt")); diff != "" {
- t.Errorf("mismatch (-want, +got):\n%s", diff)
- }
-
- // Test InsertVulncheckResults
- reportTableID := latestTableID + "-report"
- AddTable(reportTableID, tableSchema(VulncheckTableName+"-report"))
- reportTable := client.dataset.Table(reportTableID)
- // Table is created by InsertVulncheckResults.
- defer func() { must(reportTable.Delete(ctx)) }()
-
- if err := insertVulncheckResults(ctx, client, reportTableID, got, civil.DateOf(time.Now()), false); err != nil {
- t.Fatal(err)
- }
- rgot, err := readTable[ReportVulnResult](ctx, reportTable, func() *ReportVulnResult {
- return &ReportVulnResult{VulnResult: &VulnResult{}}
- })
- if err != nil {
- t.Fatal(err)
- }
- wantDate := civil.DateOf(time.Now())
- for _, r := range rgot {
- if r.ReportDate != wantDate {
- t.Errorf("got %s, want %s", r.ReportDate, wantDate)
- }
- if d := time.Minute; time.Since(r.InsertedAt) > d {
- t.Errorf("inserted at %s, more than %s ago", r.InsertedAt, d)
- }
- // Sanity check for VulnResult.
- if r.ModulePath != "a" && r.ModulePath != "b" {
- t.Errorf("got %q, want 'a' or 'b'", r.ModulePath)
- }
-
- }
-
- })
t.Run("request counts", func(t *testing.T) {
date := func(y, m, d int) civil.Date {
return civil.Date{Year: y, Month: time.Month(m), Day: d}
@@ -232,26 +80,6 @@
}
-func readTable[T any](ctx context.Context, table *bq.Table, newT func() *T) ([]*T, error) {
- var ts []*T
- if newT == nil {
- newT = func() *T { return new(T) }
- }
- iter := table.Read(ctx)
- for {
- tp := newT()
- err := iter.Next(tp)
- if err == iterator.Done {
- break
- }
- if err != nil {
- return nil, err
- }
- ts = append(ts, tp)
- }
- return ts, nil
-}
-
func TestIsNotFoundError(t *testing.T) {
if !*integration {
t.Skip("missing -integration")
diff --git a/internal/bigquery/vulncheck.go b/internal/bigquery/vulncheck.go
deleted file mode 100644
index 51b58ac..0000000
--- a/internal/bigquery/vulncheck.go
+++ /dev/null
@@ -1,235 +0,0 @@
-// Copyright 2022 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bigquery
-
-import (
- "context"
- "fmt"
- "sort"
- "time"
-
- bq "cloud.google.com/go/bigquery"
- "cloud.google.com/go/civil"
- "golang.org/x/exp/maps"
- "golang.org/x/pkgsite-metrics/internal/derrors"
- "golang.org/x/pkgsite-metrics/internal/log"
- "google.golang.org/api/iterator"
-)
-
-const VulncheckTableName = "vulncheck"
-
-// Note: before modifying VulnResult or Vuln, make sure the change
-// is a valid schema modification.
-// The only supported changes are:
-// - adding a nullable or repeated column
-// - dropping a column
-// - changing a column from required to nullable.
-// See https://cloud.google.com/bigquery/docs/managing-table-schemas for details.
-
-// VulnResult is a row in the BigQuery vulncheck table. It corresponds to a
-// result from the output for vulncheck.Source.
-type VulnResult struct {
- CreatedAt time.Time `bigquery:"created_at"`
- ModulePath string `bigquery:"module_path"`
- Version string `bigquery:"version"`
- Suffix string `bigquery:"suffix"`
- SortVersion string `bigquery:"sort_version"`
- ImportedBy int `bigquery:"imported_by"`
- Error string `bigquery:"error"`
- ErrorCategory string `bigquery:"error_category"`
- CommitTime time.Time `bigquery:"commit_time"`
- ScanSeconds float64 `bigquery:"scan_seconds"`
- ScanMemory int64 `bigquery:"scan_memory"`
- PkgsMemory int64 `bigquery:"pkgs_memory"`
- ScanMode string `bigquery:"scan_mode"`
- // Workers is the concurrency limit under which a module is
- // analyzed. Useful for interpreting memory measurements when
- // there are multiple modules analyzed in the same process.
- // 0 if no limit is specified, -1 for potential errors.
- Workers int `bigquery:"workers"`
- VulncheckWorkVersion // InferSchema flattens embedded fields
- Vulns []*Vuln `bigquery:"vulns"`
-}
-
-// VulncheckWorkVersion contains information that can be used to avoid duplicate work.
-// Given two VulncheckWorkVersion values v1 and v2 for the same module path and version,
-// if v1.Equal(v2) then it is not necessary to scan the module.
-type VulncheckWorkVersion struct {
- // The version of the currently running code. This tracks changes in the
- // logic of module scanning and processing.
- WorkerVersion string `bigquery:"worker_version"`
- // The version of the bigquery schema.
- SchemaVersion string ` bigquery:"schema_version"`
- // The version of the golang.org/x/vuln module used by the current module.
- VulnVersion string `bigquery:"x_vuln_version"`
- // When the vuln DB was last modified.
- VulnDBLastModified time.Time `bigquery:"vulndb_last_modified"`
-}
-
-func (v1 *VulncheckWorkVersion) Equal(v2 *VulncheckWorkVersion) bool {
- if v1 == nil || v2 == nil {
- return v1 == v2
- }
- return v1.WorkerVersion == v2.WorkerVersion &&
- v1.SchemaVersion == v2.SchemaVersion &&
- v1.VulnVersion == v2.VulnVersion &&
- v1.VulnDBLastModified.Equal(v2.VulnDBLastModified)
-}
-
-func (vr *VulnResult) SetUploadTime(t time.Time) { vr.CreatedAt = t }
-
-func (vr *VulnResult) AddError(err error) {
- if err == nil {
- return
- }
- vr.Error = err.Error()
- vr.ErrorCategory = derrors.CategorizeError(err)
-}
-
-// Vuln is a record in VulnResult and corresponds to an item in
-// vulncheck.Result.Vulns.
-type Vuln struct {
- ID string `bigquery:"id"`
- Symbol string `bigquery:"symbol"`
- PackagePath string `bigquery:"package_path"`
- ModulePath string `bigquery:"module_path"`
- CallSink bq.NullInt64 `bigquery:"call_sink"`
- ImportSink bq.NullInt64 `bigquery:"import_sink"`
- RequireSink bq.NullInt64 `bigquery:"require_sink"`
-}
-
-// VulncheckSchemaVersion changes whenever the vulncheck schema changes.
-var VulncheckSchemaVersion string
-
-func init() {
- s, err := bq.InferSchema(VulnResult{})
- if err != nil {
- panic(err)
- }
- VulncheckSchemaVersion = SchemaVersion(s)
- AddTable(VulncheckTableName, s)
-}
-
-// ReadVulncheckWorkVersions reads the most recent WorkVersions in the vulncheck table.
-func ReadVulncheckWorkVersions(ctx context.Context, c *Client) (_ map[[2]string]*VulncheckWorkVersion, err error) {
- defer derrors.Wrap(&err, "ReadVulncheckWorkVersions")
- m := map[[2]string]*VulncheckWorkVersion{}
- query := PartitionQuery(c.FullTableName(VulncheckTableName), "module_path, sort_version", "created_at DESC")
- iter, err := c.Query(ctx, query)
- if err != nil {
- return nil, err
- }
- err = ForEachRow(iter, func(r *VulnResult) bool {
- m[[2]string{r.ModulePath, r.Version}] = &r.VulncheckWorkVersion
- return true
- })
- if err != nil {
- return nil, err
- }
- return m, nil
-}
-
-// The module path along with the four sort columns should uniquely specify a
-// row, because we do not generate a new row for a (module, version) if the
-// other three versions are identical. (There is actually a fourth component of
-// the work version, the schema version. But since it is represented by a struct
-// in the worker code and the worker version captures every change to that code,
-// it cannot change independently of worker_version.)
-const orderByClauses = `
- vulndb_last_modified DESC, -- latest version of database
- x_vuln_version DESC, -- latest version of x/vuln
- worker_version DESC, -- latest version of x/pkgsite-metrics
- sort_version DESC, -- latest version of module
- created_at DESC -- latest insertion time
-`
-
-func FetchVulncheckResults(ctx context.Context, c *Client) (rows []*VulnResult, err error) {
- return fetchVulncheckResults(ctx, c, VulncheckTableName)
-}
-
-func fetchVulncheckResults(ctx context.Context, c *Client, tableName string) (rows []*VulnResult, err error) {
- name := c.FullTableName(tableName)
- query := PartitionQuery(name, "module_path, scan_mode", orderByClauses)
- log.Infof(ctx, "running latest query on %s", name)
- iter, err := c.Query(ctx, query)
- if err != nil {
- return nil, err
- }
- rows, err = All[VulnResult](iter)
- if err != nil {
- return nil, err
- }
- log.Infof(ctx, "got %d rows", len(rows))
-
- // Check for duplicate rows.
- modvers := map[string]int{}
- for _, r := range rows {
- modvers[r.ModulePath+"@"+r.Version+" "+r.ScanMode]++
- }
- keys := maps.Keys(modvers)
- sort.Strings(keys)
- for _, k := range keys {
- if n := modvers[k]; n > 1 {
- return nil, fmt.Errorf("%s has %d rows", k, n)
- }
- }
- return rows, nil
-}
-
-type ReportVulnResult struct {
- *VulnResult
- ReportDate civil.Date `bigquery:"report_date"` // for reporting (e.g. dashboard)
- InsertedAt time.Time `bigquery:"inserted_at"` // to disambiguate if >1 insertion for same date
-}
-
-func init() {
- s, err := bq.InferSchema(ReportVulnResult{})
- if err != nil {
- panic(err)
- }
- AddTable(VulncheckTableName+"-report", s)
-}
-
-func InsertVulncheckResults(ctx context.Context, c *Client, results []*VulnResult, date civil.Date, allowDuplicates bool) (err error) {
- return insertVulncheckResults(ctx, c, VulncheckTableName+"-report", results, date, allowDuplicates)
-}
-
-func insertVulncheckResults(ctx context.Context, c *Client, reportTableName string, results []*VulnResult, date civil.Date, allowDuplicates bool) (err error) {
- derrors.Wrap(&err, "InsertVulncheckResults(%s)", date)
- // Create the report table if it doesn't exist.
- if err := c.CreateTable(ctx, reportTableName); err != nil {
- return err
- }
-
- if !allowDuplicates {
- query := fmt.Sprintf("SELECT COUNT(*) FROM `%s` WHERE report_date = '%s'",
- c.FullTableName(reportTableName), date)
- iter, err := c.Query(ctx, query)
- if err != nil {
- return err
- }
- var count struct {
- n int
- }
- err = iter.Next(&count)
- if err != nil && err != iterator.Done {
- return err
- }
- if count.n > 0 {
- return fmt.Errorf("already have %d rows for %s", count.n, date)
- }
- }
-
- now := time.Now()
- var rows []ReportVulnResult
- for _, r := range results {
- rows = append(rows, ReportVulnResult{VulnResult: r, ReportDate: date, InsertedAt: now})
- }
-
- ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) // to avoid retrying forever on permanent errors
- defer cancel()
- const chunkSize = 1024 // Chunk rows to a void exceeding the maximum allowable request size.
- return UploadMany(ctx, c, reportTableName, rows, chunkSize)
-}
diff --git a/internal/bigquery/vulncheck_test.go b/internal/bigquery/vulncheck_test.go
deleted file mode 100644
index 658ca84..0000000
--- a/internal/bigquery/vulncheck_test.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2022 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bigquery
-
-import (
- "testing"
-
- bq "cloud.google.com/go/bigquery"
-)
-
-func TestSchemaString(t *testing.T) {
- type nest struct {
- N []byte
- M float64
- }
-
- type s struct {
- A string
- B int
- C []bool
- D nest
- }
- const want = "A,req:STRING;B,req:INTEGER;C,rep:BOOLEAN;D,req:(N,req:BYTES;M,req:FLOAT)"
- schema, err := bq.InferSchema(s{})
- if err != nil {
- t.Fatal(err)
- }
- got := schemaString(schema)
- if got != want {
- t.Errorf("\ngot %q\nwant %q", got, want)
- }
-}
diff --git a/internal/vulncheck/vulncheck.go b/internal/vulncheck/vulncheck.go
new file mode 100644
index 0000000..1ba9019
--- /dev/null
+++ b/internal/vulncheck/vulncheck.go
@@ -0,0 +1,358 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vulncheck
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "sort"
+ "strings"
+ "time"
+
+ bq "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+ "golang.org/x/exp/maps"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "golang.org/x/pkgsite-metrics/internal/scan"
+ "golang.org/x/vuln/exp/govulncheck"
+ "golang.org/x/vuln/vulncheck"
+ "google.golang.org/api/iterator"
+)
+
+// EnqueueQueryParams for vulncheck/enqueue
+type EnqueueQueryParams struct {
+ Suffix string // appended to task queue IDs to generate unique tasks
+ Mode string // type of analysis to run
+ Min int // minimum import-by count for a module to be included
+ File string // path to file containing modules; if missing, use DB
+}
+
+// Request contains information passed
+// to a scan endpoint.
+type Request struct {
+ scan.ModuleURLPath
+ QueryParams
+}
+
+// QueryParams has query parameters for a vulncheck scan request.
+type QueryParams struct {
+ ImportedBy int // imported-by count
+ Mode string // vulncheck mode (VTA, etc)
+ Insecure bool // if true, run outside sandbox
+ Serve bool // serve results back to client instead of writing them to BigQuery
+}
+
+// These methods implement queue.Task.
+func (r *Request) Name() string { return r.Module + "@" + r.Version }
+
+func (r *Request) Path() string { return r.ModuleURLPath.Path() }
+
+func (r *Request) Params() string {
+ return scan.FormatParams(r.QueryParams)
+}
+
+// ParseRequest parses an http request r for an endpoint
+// prefix and produces a corresponding ScanRequest.
+//
+// The module and version should have one of the following three forms:
+// - <module>/@v/<version>
+// - <module>@<version>
+// - <module>/@latest
+//
+// (These are the same forms that the module proxy accepts.)
+func ParseRequest(r *http.Request, prefix string) (*Request, error) {
+ mp, err := scan.ParseModuleURLPath(strings.TrimPrefix(r.URL.Path, prefix))
+ if err != nil {
+ return nil, err
+ }
+
+ rp := QueryParams{ImportedBy: -1}
+ if err := scan.ParseParams(r, &rp); err != nil {
+ return nil, err
+ }
+ if rp.ImportedBy < 0 {
+ return nil, errors.New(`missing or negative "importedby" query param`)
+ }
+ return &Request{
+ ModuleURLPath: mp,
+ QueryParams: rp,
+ }, nil
+}
+
+func ConvertVulncheckOutput(v *vulncheck.Vuln) *Vuln {
+ return &Vuln{
+ ID: v.OSV.ID,
+ ModulePath: v.ModPath,
+ PackagePath: v.PkgPath,
+ Symbol: v.Symbol,
+ CallSink: bigquery.NullInt(v.CallSink),
+ ImportSink: bigquery.NullInt(v.ImportSink),
+ RequireSink: bigquery.NullInt(v.RequireSink),
+ }
+}
+
+func ConvertGovulncheckOutput(v *govulncheck.Vuln) (vulns []*Vuln) {
+ for _, module := range v.Modules {
+ for pkgNum, pkg := range module.Packages {
+ addedSymbols := make(map[string]bool)
+ baseVuln := &Vuln{
+ ID: v.OSV.ID,
+ ModulePath: module.Path,
+ PackagePath: pkg.Path,
+ CallSink: bigquery.NullInt(0),
+ ImportSink: bigquery.NullInt(pkgNum + 1),
+ RequireSink: bigquery.NullInt(pkgNum + 1),
+ }
+
+ // For each called symbol, reconstruct sinks and create the corresponding bigquery vuln
+ for symbolNum, cs := range pkg.CallStacks {
+ addedSymbols[cs.Symbol] = true
+ toAdd := *baseVuln
+ toAdd.Symbol = cs.Symbol
+ toAdd.CallSink = bigquery.NullInt(symbolNum + 1)
+ vulns = append(vulns, &toAdd)
+ }
+
+ // Find the rest of the vulnerable imported symbols that haven't been called
+ // and create corresponding bigquery vulns
+ for _, affected := range v.OSV.Affected {
+ if affected.Package.Name == module.Path {
+ for _, imp := range affected.EcosystemSpecific.Imports {
+ if imp.Path == pkg.Path {
+ for _, symbol := range imp.Symbols {
+ if !addedSymbols[symbol] {
+ toAdd := *baseVuln
+ toAdd.Symbol = symbol
+ vulns = append(vulns, &toAdd)
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return vulns
+}
+
+const TableName = "vulncheck"
+
+// Note: before modifying Result or Vuln, make sure the change
+// is a valid schema modification.
+// The only supported changes are:
+// - adding a nullable or repeated column
+// - dropping a column
+// - changing a column from required to nullable.
+// See https://cloud.google.com/bigquery/docs/managing-table-schemas for details.
+
+// Result is a row in the BigQuery vulncheck table. It corresponds to a
+// result from the output for vulncheck.Source.
+type Result struct {
+ CreatedAt time.Time `bigquery:"created_at"`
+ ModulePath string `bigquery:"module_path"`
+ Version string `bigquery:"version"`
+ Suffix string `bigquery:"suffix"`
+ SortVersion string `bigquery:"sort_version"`
+ ImportedBy int `bigquery:"imported_by"`
+ Error string `bigquery:"error"`
+ ErrorCategory string `bigquery:"error_category"`
+ CommitTime time.Time `bigquery:"commit_time"`
+ ScanSeconds float64 `bigquery:"scan_seconds"`
+ ScanMemory int64 `bigquery:"scan_memory"`
+ PkgsMemory int64 `bigquery:"pkgs_memory"`
+ ScanMode string `bigquery:"scan_mode"`
+ // Workers is the concurrency limit under which a module is
+ // analyzed. Useful for interpreting memory measurements when
+ // there are multiple modules analyzed in the same process.
+ // 0 if no limit is specified, -1 for potential errors.
+ Workers int `bigquery:"workers"`
+ WorkVersion // InferSchema flattens embedded fields
+ Vulns []*Vuln `bigquery:"vulns"`
+}
+
+// WorkVersion contains information that can be used to avoid duplicate work.
+// Given two WorkVersion values v1 and v2 for the same module path and version,
+// if v1.Equal(v2) then it is not necessary to scan the module.
+type WorkVersion struct {
+ // The version of the currently running code. This tracks changes in the
+ // logic of module scanning and processing.
+ WorkerVersion string `bigquery:"worker_version"`
+ // The version of the bigquery schema.
+ SchemaVersion string ` bigquery:"schema_version"`
+ // The version of the golang.org/x/vuln module used by the current module.
+ VulnVersion string `bigquery:"x_vuln_version"`
+ // When the vuln DB was last modified.
+ VulnDBLastModified time.Time `bigquery:"vulndb_last_modified"`
+}
+
+func (v1 *WorkVersion) Equal(v2 *WorkVersion) bool {
+ if v1 == nil || v2 == nil {
+ return v1 == v2
+ }
+ return v1.WorkerVersion == v2.WorkerVersion &&
+ v1.SchemaVersion == v2.SchemaVersion &&
+ v1.VulnVersion == v2.VulnVersion &&
+ v1.VulnDBLastModified.Equal(v2.VulnDBLastModified)
+}
+
+func (vr *Result) SetUploadTime(t time.Time) { vr.CreatedAt = t }
+
+func (vr *Result) AddError(err error) {
+ if err == nil {
+ return
+ }
+ vr.Error = err.Error()
+ vr.ErrorCategory = derrors.CategorizeError(err)
+}
+
+// Vuln is a record in Result and corresponds to an item in
+// vulncheck.Result.Vulns.
+type Vuln struct {
+ ID string `bigquery:"id"`
+ Symbol string `bigquery:"symbol"`
+ PackagePath string `bigquery:"package_path"`
+ ModulePath string `bigquery:"module_path"`
+ CallSink bq.NullInt64 `bigquery:"call_sink"`
+ ImportSink bq.NullInt64 `bigquery:"import_sink"`
+ RequireSink bq.NullInt64 `bigquery:"require_sink"`
+}
+
+// SchemaVersion changes whenever the vulncheck schema changes.
+var SchemaVersion string
+
+func init() {
+ s, err := bigquery.InferSchema(Result{})
+ if err != nil {
+ panic(err)
+ }
+ SchemaVersion = bigquery.SchemaVersion(s)
+ bigquery.AddTable(TableName, s)
+}
+
+// ReadWorkVersions reads the most recent WorkVersions in the vulncheck table.
+func ReadWorkVersions(ctx context.Context, c *bigquery.Client) (_ map[[2]string]*WorkVersion, err error) {
+ defer derrors.Wrap(&err, "ReadWorkVersions")
+ m := map[[2]string]*WorkVersion{}
+ query := bigquery.PartitionQuery(c.FullTableName(TableName), "module_path, sort_version", "created_at DESC")
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return nil, err
+ }
+ err = bigquery.ForEachRow(iter, func(r *Result) bool {
+ m[[2]string{r.ModulePath, r.Version}] = &r.WorkVersion
+ return true
+ })
+ if err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// The module path along with the four sort columns should uniquely specify a
+// row, because we do not generate a new row for a (module, version) if the
+// other three versions are identical. (There is actually a fourth component of
+// the work version, the schema version. But since it is represented by a struct
+// in the worker code and the worker version captures every change to that code,
+// it cannot change independently of worker_version.)
+const orderByClauses = `
+ vulndb_last_modified DESC, -- latest version of database
+ x_vuln_version DESC, -- latest version of x/vuln
+ worker_version DESC, -- latest version of x/pkgsite-metrics
+ sort_version DESC, -- latest version of module
+ created_at DESC -- latest insertion time
+`
+
+func FetchResults(ctx context.Context, c *bigquery.Client) (rows []*Result, err error) {
+ return fetchResults(ctx, c, TableName)
+}
+
+func fetchResults(ctx context.Context, c *bigquery.Client, tableName string) (rows []*Result, err error) {
+ name := c.FullTableName(tableName)
+ query := bigquery.PartitionQuery(name, "module_path, scan_mode", orderByClauses)
+ log.Infof(ctx, "running latest query on %s", name)
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return nil, err
+ }
+ rows, err = bigquery.All[Result](iter)
+ if err != nil {
+ return nil, err
+ }
+ log.Infof(ctx, "got %d rows", len(rows))
+
+ // Check for duplicate rows.
+ modvers := map[string]int{}
+ for _, r := range rows {
+ modvers[r.ModulePath+"@"+r.Version+" "+r.ScanMode]++
+ }
+ keys := maps.Keys(modvers)
+ sort.Strings(keys)
+ for _, k := range keys {
+ if n := modvers[k]; n > 1 {
+ return nil, fmt.Errorf("%s has %d rows", k, n)
+ }
+ }
+ return rows, nil
+}
+
+type ReportResult struct {
+ *Result
+ ReportDate civil.Date `bigquery:"report_date"` // for reporting (e.g. dashboard)
+ InsertedAt time.Time `bigquery:"inserted_at"` // to disambiguate if >1 insertion for same date
+}
+
+func init() {
+ s, err := bigquery.InferSchema(ReportResult{})
+ if err != nil {
+ panic(err)
+ }
+ bigquery.AddTable(TableName+"-report", s)
+}
+
+func InsertResults(ctx context.Context, c *bigquery.Client, results []*Result, date civil.Date, allowDuplicates bool) (err error) {
+ return insertResults(ctx, c, TableName+"-report", results, date, allowDuplicates)
+}
+
+func insertResults(ctx context.Context, c *bigquery.Client, reportTableName string, results []*Result, date civil.Date, allowDuplicates bool) (err error) {
+ derrors.Wrap(&err, "InsertResults(%s)", date)
+ // Create the report table if it doesn't exist.
+ if err := c.CreateTable(ctx, reportTableName); err != nil {
+ return err
+ }
+
+ if !allowDuplicates {
+ query := fmt.Sprintf("SELECT COUNT(*) FROM `%s` WHERE report_date = '%s'",
+ c.FullTableName(reportTableName), date)
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return err
+ }
+ var count struct {
+ n int
+ }
+ err = iter.Next(&count)
+ if err != nil && err != iterator.Done {
+ return err
+ }
+ if count.n > 0 {
+ return fmt.Errorf("already have %d rows for %s", count.n, date)
+ }
+ }
+
+ now := time.Now()
+ var rows []ReportResult
+ for _, r := range results {
+ rows = append(rows, ReportResult{Result: r, ReportDate: date, InsertedAt: now})
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) // to avoid retrying forever on permanent errors
+ defer cancel()
+ const chunkSize = 1024 // Chunk rows to a void exceeding the maximum allowable request size.
+ return bigquery.UploadMany(ctx, c, reportTableName, rows, chunkSize)
+}
diff --git a/internal/vulncheck/vulncheck_test.go b/internal/vulncheck/vulncheck_test.go
new file mode 100644
index 0000000..145d23a
--- /dev/null
+++ b/internal/vulncheck/vulncheck_test.go
@@ -0,0 +1,368 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vulncheck
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "sort"
+ "testing"
+ "time"
+
+ bq "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/version"
+ "golang.org/x/vuln/exp/govulncheck"
+ "golang.org/x/vuln/osv"
+ "google.golang.org/api/iterator"
+)
+
+func TestConvertGovulncheckOutput(t *testing.T) {
+ var (
+ osvEntry = &osv.Entry{
+ ID: "GO-YYYY-1234",
+ Affected: []osv.Affected{
+ {
+ Package: osv.Package{
+ Name: "example.com/repo/module",
+ Ecosystem: "Go",
+ },
+ EcosystemSpecific: osv.EcosystemSpecific{
+ Imports: []osv.EcosystemSpecificImport{
+ {
+ Path: "example.com/repo/module/package",
+ Symbols: []string{
+ "Symbol",
+ "Another",
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ vuln1 = &govulncheck.Vuln{
+ OSV: osvEntry,
+ Modules: []*govulncheck.Module{
+ {
+ Path: "example.com/repo/module",
+ Packages: []*govulncheck.Package{
+ {
+ Path: "example.com/repo/module/package",
+ CallStacks: []govulncheck.CallStack{
+ {
+ Symbol: "Symbol",
+ Summary: "example.go:1:1 xyz.func calls pkgPath.Symbol",
+ Frames: []*govulncheck.StackFrame{},
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ vuln2 = &govulncheck.Vuln{
+ OSV: osvEntry,
+ Modules: []*govulncheck.Module{
+ {
+ Path: "example.com/repo/module",
+ Packages: []*govulncheck.Package{
+ {
+ Path: "example.com/repo/module/package",
+ },
+ },
+ },
+ },
+ }
+ )
+ tests := []struct {
+ name string
+ vuln *govulncheck.Vuln
+ wantVulns []*Vuln
+ }{
+ {
+ name: "Call One Symbol",
+ vuln: vuln1,
+ wantVulns: []*Vuln{
+ {
+ ID: "GO-YYYY-1234",
+ Symbol: "Symbol",
+ PackagePath: "example.com/repo/module/package",
+ ModulePath: "example.com/repo/module",
+ CallSink: bigquery.NullInt(1),
+ ImportSink: bigquery.NullInt(1),
+ RequireSink: bigquery.NullInt(1),
+ },
+ {
+ ID: "GO-YYYY-1234",
+ Symbol: "Another",
+ PackagePath: "example.com/repo/module/package",
+ ModulePath: "example.com/repo/module",
+ CallSink: bigquery.NullInt(0),
+ ImportSink: bigquery.NullInt(1),
+ RequireSink: bigquery.NullInt(1),
+ },
+ },
+ },
+ {
+ name: "Call no symbols",
+ vuln: vuln2,
+ wantVulns: []*Vuln{
+ {
+ ID: "GO-YYYY-1234",
+ PackagePath: "example.com/repo/module/package",
+ ModulePath: "example.com/repo/module",
+ Symbol: "Symbol",
+ CallSink: bigquery.NullInt(0),
+ ImportSink: bigquery.NullInt(1),
+ RequireSink: bigquery.NullInt(1),
+ },
+ {
+ ID: "GO-YYYY-1234",
+ PackagePath: "example.com/repo/module/package",
+ ModulePath: "example.com/repo/module",
+ Symbol: "Another",
+ CallSink: bigquery.NullInt(0),
+ ImportSink: bigquery.NullInt(1),
+ RequireSink: bigquery.NullInt(1),
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if diff := cmp.Diff(ConvertGovulncheckOutput(tt.vuln), tt.wantVulns, cmpopts.EquateEmpty()); diff != "" {
+ t.Errorf("mismatch (-got, +want): %s", diff)
+ }
+ })
+ }
+}
+
+func TestSchemaString(t *testing.T) {
+ type nest struct {
+ N []byte
+ M float64
+ }
+
+ type s struct {
+ A string
+ B int
+ C []bool
+ D nest
+ }
+ const want = "A,req:STRING;B,req:INTEGER;C,rep:BOOLEAN;D,req:(N,req:BYTES;M,req:FLOAT)"
+ schema, err := bigquery.InferSchema(s{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := bigquery.SchemaString(schema)
+ if got != want {
+ t.Errorf("\ngot %q\nwant %q", got, want)
+ }
+}
+
+var integration = flag.Bool("integration", false, "test against actual service")
+
+func TestIntegration(t *testing.T) {
+ must := func(err error) {
+ t.Helper()
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if !*integration {
+ t.Skip("missing -integration")
+ }
+ ctx := context.Background()
+ const projectID = "go-ecosystem"
+
+ // Create a new dataset ID to avoid problems with re-using existing tables.
+ dsID := fmt.Sprintf("test_%s", time.Now().Format("20060102T030405"))
+ t.Logf("using dataset %s", dsID)
+ client, err := bigquery.NewClientCreate(ctx, projectID, dsID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ must(client.Dataset().Delete(ctx))
+ }()
+
+ if _, err := client.CreateOrUpdateTable(ctx, TableName); err != nil {
+ t.Fatal(err)
+ }
+ defer func() { must(client.Table(TableName).Delete(ctx)) }()
+
+ tm := time.Date(2022, 7, 21, 0, 0, 0, 0, time.UTC)
+ row := &Result{
+ ModulePath: "m",
+ Version: "v",
+ SortVersion: "sv",
+ ImportedBy: 10,
+ WorkVersion: WorkVersion{
+ WorkerVersion: "1",
+ SchemaVersion: "s",
+ VulnVersion: "2",
+ VulnDBLastModified: tm,
+ },
+ }
+
+ t.Run("upload", func(t *testing.T) {
+ must(client.Upload(ctx, TableName, row))
+ // Round, strip monotonic data and convert to UTC.
+ // Discrepancies of a few microseconds have been seen, so round to seconds
+ // just to be safe.
+ row.CreatedAt = row.CreatedAt.Round(time.Second).UTC()
+ gots, err := readTable[Result](ctx, client.Table(TableName), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if g, w := len(gots), 1; g != w {
+ t.Fatalf("got %d, rows, wanted %d", g, w)
+ }
+ got := gots[0]
+ got.CreatedAt = got.CreatedAt.Round(time.Second)
+ if diff := cmp.Diff(row, got); diff != "" {
+ t.Errorf("mismatch (-want, +got):\n%s", diff)
+ }
+ })
+ t.Run("work versions", func(t *testing.T) {
+ wv, err := ReadWorkVersions(ctx, client)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wgot := wv[[2]string{"m", "v"}]
+ if wgot == nil {
+ t.Fatal("got nil, wanted work version")
+ }
+ if want := &row.WorkVersion; !wgot.Equal(want) {
+ t.Errorf("got %+v, want %+v", wgot, want)
+ }
+
+ if got := wv[[2]string{"m", "v2"}]; got != nil {
+ t.Errorf("got %v; want nil", got)
+ }
+ })
+
+ t.Run("latest", func(t *testing.T) {
+ latestTableID := TableName + "-latest"
+ bigquery.AddTable(latestTableID, bigquery.TableSchema(TableName))
+ must(client.CreateTable(ctx, latestTableID))
+ defer func() { must(client.Table(latestTableID).Delete(ctx)) }()
+
+ var want []*Result
+ // Module "a": same work version, should get the latest module version.
+ a1 := &Result{
+ ModulePath: "a",
+ Version: "v1.0.0",
+ ScanMode: "M1",
+ WorkVersion: WorkVersion{
+ WorkerVersion: "1",
+ SchemaVersion: "s",
+ VulnVersion: "2",
+ VulnDBLastModified: tm,
+ },
+ }
+ a2 := *a1
+ a2.Version = "v1.1.0"
+ want = append(want, &a2)
+
+ // Different scan mode: should get this one too.
+ a3 := a2
+ a3.ScanMode = "M2"
+ want = append(want, &a3)
+
+ // Module "b": same module version, should get the latest work version.
+ b1 := &Result{
+ ModulePath: "b",
+ Version: "v1.0.0",
+ WorkVersion: WorkVersion{
+ WorkerVersion: "1",
+ SchemaVersion: "s",
+ VulnVersion: "2",
+ VulnDBLastModified: tm,
+ },
+ }
+ b2 := *b1
+ b2.WorkerVersion = "0"
+ want = append(want, b1)
+
+ vrs := []*Result{
+ a1, &a2, &a3,
+ b1, &b2,
+ }
+ for _, vr := range vrs {
+ vr.SortVersion = version.ForSorting(vr.Version)
+ }
+ must(bigquery.UploadMany(ctx, client, latestTableID, vrs, 20))
+
+ got, err := fetchResults(ctx, client, latestTableID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sort.Slice(got, func(i, j int) bool { return got[i].ModulePath < got[j].ModulePath })
+ if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(Result{}, "CreatedAt")); diff != "" {
+ t.Errorf("mismatch (-want, +got):\n%s", diff)
+ }
+
+ // Test InsertVulncheckResults
+ reportTableID := latestTableID + "-report"
+ bigquery.AddTable(reportTableID, bigquery.TableSchema(TableName+"-report"))
+ reportTable := client.Dataset().Table(reportTableID)
+ // Table is created by InsertVulncheckResults.
+ defer func() { must(reportTable.Delete(ctx)) }()
+
+ if err := insertResults(ctx, client, reportTableID, got, civil.DateOf(time.Now()), false); err != nil {
+ t.Fatal(err)
+ }
+ rgot, err := readTable[ReportResult](ctx, reportTable, func() *ReportResult {
+ return &ReportResult{Result: &Result{}}
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantDate := civil.DateOf(time.Now())
+ for _, r := range rgot {
+ if r.ReportDate != wantDate {
+ t.Errorf("got %s, want %s", r.ReportDate, wantDate)
+ }
+ if d := time.Minute; time.Since(r.InsertedAt) > d {
+ t.Errorf("inserted at %s, more than %s ago", r.InsertedAt, d)
+ }
+ // Sanity check for Result.
+ if r.ModulePath != "a" && r.ModulePath != "b" {
+ t.Errorf("got %q, want 'a' or 'b'", r.ModulePath)
+ }
+
+ }
+
+ })
+}
+
+func readTable[T any](ctx context.Context, table *bq.Table, newT func() *T) ([]*T, error) {
+ var ts []*T
+ if newT == nil {
+ newT = func() *T { return new(T) }
+ }
+ iter := table.Read(ctx)
+ for {
+ tp := newT()
+ err := iter.Next(tp)
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ ts = append(ts, tp)
+ }
+ return ts, nil
+}
diff --git a/internal/worker/enqueue.go b/internal/worker/enqueue.go
index bc636f0..9471fc4 100644
--- a/internal/worker/enqueue.go
+++ b/internal/worker/enqueue.go
@@ -14,6 +14,7 @@
"golang.org/x/pkgsite-metrics/internal/pkgsitedb"
"golang.org/x/pkgsite-metrics/internal/queue"
"golang.org/x/pkgsite-metrics/internal/scan"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
const defaultMinImportedByCount = 10
@@ -72,15 +73,15 @@
return nil
}
-func moduleSpecsToScanRequests(modspecs []scan.ModuleSpec, mode string) []*vulncheckRequest {
- var sreqs []*vulncheckRequest
+func moduleSpecsToScanRequests(modspecs []scan.ModuleSpec, mode string) []*ivulncheck.Request {
+ var sreqs []*ivulncheck.Request
for _, ms := range modspecs {
- sreqs = append(sreqs, &vulncheckRequest{
- scan.ModuleURLPath{
+ sreqs = append(sreqs, &ivulncheck.Request{
+ ModuleURLPath: scan.ModuleURLPath{
Module: ms.Path,
Version: ms.Version,
},
- vulncheckRequestParams{
+ QueryParams: ivulncheck.QueryParams{
ImportedBy: ms.ImportedBy,
Mode: mode,
},
diff --git a/internal/worker/vulncheck.go b/internal/worker/vulncheck.go
index 92b860a..a91926d 100644
--- a/internal/worker/vulncheck.go
+++ b/internal/worker/vulncheck.go
@@ -11,24 +11,24 @@
"net/http"
"runtime/debug"
- "golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/derrors"
"golang.org/x/pkgsite-metrics/internal/log"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
type VulncheckServer struct {
*Server
- storedWorkVersions map[[2]string]*bigquery.VulncheckWorkVersion
- workVersion *bigquery.VulncheckWorkVersion
+ storedWorkVersions map[[2]string]*ivulncheck.WorkVersion
+ workVersion *ivulncheck.WorkVersion
}
func newVulncheckServer(ctx context.Context, s *Server) (*VulncheckServer, error) {
var (
- swv map[[2]string]*bigquery.VulncheckWorkVersion
+ swv map[[2]string]*ivulncheck.WorkVersion
err error
)
if s.bqClient != nil {
- swv, err = bigquery.ReadVulncheckWorkVersions(ctx, s.bqClient)
+ swv, err = ivulncheck.ReadWorkVersions(ctx, s.bqClient)
if err != nil {
return nil, err
}
@@ -40,7 +40,7 @@
}, nil
}
-func (h *VulncheckServer) getWorkVersion(ctx context.Context) (_ *bigquery.VulncheckWorkVersion, err error) {
+func (h *VulncheckServer) getWorkVersion(ctx context.Context) (_ *ivulncheck.WorkVersion, err error) {
defer derrors.Wrap(&err, "VulncheckServer.getWorkVersion")
h.mu.Lock()
defer h.mu.Unlock()
@@ -54,10 +54,10 @@
if err != nil {
return nil, err
}
- h.workVersion = &bigquery.VulncheckWorkVersion{
+ h.workVersion = &ivulncheck.WorkVersion{
VulnDBLastModified: lmt,
WorkerVersion: h.cfg.VersionID,
- SchemaVersion: bigquery.VulncheckSchemaVersion,
+ SchemaVersion: ivulncheck.SchemaVersion,
VulnVersion: vulnVersion,
}
log.Infof(ctx, "vulncheck work version: %+v", h.workVersion)
@@ -101,11 +101,11 @@
if h.bqClient == nil {
return nil, errBQDisabled
}
- table := h.bqClient.FullTableName(bigquery.VulncheckTableName)
+ table := h.bqClient.FullTableName(ivulncheck.TableName)
page := newPage(table)
page.basePage = newBasePage()
- rows, err := bigquery.FetchVulncheckResults(ctx, h.bqClient)
+ rows, err := ivulncheck.FetchResults(ctx, h.bqClient)
if err != nil {
return nil, err
}
@@ -197,7 +197,7 @@
return (float64(v.NumModulesNoVuln()) / float64(v.NumModulesSuccess)) * 100
}
-func (r *VulncheckResult) update(row *bigquery.VulnResult) {
+func (r *VulncheckResult) update(row *ivulncheck.Result) {
r.NumModulesScanned++
if row.Error != "" {
r.NumModulesError++
@@ -243,7 +243,7 @@
// handleVulncheckRows populates page based on vulncheck result rows and
// returns statistics for each vulnerability detected.
-func handleVulncheckRows(ctx context.Context, page *VulncheckPage, rows []*bigquery.VulnResult) map[string]*ReportResult {
+func handleVulncheckRows(ctx context.Context, page *VulncheckPage, rows []*ivulncheck.Result) map[string]*ReportResult {
vulnsScanned := map[string]*ReportResult{}
for _, row := range rows {
switch row.ScanMode {
diff --git a/internal/worker/vulncheck_enqueue.go b/internal/worker/vulncheck_enqueue.go
index 18fafa3..6f80a36 100644
--- a/internal/worker/vulncheck_enqueue.go
+++ b/internal/worker/vulncheck_enqueue.go
@@ -19,17 +19,10 @@
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/queue"
"golang.org/x/pkgsite-metrics/internal/scan"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
"google.golang.org/api/iterator"
)
-// query params for vulncheck/enqueue
-type vulncheckEnqueueParams struct {
- Suffix string // appended to task queue IDs to generate unique tasks
- Mode string // type of analysis to run
- Min int // minimum import-by count for a module to be included
- File string // path to file containing modules; if missing, use DB
-}
-
// handleEnqueue enqueues multiple modules for a single vulncheck mode.
func (h *VulncheckServer) handleEnqueue(w http.ResponseWriter, r *http.Request) error {
return h.enqueue(r, false)
@@ -42,7 +35,7 @@
func (h *VulncheckServer) enqueue(r *http.Request, allModes bool) error {
ctx := r.Context()
- params := &vulncheckEnqueueParams{Min: defaultMinImportedByCount}
+ params := &ivulncheck.EnqueueQueryParams{Min: defaultMinImportedByCount}
if err := scan.ParseParams(r, params); err != nil {
return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
}
@@ -74,14 +67,14 @@
return []string{mode}, nil
}
-func createVulncheckQueueTasks(ctx context.Context, cfg *config.Config, params *vulncheckEnqueueParams, modes []string) (_ []queue.Task, err error) {
+func createVulncheckQueueTasks(ctx context.Context, cfg *config.Config, params *ivulncheck.EnqueueQueryParams, modes []string) (_ []queue.Task, err error) {
defer derrors.Wrap(&err, "createVulncheckQueueTasks(%v)", modes)
var (
tasks []queue.Task
modspecs []scan.ModuleSpec
)
for _, mode := range modes {
- var reqs []*vulncheckRequest
+ var reqs []*ivulncheck.Request
if mode == ModeBinary {
reqs, err = readBinaries(ctx, cfg.BinaryBucket)
if err != nil {
@@ -120,7 +113,7 @@
// binaryDir is the directory in the GCS bucket that contains binaries that should be scanned.
const binaryDir = "binaries"
-func readBinaries(ctx context.Context, bucketName string) (reqs []*vulncheckRequest, err error) {
+func readBinaries(ctx context.Context, bucketName string) (reqs []*ivulncheck.Request, err error) {
defer derrors.Wrap(&err, "readBinaries(%q)", bucketName)
if bucketName == "" {
log.Infof(ctx, "binary bucket not configured; not enqueuing binaries")
@@ -143,9 +136,9 @@
if err != nil {
return nil, err
}
- reqs = append(reqs, &vulncheckRequest{
- ModuleURLPath: mp,
- vulncheckRequestParams: vulncheckRequestParams{Mode: ModeBinary},
+ reqs = append(reqs, &ivulncheck.Request{
+ ModuleURLPath: mp,
+ QueryParams: ivulncheck.QueryParams{Mode: ModeBinary},
})
}
return reqs, nil
diff --git a/internal/worker/vulncheck_enqueue_test.go b/internal/worker/vulncheck_enqueue_test.go
index d54bc35..befea8b 100644
--- a/internal/worker/vulncheck_enqueue_test.go
+++ b/internal/worker/vulncheck_enqueue_test.go
@@ -14,6 +14,7 @@
"golang.org/x/pkgsite-metrics/internal/config"
"golang.org/x/pkgsite-metrics/internal/queue"
"golang.org/x/pkgsite-metrics/internal/scan"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
var binaryBucket = flag.String("binary-bucket", "", "bucket for scannable binaries")
@@ -26,13 +27,13 @@
if err != nil {
t.Fatal(err)
}
- want := &vulncheckRequest{
- scan.ModuleURLPath{
+ want := &ivulncheck.Request{
+ ModuleURLPath: scan.ModuleURLPath{
Module: "golang.org/x/pkgsite",
Version: "v0.0.0-20221004150836-873fb37c2479",
Suffix: "cmd/worker",
},
- vulncheckRequestParams{Mode: ModeBinary},
+ QueryParams: ivulncheck.QueryParams{Mode: ModeBinary},
}
found := false
for _, sr := range sreqs {
@@ -50,14 +51,14 @@
}
func TestCreateQueueTasks(t *testing.T) {
- vreq := func(path, version, mode string, importedBy int) *vulncheckRequest {
- return &vulncheckRequest{
- scan.ModuleURLPath{Module: path, Version: version},
- vulncheckRequestParams{Mode: mode, ImportedBy: importedBy},
+ vreq := func(path, version, mode string, importedBy int) *ivulncheck.Request {
+ return &ivulncheck.Request{
+ ModuleURLPath: scan.ModuleURLPath{Module: path, Version: version},
+ QueryParams: ivulncheck.QueryParams{Mode: mode, ImportedBy: importedBy},
}
}
- params := &vulncheckEnqueueParams{Min: 8, File: "testdata/modules.txt"}
+ params := &ivulncheck.EnqueueQueryParams{Min: 8, File: "testdata/modules.txt"}
gotTasks, err := createVulncheckQueueTasks(context.Background(), &config.Config{}, params, []string{ModeVTAStacks})
if err != nil {
t.Fatal(err)
@@ -67,7 +68,7 @@
vreq("github.com/pkg/errors", "v0.9.1", ModeVTAStacks, 10),
vreq("golang.org/x/net", "v0.4.0", ModeVTAStacks, 20),
}
- if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(vulncheckRequest{})); diff != "" {
+ if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(ivulncheck.Request{})); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
@@ -87,7 +88,7 @@
vreq("golang.org/x/net", "v0.4.0", mode, 20))
}
- if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(vulncheckRequest{})); diff != "" {
+ if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(ivulncheck.Request{})); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
}
diff --git a/internal/worker/vulncheck_results.go b/internal/worker/vulncheck_results.go
index 31e9777..c743279 100644
--- a/internal/worker/vulncheck_results.go
+++ b/internal/worker/vulncheck_results.go
@@ -11,10 +11,10 @@
"cloud.google.com/go/civil"
"golang.org/x/exp/event"
- "golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/derrors"
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/scan"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
var insertResultsCounter = event.NewCounter("insert-results", &event.MetricOptions{Namespace: metricNamespace})
@@ -44,12 +44,12 @@
}
ctx := r.Context()
log.Infof(ctx, "reading results")
- results, err := bigquery.FetchVulncheckResults(ctx, h.bqClient)
+ results, err := ivulncheck.FetchResults(ctx, h.bqClient)
if err != nil {
return err
}
log.Infof(ctx, "inserting %d results for %s", len(results), date)
- if err := bigquery.InsertVulncheckResults(ctx, h.bqClient, results, date, allowDuplicates); err != nil {
+ if err := ivulncheck.InsertResults(ctx, h.bqClient, results, date, allowDuplicates); err != nil {
return err
}
fmt.Fprintf(w, "%d results for %s inserted successfully.\n", len(results), date)
diff --git a/internal/worker/vulncheck_scan.go b/internal/worker/vulncheck_scan.go
index f2f18a3..54795ce 100644
--- a/internal/worker/vulncheck_scan.go
+++ b/internal/worker/vulncheck_scan.go
@@ -31,9 +31,9 @@
"golang.org/x/pkgsite-metrics/internal/modules"
"golang.org/x/pkgsite-metrics/internal/proxy"
"golang.org/x/pkgsite-metrics/internal/sandbox"
- "golang.org/x/pkgsite-metrics/internal/scan"
"golang.org/x/pkgsite-metrics/internal/version"
- vulnc "golang.org/x/vuln/client"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
+ vulnclient "golang.org/x/vuln/client"
"golang.org/x/vuln/exp/govulncheck"
"golang.org/x/vuln/vulncheck"
)
@@ -76,7 +76,7 @@
var scanCounter = event.NewCounter("scans", &event.MetricOptions{Namespace: metricNamespace})
// path: /vulncheck/scan/MODULE_VERSION_SUFFIX?params
-// See parseVulncheckRequest for allowed path forms and query params.
+// See internal/vulncheck.ParseRequest for allowed path forms and query params.
func (h *VulncheckServer) handleScan(w http.ResponseWriter, r *http.Request) (err error) {
defer derrors.Wrap(&err, "handleScan")
@@ -85,7 +85,7 @@
}()
ctx := r.Context()
- sreq, err := parseVulncheckRequest(r, "/vulncheck/scan")
+ sreq, err := ivulncheck.ParseRequest(r, "/vulncheck/scan")
if err != nil {
return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
}
@@ -127,16 +127,16 @@
return nil
}
var err error
- h.storedWorkVersions, err = bigquery.ReadVulncheckWorkVersions(ctx, h.bqClient)
+ h.storedWorkVersions, err = ivulncheck.ReadWorkVersions(ctx, h.bqClient)
return err
}
// A scanner holds state for scanning modules.
type scanner struct {
proxyClient *proxy.Client
- dbClient vulnc.Client
+ dbClient vulnclient.Client
bqClient *bigquery.Client
- workVersion *bigquery.VulncheckWorkVersion
+ workVersion *ivulncheck.WorkVersion
goMemLimit uint64
gcsBucket *storage.BucketHandle
insecure bool
@@ -182,14 +182,14 @@
return s.err
}
-func (s *scanner) ScanModule(ctx context.Context, w http.ResponseWriter, sreq *vulncheckRequest) error {
+func (s *scanner) ScanModule(ctx context.Context, w http.ResponseWriter, sreq *ivulncheck.Request) error {
if sreq.Module == "std" {
return nil // ignore the standard library
}
- row := &bigquery.VulnResult{
- ModulePath: sreq.Module,
- Suffix: sreq.Suffix,
- VulncheckWorkVersion: *s.workVersion,
+ row := &ivulncheck.Result{
+ ModulePath: sreq.Module,
+ Suffix: sreq.Suffix,
+ WorkVersion: *s.workVersion,
}
// Scan the version.
log.Infof(ctx, "fetching proxy info: %s@%s", sreq.Module, sreq.Version)
@@ -238,7 +238,7 @@
log.Infof(ctx, "bigquery disabled, not uploading")
} else {
log.Infof(ctx, "uploading to bigquery: %s", sreq.Path())
- if err := s.bqClient.Upload(ctx, bigquery.VulncheckTableName, row); err != nil {
+ if err := s.bqClient.Upload(ctx, ivulncheck.TableName, row); err != nil {
// This is often caused by:
// "Upload: googleapi: got HTTP response code 413 with body"
// which happens for some modules.
@@ -259,7 +259,7 @@
// runScanModule fetches the module version from the proxy, and analyzes it for
// vulnerabilities.
-func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) (bvulns []*bigquery.Vuln, err error) {
+func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) (bvulns []*ivulncheck.Vuln, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%w: %v\n\n%s", derrors.ScanModulePanicError, e, debug.Stack())
@@ -290,7 +290,7 @@
return nil, err
}
for _, v := range vulns {
- bvulns = append(bvulns, convertVuln(v))
+ bvulns = append(bvulns, ivulncheck.ConvertVulncheckOutput(v))
}
} else { // Govulncheck mode
var vulns []*govulncheck.Vuln
@@ -307,7 +307,7 @@
return nil, err
}
for _, v := range vulns {
- bvulns = append(bvulns, convertGovulncheckOutput(v)...)
+ bvulns = append(bvulns, ivulncheck.ConvertGovulncheckOutput(v)...)
}
}
return bvulns, nil
@@ -704,7 +704,7 @@
strings.Contains(s, "connection")
}
-func vulncheckConfig(dbClient vulnc.Client, mode string) *vulncheck.Config {
+func vulncheckConfig(dbClient vulnclient.Client, mode string) *vulncheck.Config {
cfg := &vulncheck.Config{Client: dbClient}
switch mode {
case ModeImports:
@@ -715,62 +715,6 @@
return cfg
}
-func convertVuln(v *vulncheck.Vuln) *bigquery.Vuln {
- return &bigquery.Vuln{
- ID: v.OSV.ID,
- ModulePath: v.ModPath,
- PackagePath: v.PkgPath,
- Symbol: v.Symbol,
- CallSink: bigquery.NullInt(v.CallSink),
- ImportSink: bigquery.NullInt(v.ImportSink),
- RequireSink: bigquery.NullInt(v.RequireSink),
- }
-}
-
-func convertGovulncheckOutput(v *govulncheck.Vuln) (vulns []*bigquery.Vuln) {
- for _, module := range v.Modules {
- for pkgNum, pkg := range module.Packages {
- addedSymbols := make(map[string]bool)
- baseVuln := &bigquery.Vuln{
- ID: v.OSV.ID,
- ModulePath: module.Path,
- PackagePath: pkg.Path,
- CallSink: bigquery.NullInt(0),
- ImportSink: bigquery.NullInt(pkgNum + 1),
- RequireSink: bigquery.NullInt(pkgNum + 1),
- }
-
- // For each called symbol, reconstruct sinks and create the corresponding bigquery vuln
- for symbolNum, cs := range pkg.CallStacks {
- addedSymbols[cs.Symbol] = true
- toAdd := *baseVuln
- toAdd.Symbol = cs.Symbol
- toAdd.CallSink = bigquery.NullInt(symbolNum + 1)
- vulns = append(vulns, &toAdd)
- }
-
- // Find the rest of the vulnerable imported symbols that haven't been called
- // and create corresponding bigquery vulns
- for _, affected := range v.OSV.Affected {
- if affected.Package.Name == module.Path {
- for _, imp := range affected.EcosystemSpecific.Imports {
- if imp.Path == pkg.Path {
- for _, symbol := range imp.Symbols {
- if !addedSymbols[symbol] {
- toAdd := *baseVuln
- toAdd.Symbol = symbol
- vulns = append(vulns, &toAdd)
- }
- }
- }
- }
- }
- }
- }
- }
- return vulns
-}
-
// currHeapUsage computes currently allocate heap bytes.
func currHeapUsage() uint64 {
var stats runtime.MemStats
@@ -889,58 +833,6 @@
}
}
-// vulncheckRequest contains information passed
-// to a scan endpoint.
-type vulncheckRequest struct {
- scan.ModuleURLPath
- vulncheckRequestParams
-}
-
-// vulncheckRequestParams has query parameters for a vulncheck scan request.
-type vulncheckRequestParams struct {
- ImportedBy int // imported-by count
- Mode string // vulncheck mode (VTA, etc)
- Insecure bool // if true, run outside sandbox
- Serve bool // serve results back to client instead of writing them to BigQuery
-}
-
-// These methods implement queue.Task.
-func (r *vulncheckRequest) Name() string { return r.Module + "@" + r.Version }
-
-func (r *vulncheckRequest) Path() string { return r.ModuleURLPath.Path() }
-
-func (r *vulncheckRequest) Params() string {
- return scan.FormatParams(r.vulncheckRequestParams)
-}
-
-// parseVulncheckRequest parses an http request r for an endpoint
-// prefix and produces a corresponding ScanRequest.
-//
-// The module and version should have one of the following three forms:
-// - <module>/@v/<version>
-// - <module>@<version>
-// - <module>/@latest
-//
-// (These are the same forms that the module proxy accepts.)
-func parseVulncheckRequest(r *http.Request, prefix string) (*vulncheckRequest, error) {
- mp, err := scan.ParseModuleURLPath(strings.TrimPrefix(r.URL.Path, prefix))
- if err != nil {
- return nil, err
- }
-
- rp := vulncheckRequestParams{ImportedBy: -1}
- if err := scan.ParseParams(r, &rp); err != nil {
- return nil, err
- }
- if rp.ImportedBy < 0 {
- return nil, errors.New(`missing or negative "importedby" query param`)
- }
- return &vulncheckRequest{
- ModuleURLPath: mp,
- vulncheckRequestParams: rp,
- }, nil
-}
-
// diskUsage runs the du command to determine how much disk space the given
// directories occupy.
func diskUsage(dirs ...string) string {
diff --git a/internal/worker/vulncheck_scan_test.go b/internal/worker/vulncheck_scan_test.go
index 18589b2..8a4c7dd 100644
--- a/internal/worker/vulncheck_scan_test.go
+++ b/internal/worker/vulncheck_scan_test.go
@@ -14,14 +14,10 @@
"cloud.google.com/go/storage"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/config"
"golang.org/x/pkgsite-metrics/internal/derrors"
"golang.org/x/pkgsite-metrics/internal/proxy"
vulnc "golang.org/x/vuln/client"
- "golang.org/x/vuln/exp/govulncheck"
- "golang.org/x/vuln/osv"
"golang.org/x/vuln/vulncheck"
)
@@ -172,126 +168,3 @@
t.Errorf("got %+v, want %+v", got, want)
}
}
-
-func TestConvertGovulncheckOutput(t *testing.T) {
- var (
- osvEntry = &osv.Entry{
- ID: "GO-YYYY-1234",
- Affected: []osv.Affected{
- {
- Package: osv.Package{
- Name: "example.com/repo/module",
- Ecosystem: "Go",
- },
- EcosystemSpecific: osv.EcosystemSpecific{
- Imports: []osv.EcosystemSpecificImport{
- {
- Path: "example.com/repo/module/package",
- Symbols: []string{
- "Symbol",
- "Another",
- },
- },
- },
- },
- },
- },
- }
-
- vuln1 = &govulncheck.Vuln{
- OSV: osvEntry,
- Modules: []*govulncheck.Module{
- {
- Path: "example.com/repo/module",
- Packages: []*govulncheck.Package{
- {
- Path: "example.com/repo/module/package",
- CallStacks: []govulncheck.CallStack{
- {
- Symbol: "Symbol",
- Summary: "example.go:1:1 xyz.func calls pkgPath.Symbol",
- Frames: []*govulncheck.StackFrame{},
- },
- },
- },
- },
- },
- },
- }
-
- vuln2 = &govulncheck.Vuln{
- OSV: osvEntry,
- Modules: []*govulncheck.Module{
- {
- Path: "example.com/repo/module",
- Packages: []*govulncheck.Package{
- {
- Path: "example.com/repo/module/package",
- },
- },
- },
- },
- }
- )
- tests := []struct {
- name string
- vuln *govulncheck.Vuln
- wantVulns []*bigquery.Vuln
- }{
- {
- name: "Call One Symbol",
- vuln: vuln1,
- wantVulns: []*bigquery.Vuln{
- {
- ID: "GO-YYYY-1234",
- Symbol: "Symbol",
- PackagePath: "example.com/repo/module/package",
- ModulePath: "example.com/repo/module",
- CallSink: bigquery.NullInt(1),
- ImportSink: bigquery.NullInt(1),
- RequireSink: bigquery.NullInt(1),
- },
- {
- ID: "GO-YYYY-1234",
- Symbol: "Another",
- PackagePath: "example.com/repo/module/package",
- ModulePath: "example.com/repo/module",
- CallSink: bigquery.NullInt(0),
- ImportSink: bigquery.NullInt(1),
- RequireSink: bigquery.NullInt(1),
- },
- },
- },
- {
- name: "Call no symbols",
- vuln: vuln2,
- wantVulns: []*bigquery.Vuln{
- {
- ID: "GO-YYYY-1234",
- PackagePath: "example.com/repo/module/package",
- ModulePath: "example.com/repo/module",
- Symbol: "Symbol",
- CallSink: bigquery.NullInt(0),
- ImportSink: bigquery.NullInt(1),
- RequireSink: bigquery.NullInt(1),
- },
- {
- ID: "GO-YYYY-1234",
- PackagePath: "example.com/repo/module/package",
- ModulePath: "example.com/repo/module",
- Symbol: "Another",
- CallSink: bigquery.NullInt(0),
- ImportSink: bigquery.NullInt(1),
- RequireSink: bigquery.NullInt(1),
- },
- },
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if diff := cmp.Diff(convertGovulncheckOutput(tt.vuln), tt.wantVulns, cmpopts.EquateEmpty()); diff != "" {
- t.Errorf("mismatch (-got, +want): %s", diff)
- }
- })
- }
-}
diff --git a/internal/worker/vulncheck_test.go b/internal/worker/vulncheck_test.go
index f876711..65a933a 100644
--- a/internal/worker/vulncheck_test.go
+++ b/internal/worker/vulncheck_test.go
@@ -10,21 +10,22 @@
"github.com/google/go-cmp/cmp"
"golang.org/x/pkgsite-metrics/internal/bigquery"
+ ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
func TestVulnsScanned(t *testing.T) {
p := newPage("test")
- vuln1A := &bigquery.Vuln{ID: "1", Symbol: "A", CallSink: bigquery.NullInt(100)}
- vuln1B := &bigquery.Vuln{ID: "1", Symbol: "B", CallSink: bigquery.NullInt(101)}
- vuln1C := &bigquery.Vuln{ID: "1", Symbol: "C"}
- vuln2A := &bigquery.Vuln{ID: "2", Symbol: "A"}
+ vuln1A := &ivulncheck.Vuln{ID: "1", Symbol: "A", CallSink: bigquery.NullInt(100)}
+ vuln1B := &ivulncheck.Vuln{ID: "1", Symbol: "B", CallSink: bigquery.NullInt(101)}
+ vuln1C := &ivulncheck.Vuln{ID: "1", Symbol: "C"}
+ vuln2A := &ivulncheck.Vuln{ID: "2", Symbol: "A"}
- rows := []*bigquery.VulnResult{
- {ModulePath: "m1", ScanMode: ModeImports, Vulns: []*bigquery.Vuln{vuln1A, vuln1B, vuln1C, vuln2A}},
- {ModulePath: "m1", ScanMode: ModeVTAStacks, Vulns: []*bigquery.Vuln{vuln1A, vuln1B}},
- {ModulePath: "m2", ScanMode: ModeImports, Vulns: []*bigquery.Vuln{vuln2A}},
- {ModulePath: "m2", ScanMode: ModeVTAStacks, Vulns: []*bigquery.Vuln{}},
+ rows := []*ivulncheck.Result{
+ {ModulePath: "m1", ScanMode: ModeImports, Vulns: []*ivulncheck.Vuln{vuln1A, vuln1B, vuln1C, vuln2A}},
+ {ModulePath: "m1", ScanMode: ModeVTAStacks, Vulns: []*ivulncheck.Vuln{vuln1A, vuln1B}},
+ {ModulePath: "m2", ScanMode: ModeImports, Vulns: []*ivulncheck.Vuln{vuln2A}},
+ {ModulePath: "m2", ScanMode: ModeVTAStacks, Vulns: []*ivulncheck.Vuln{}},
}
got := handleVulncheckRows(context.Background(), p, rows)
