internal/govulncheck: rename internal vulncheck package to govulncheck
This changes is part of a series of CLs that replaces the uses of
"vulncheck" with "govulncheck."
Change-Id: I4849087e1bd28e0dac6b2a6b80e5c3298c07c659
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/476058
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/internal/govulncheck/govulncheck.go b/internal/govulncheck/govulncheck.go
index 0c5297f..dac90f0 100644
--- a/internal/govulncheck/govulncheck.go
+++ b/internal/govulncheck/govulncheck.go
@@ -5,12 +5,355 @@
package govulncheck
import (
+ "context"
"encoding/json"
"errors"
+ "fmt"
+ "net/http"
+ "sort"
+ "strings"
+ "time"
+ bq "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+ "golang.org/x/exp/maps"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "golang.org/x/pkgsite-metrics/internal/scan"
"golang.org/x/vuln/exp/govulncheck"
+ "google.golang.org/api/iterator"
)
+// EnqueueQueryParams for vulncheck/enqueue
+type EnqueueQueryParams struct {
+ Suffix string // appended to task queue IDs to generate unique tasks
+ Mode string // type of analysis to run
+ Min int // minimum import-by count for a module to be included
+ File string // path to file containing modules; if missing, use DB
+}
+
+// Request contains information passed
+// to a scan endpoint.
+type Request struct {
+ scan.ModuleURLPath
+ QueryParams
+}
+
+// QueryParams has query parameters for a vulncheck scan request.
+type QueryParams struct {
+ ImportedBy int // imported-by count
+ Mode string // vulncheck mode (VTA, etc)
+ Insecure bool // if true, run outside sandbox
+ Serve bool // serve results back to client instead of writing them to BigQuery
+}
+
+// These methods implement queue.Task.
+func (r *Request) Name() string { return r.Module + "@" + r.Version }
+
+func (r *Request) Path() string { return r.ModuleURLPath.Path() }
+
+func (r *Request) Params() string {
+ return scan.FormatParams(r.QueryParams)
+}
+
+// ParseRequest parses an http request r for an endpoint
+// prefix and produces a corresponding ScanRequest.
+//
+// The module and version should have one of the following three forms:
+// - <module>/@v/<version>
+// - <module>@<version>
+// - <module>/@latest
+//
+// (These are the same forms that the module proxy accepts.)
+func ParseRequest(r *http.Request, prefix string) (*Request, error) {
+ mp, err := scan.ParseModuleURLPath(strings.TrimPrefix(r.URL.Path, prefix))
+ if err != nil {
+ return nil, err
+ }
+
+ rp := QueryParams{ImportedBy: -1}
+ if err := scan.ParseParams(r, &rp); err != nil {
+ return nil, err
+ }
+ if rp.ImportedBy < 0 {
+ return nil, errors.New(`missing or negative "importedby" query param`)
+ }
+ return &Request{
+ ModuleURLPath: mp,
+ QueryParams: rp,
+ }, nil
+}
+
+func ConvertGovulncheckOutput(v *govulncheck.Vuln) (vulns []*Vuln) {
+ for _, module := range v.Modules {
+ for pkgNum, pkg := range module.Packages {
+ addedSymbols := make(map[string]bool)
+ baseVuln := &Vuln{
+ ID: v.OSV.ID,
+ ModulePath: module.Path,
+ PackagePath: pkg.Path,
+ CallSink: bigquery.NullInt(0),
+ ImportSink: bigquery.NullInt(pkgNum + 1),
+ RequireSink: bigquery.NullInt(pkgNum + 1),
+ }
+
+ // For each called symbol, reconstruct sinks and create the corresponding bigquery vuln
+ for symbolNum, cs := range pkg.CallStacks {
+ addedSymbols[cs.Symbol] = true
+ toAdd := *baseVuln
+ toAdd.Symbol = cs.Symbol
+ toAdd.CallSink = bigquery.NullInt(symbolNum + 1)
+ vulns = append(vulns, &toAdd)
+ }
+
+ // Find the rest of the vulnerable imported symbols that haven't been called
+ // and create corresponding bigquery vulns
+ for _, affected := range v.OSV.Affected {
+ if affected.Package.Name == module.Path {
+ for _, imp := range affected.EcosystemSpecific.Imports {
+ if imp.Path == pkg.Path {
+ for _, symbol := range imp.Symbols {
+ if !addedSymbols[symbol] {
+ toAdd := *baseVuln
+ toAdd.Symbol = symbol
+ vulns = append(vulns, &toAdd)
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return vulns
+}
+
+const TableName = "vulncheck"
+
+// Note: before modifying Result or Vuln, make sure the change
+// is a valid schema modification.
+// The only supported changes are:
+// - adding a nullable or repeated column
+// - dropping a column
+// - changing a column from required to nullable.
+// See https://cloud.google.com/bigquery/docs/managing-table-schemas for details.
+
+// Result is a row in the BigQuery vulncheck table. It corresponds to a
+// result from the output for vulncheck.Source.
+type Result struct {
+ CreatedAt time.Time `bigquery:"created_at"`
+ ModulePath string `bigquery:"module_path"`
+ Version string `bigquery:"version"`
+ Suffix string `bigquery:"suffix"`
+ SortVersion string `bigquery:"sort_version"`
+ ImportedBy int `bigquery:"imported_by"`
+ Error string `bigquery:"error"`
+ ErrorCategory string `bigquery:"error_category"`
+ CommitTime time.Time `bigquery:"commit_time"`
+ ScanSeconds float64 `bigquery:"scan_seconds"`
+ ScanMemory int64 `bigquery:"scan_memory"`
+ PkgsMemory int64 `bigquery:"pkgs_memory"`
+ ScanMode string `bigquery:"scan_mode"`
+ // Workers is the concurrency limit under which a module is
+ // analyzed. Useful for interpreting memory measurements when
+ // there are multiple modules analyzed in the same process.
+ // 0 if no limit is specified, -1 for potential errors.
+ Workers int `bigquery:"workers"`
+ WorkVersion // InferSchema flattens embedded fields
+ Vulns []*Vuln `bigquery:"vulns"`
+}
+
+// WorkVersion contains information that can be used to avoid duplicate work.
+// Given two WorkVersion values v1 and v2 for the same module path and version,
+// if v1.Equal(v2) then it is not necessary to scan the module.
+type WorkVersion struct {
+ // The version of the currently running code. This tracks changes in the
+ // logic of module scanning and processing.
+ WorkerVersion string `bigquery:"worker_version"`
+ // The version of the bigquery schema.
+ SchemaVersion string ` bigquery:"schema_version"`
+ // The version of the golang.org/x/vuln module used by the current module.
+ VulnVersion string `bigquery:"x_vuln_version"`
+ // When the vuln DB was last modified.
+ VulnDBLastModified time.Time `bigquery:"vulndb_last_modified"`
+}
+
+func (v1 *WorkVersion) Equal(v2 *WorkVersion) bool {
+ if v1 == nil || v2 == nil {
+ return v1 == v2
+ }
+ return v1.WorkerVersion == v2.WorkerVersion &&
+ v1.SchemaVersion == v2.SchemaVersion &&
+ v1.VulnVersion == v2.VulnVersion &&
+ v1.VulnDBLastModified.Equal(v2.VulnDBLastModified)
+}
+
+func (vr *Result) SetUploadTime(t time.Time) { vr.CreatedAt = t }
+
+func (vr *Result) AddError(err error) {
+ if err == nil {
+ return
+ }
+ vr.Error = err.Error()
+ vr.ErrorCategory = derrors.CategorizeError(err)
+}
+
+// Vuln is a record in Result and corresponds to an item in
+// vulncheck.Result.Vulns.
+type Vuln struct {
+ ID string `bigquery:"id"`
+ Symbol string `bigquery:"symbol"`
+ PackagePath string `bigquery:"package_path"`
+ ModulePath string `bigquery:"module_path"`
+ CallSink bq.NullInt64 `bigquery:"call_sink"`
+ ImportSink bq.NullInt64 `bigquery:"import_sink"`
+ RequireSink bq.NullInt64 `bigquery:"require_sink"`
+}
+
+// SchemaVersion changes whenever the vulncheck schema changes.
+var SchemaVersion string
+
+func init() {
+ s, err := bigquery.InferSchema(Result{})
+ if err != nil {
+ panic(err)
+ }
+ SchemaVersion = bigquery.SchemaVersion(s)
+ bigquery.AddTable(TableName, s)
+}
+
+// ReadWorkVersions reads the most recent WorkVersions in the vulncheck table.
+func ReadWorkVersions(ctx context.Context, c *bigquery.Client) (_ map[[2]string]*WorkVersion, err error) {
+ defer derrors.Wrap(&err, "ReadWorkVersions")
+ m := map[[2]string]*WorkVersion{}
+ query := bigquery.PartitionQuery{
+ Table: c.FullTableName(TableName),
+ Columns: "module_path, version, worker_version, schema_version, x_vuln_version, vulndb_last_modified",
+ PartitionOn: "module_path, sort_version",
+ OrderBy: "created_at DESC",
+ }.String()
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return nil, err
+ }
+ err = bigquery.ForEachRow(iter, func(r *Result) bool {
+ m[[2]string{r.ModulePath, r.Version}] = &r.WorkVersion
+ return true
+ })
+ if err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// The module path along with the four sort columns should uniquely specify a
+// row, because we do not generate a new row for a (module, version) if the
+// other three versions are identical. (There is actually a fourth component of
+// the work version, the schema version. But since it is represented by a struct
+// in the worker code and the worker version captures every change to that code,
+// it cannot change independently of worker_version.)
+const orderByClauses = `
+ vulndb_last_modified DESC, -- latest version of database
+ x_vuln_version DESC, -- latest version of x/vuln
+ worker_version DESC, -- latest version of x/pkgsite-metrics
+ sort_version DESC, -- latest version of module
+ created_at DESC -- latest insertion time
+`
+
+func FetchResults(ctx context.Context, c *bigquery.Client) (rows []*Result, err error) {
+ return fetchResults(ctx, c, TableName)
+}
+
+func fetchResults(ctx context.Context, c *bigquery.Client, tableName string) (rows []*Result, err error) {
+ name := c.FullTableName(tableName)
+ query := bigquery.PartitionQuery{
+ Table: name,
+ PartitionOn: "module_path, scan_mode",
+ OrderBy: orderByClauses,
+ }.String()
+ log.Infof(ctx, "running latest query on %s", name)
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return nil, err
+ }
+ rows, err = bigquery.All[Result](iter)
+ if err != nil {
+ return nil, err
+ }
+ log.Infof(ctx, "got %d rows", len(rows))
+
+ // Check for duplicate rows.
+ modvers := map[string]int{}
+ for _, r := range rows {
+ modvers[r.ModulePath+"@"+r.Version+" "+r.ScanMode]++
+ }
+ keys := maps.Keys(modvers)
+ sort.Strings(keys)
+ for _, k := range keys {
+ if n := modvers[k]; n > 1 {
+ return nil, fmt.Errorf("%s has %d rows", k, n)
+ }
+ }
+ return rows, nil
+}
+
+type ReportResult struct {
+ *Result
+ ReportDate civil.Date `bigquery:"report_date"` // for reporting (e.g. dashboard)
+ InsertedAt time.Time `bigquery:"inserted_at"` // to disambiguate if >1 insertion for same date
+}
+
+func init() {
+ s, err := bigquery.InferSchema(ReportResult{})
+ if err != nil {
+ panic(err)
+ }
+ bigquery.AddTable(TableName+"-report", s)
+}
+
+func InsertResults(ctx context.Context, c *bigquery.Client, results []*Result, date civil.Date, allowDuplicates bool) (err error) {
+ return insertResults(ctx, c, TableName+"-report", results, date, allowDuplicates)
+}
+
+func insertResults(ctx context.Context, c *bigquery.Client, reportTableName string, results []*Result, date civil.Date, allowDuplicates bool) (err error) {
+ derrors.Wrap(&err, "InsertResults(%s)", date)
+ // Create the report table if it doesn't exist.
+ if err := c.CreateTable(ctx, reportTableName); err != nil {
+ return err
+ }
+
+ if !allowDuplicates {
+ query := fmt.Sprintf("SELECT COUNT(*) FROM `%s` WHERE report_date = '%s'",
+ c.FullTableName(reportTableName), date)
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return err
+ }
+ var count struct {
+ n int
+ }
+ err = iter.Next(&count)
+ if err != nil && err != iterator.Done {
+ return err
+ }
+ if count.n > 0 {
+ return fmt.Errorf("already have %d rows for %s", count.n, date)
+ }
+ }
+
+ now := time.Now()
+ var rows []ReportResult
+ for _, r := range results {
+ rows = append(rows, ReportResult{Result: r, ReportDate: date, InsertedAt: now})
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) // to avoid retrying forever on permanent errors
+ defer cancel()
+ const chunkSize = 1024 // Chunk rows to a void exceeding the maximum allowable request size.
+ return bigquery.UploadMany(ctx, c, reportTableName, rows, chunkSize)
+}
+
// ScanStats represent monitoring information about a given
// run of govulncheck or vulncheck
type ScanStats struct {
diff --git a/internal/vulncheck/vulncheck_test.go b/internal/govulncheck/govulncheck_test.go
similarity index 99%
rename from internal/vulncheck/vulncheck_test.go
rename to internal/govulncheck/govulncheck_test.go
index 145d23a..e4da8f7 100644
--- a/internal/vulncheck/vulncheck_test.go
+++ b/internal/govulncheck/govulncheck_test.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package vulncheck
+package govulncheck
import (
"context"
diff --git a/internal/vulncheck/vulncheck.go b/internal/vulncheck/vulncheck.go
deleted file mode 100644
index 25205d1..0000000
--- a/internal/vulncheck/vulncheck.go
+++ /dev/null
@@ -1,367 +0,0 @@
-// Copyright 2022 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vulncheck
-
-import (
- "context"
- "errors"
- "fmt"
- "net/http"
- "sort"
- "strings"
- "time"
-
- bq "cloud.google.com/go/bigquery"
- "cloud.google.com/go/civil"
- "golang.org/x/exp/maps"
- "golang.org/x/pkgsite-metrics/internal/bigquery"
- "golang.org/x/pkgsite-metrics/internal/derrors"
- "golang.org/x/pkgsite-metrics/internal/log"
- "golang.org/x/pkgsite-metrics/internal/scan"
- "golang.org/x/vuln/exp/govulncheck"
- "golang.org/x/vuln/vulncheck"
- "google.golang.org/api/iterator"
-)
-
-// EnqueueQueryParams for vulncheck/enqueue
-type EnqueueQueryParams struct {
- Suffix string // appended to task queue IDs to generate unique tasks
- Mode string // type of analysis to run
- Min int // minimum import-by count for a module to be included
- File string // path to file containing modules; if missing, use DB
-}
-
-// Request contains information passed
-// to a scan endpoint.
-type Request struct {
- scan.ModuleURLPath
- QueryParams
-}
-
-// QueryParams has query parameters for a vulncheck scan request.
-type QueryParams struct {
- ImportedBy int // imported-by count
- Mode string // vulncheck mode (VTA, etc)
- Insecure bool // if true, run outside sandbox
- Serve bool // serve results back to client instead of writing them to BigQuery
-}
-
-// These methods implement queue.Task.
-func (r *Request) Name() string { return r.Module + "@" + r.Version }
-
-func (r *Request) Path() string { return r.ModuleURLPath.Path() }
-
-func (r *Request) Params() string {
- return scan.FormatParams(r.QueryParams)
-}
-
-// ParseRequest parses an http request r for an endpoint
-// prefix and produces a corresponding ScanRequest.
-//
-// The module and version should have one of the following three forms:
-// - <module>/@v/<version>
-// - <module>@<version>
-// - <module>/@latest
-//
-// (These are the same forms that the module proxy accepts.)
-func ParseRequest(r *http.Request, prefix string) (*Request, error) {
- mp, err := scan.ParseModuleURLPath(strings.TrimPrefix(r.URL.Path, prefix))
- if err != nil {
- return nil, err
- }
-
- rp := QueryParams{ImportedBy: -1}
- if err := scan.ParseParams(r, &rp); err != nil {
- return nil, err
- }
- if rp.ImportedBy < 0 {
- return nil, errors.New(`missing or negative "importedby" query param`)
- }
- return &Request{
- ModuleURLPath: mp,
- QueryParams: rp,
- }, nil
-}
-
-func ConvertVulncheckOutput(v *vulncheck.Vuln) *Vuln {
- return &Vuln{
- ID: v.OSV.ID,
- ModulePath: v.ModPath,
- PackagePath: v.PkgPath,
- Symbol: v.Symbol,
- CallSink: bigquery.NullInt(v.CallSink),
- ImportSink: bigquery.NullInt(v.ImportSink),
- RequireSink: bigquery.NullInt(v.RequireSink),
- }
-}
-
-func ConvertGovulncheckOutput(v *govulncheck.Vuln) (vulns []*Vuln) {
- for _, module := range v.Modules {
- for pkgNum, pkg := range module.Packages {
- addedSymbols := make(map[string]bool)
- baseVuln := &Vuln{
- ID: v.OSV.ID,
- ModulePath: module.Path,
- PackagePath: pkg.Path,
- CallSink: bigquery.NullInt(0),
- ImportSink: bigquery.NullInt(pkgNum + 1),
- RequireSink: bigquery.NullInt(pkgNum + 1),
- }
-
- // For each called symbol, reconstruct sinks and create the corresponding bigquery vuln
- for symbolNum, cs := range pkg.CallStacks {
- addedSymbols[cs.Symbol] = true
- toAdd := *baseVuln
- toAdd.Symbol = cs.Symbol
- toAdd.CallSink = bigquery.NullInt(symbolNum + 1)
- vulns = append(vulns, &toAdd)
- }
-
- // Find the rest of the vulnerable imported symbols that haven't been called
- // and create corresponding bigquery vulns
- for _, affected := range v.OSV.Affected {
- if affected.Package.Name == module.Path {
- for _, imp := range affected.EcosystemSpecific.Imports {
- if imp.Path == pkg.Path {
- for _, symbol := range imp.Symbols {
- if !addedSymbols[symbol] {
- toAdd := *baseVuln
- toAdd.Symbol = symbol
- vulns = append(vulns, &toAdd)
- }
- }
- }
- }
- }
- }
- }
- }
- return vulns
-}
-
-const TableName = "vulncheck"
-
-// Note: before modifying Result or Vuln, make sure the change
-// is a valid schema modification.
-// The only supported changes are:
-// - adding a nullable or repeated column
-// - dropping a column
-// - changing a column from required to nullable.
-// See https://cloud.google.com/bigquery/docs/managing-table-schemas for details.
-
-// Result is a row in the BigQuery vulncheck table. It corresponds to a
-// result from the output for vulncheck.Source.
-type Result struct {
- CreatedAt time.Time `bigquery:"created_at"`
- ModulePath string `bigquery:"module_path"`
- Version string `bigquery:"version"`
- Suffix string `bigquery:"suffix"`
- SortVersion string `bigquery:"sort_version"`
- ImportedBy int `bigquery:"imported_by"`
- Error string `bigquery:"error"`
- ErrorCategory string `bigquery:"error_category"`
- CommitTime time.Time `bigquery:"commit_time"`
- ScanSeconds float64 `bigquery:"scan_seconds"`
- ScanMemory int64 `bigquery:"scan_memory"`
- PkgsMemory int64 `bigquery:"pkgs_memory"`
- ScanMode string `bigquery:"scan_mode"`
- // Workers is the concurrency limit under which a module is
- // analyzed. Useful for interpreting memory measurements when
- // there are multiple modules analyzed in the same process.
- // 0 if no limit is specified, -1 for potential errors.
- Workers int `bigquery:"workers"`
- WorkVersion // InferSchema flattens embedded fields
- Vulns []*Vuln `bigquery:"vulns"`
-}
-
-// WorkVersion contains information that can be used to avoid duplicate work.
-// Given two WorkVersion values v1 and v2 for the same module path and version,
-// if v1.Equal(v2) then it is not necessary to scan the module.
-type WorkVersion struct {
- // The version of the currently running code. This tracks changes in the
- // logic of module scanning and processing.
- WorkerVersion string `bigquery:"worker_version"`
- // The version of the bigquery schema.
- SchemaVersion string ` bigquery:"schema_version"`
- // The version of the golang.org/x/vuln module used by the current module.
- VulnVersion string `bigquery:"x_vuln_version"`
- // When the vuln DB was last modified.
- VulnDBLastModified time.Time `bigquery:"vulndb_last_modified"`
-}
-
-func (v1 *WorkVersion) Equal(v2 *WorkVersion) bool {
- if v1 == nil || v2 == nil {
- return v1 == v2
- }
- return v1.WorkerVersion == v2.WorkerVersion &&
- v1.SchemaVersion == v2.SchemaVersion &&
- v1.VulnVersion == v2.VulnVersion &&
- v1.VulnDBLastModified.Equal(v2.VulnDBLastModified)
-}
-
-func (vr *Result) SetUploadTime(t time.Time) { vr.CreatedAt = t }
-
-func (vr *Result) AddError(err error) {
- if err == nil {
- return
- }
- vr.Error = err.Error()
- vr.ErrorCategory = derrors.CategorizeError(err)
-}
-
-// Vuln is a record in Result and corresponds to an item in
-// vulncheck.Result.Vulns.
-type Vuln struct {
- ID string `bigquery:"id"`
- Symbol string `bigquery:"symbol"`
- PackagePath string `bigquery:"package_path"`
- ModulePath string `bigquery:"module_path"`
- CallSink bq.NullInt64 `bigquery:"call_sink"`
- ImportSink bq.NullInt64 `bigquery:"import_sink"`
- RequireSink bq.NullInt64 `bigquery:"require_sink"`
-}
-
-// SchemaVersion changes whenever the vulncheck schema changes.
-var SchemaVersion string
-
-func init() {
- s, err := bigquery.InferSchema(Result{})
- if err != nil {
- panic(err)
- }
- SchemaVersion = bigquery.SchemaVersion(s)
- bigquery.AddTable(TableName, s)
-}
-
-// ReadWorkVersions reads the most recent WorkVersions in the vulncheck table.
-func ReadWorkVersions(ctx context.Context, c *bigquery.Client) (_ map[[2]string]*WorkVersion, err error) {
- defer derrors.Wrap(&err, "ReadWorkVersions")
- m := map[[2]string]*WorkVersion{}
- query := bigquery.PartitionQuery{
- Table: c.FullTableName(TableName),
- Columns: "module_path, version, worker_version, schema_version, x_vuln_version, vulndb_last_modified",
- PartitionOn: "module_path, sort_version",
- OrderBy: "created_at DESC",
- }.String()
- iter, err := c.Query(ctx, query)
- if err != nil {
- return nil, err
- }
- err = bigquery.ForEachRow(iter, func(r *Result) bool {
- m[[2]string{r.ModulePath, r.Version}] = &r.WorkVersion
- return true
- })
- if err != nil {
- return nil, err
- }
- return m, nil
-}
-
-// The module path along with the four sort columns should uniquely specify a
-// row, because we do not generate a new row for a (module, version) if the
-// other three versions are identical. (There is actually a fourth component of
-// the work version, the schema version. But since it is represented by a struct
-// in the worker code and the worker version captures every change to that code,
-// it cannot change independently of worker_version.)
-const orderByClauses = `
- vulndb_last_modified DESC, -- latest version of database
- x_vuln_version DESC, -- latest version of x/vuln
- worker_version DESC, -- latest version of x/pkgsite-metrics
- sort_version DESC, -- latest version of module
- created_at DESC -- latest insertion time
-`
-
-func FetchResults(ctx context.Context, c *bigquery.Client) (rows []*Result, err error) {
- return fetchResults(ctx, c, TableName)
-}
-
-func fetchResults(ctx context.Context, c *bigquery.Client, tableName string) (rows []*Result, err error) {
- name := c.FullTableName(tableName)
- query := bigquery.PartitionQuery{
- Table: name,
- PartitionOn: "module_path, scan_mode",
- OrderBy: orderByClauses,
- }.String()
- log.Infof(ctx, "running latest query on %s", name)
- iter, err := c.Query(ctx, query)
- if err != nil {
- return nil, err
- }
- rows, err = bigquery.All[Result](iter)
- if err != nil {
- return nil, err
- }
- log.Infof(ctx, "got %d rows", len(rows))
-
- // Check for duplicate rows.
- modvers := map[string]int{}
- for _, r := range rows {
- modvers[r.ModulePath+"@"+r.Version+" "+r.ScanMode]++
- }
- keys := maps.Keys(modvers)
- sort.Strings(keys)
- for _, k := range keys {
- if n := modvers[k]; n > 1 {
- return nil, fmt.Errorf("%s has %d rows", k, n)
- }
- }
- return rows, nil
-}
-
-type ReportResult struct {
- *Result
- ReportDate civil.Date `bigquery:"report_date"` // for reporting (e.g. dashboard)
- InsertedAt time.Time `bigquery:"inserted_at"` // to disambiguate if >1 insertion for same date
-}
-
-func init() {
- s, err := bigquery.InferSchema(ReportResult{})
- if err != nil {
- panic(err)
- }
- bigquery.AddTable(TableName+"-report", s)
-}
-
-func InsertResults(ctx context.Context, c *bigquery.Client, results []*Result, date civil.Date, allowDuplicates bool) (err error) {
- return insertResults(ctx, c, TableName+"-report", results, date, allowDuplicates)
-}
-
-func insertResults(ctx context.Context, c *bigquery.Client, reportTableName string, results []*Result, date civil.Date, allowDuplicates bool) (err error) {
- derrors.Wrap(&err, "InsertResults(%s)", date)
- // Create the report table if it doesn't exist.
- if err := c.CreateTable(ctx, reportTableName); err != nil {
- return err
- }
-
- if !allowDuplicates {
- query := fmt.Sprintf("SELECT COUNT(*) FROM `%s` WHERE report_date = '%s'",
- c.FullTableName(reportTableName), date)
- iter, err := c.Query(ctx, query)
- if err != nil {
- return err
- }
- var count struct {
- n int
- }
- err = iter.Next(&count)
- if err != nil && err != iterator.Done {
- return err
- }
- if count.n > 0 {
- return fmt.Errorf("already have %d rows for %s", count.n, date)
- }
- }
-
- now := time.Now()
- var rows []ReportResult
- for _, r := range results {
- rows = append(rows, ReportResult{Result: r, ReportDate: date, InsertedAt: now})
- }
-
- ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) // to avoid retrying forever on permanent errors
- defer cancel()
- const chunkSize = 1024 // Chunk rows to a void exceeding the maximum allowable request size.
- return bigquery.UploadMany(ctx, c, reportTableName, rows, chunkSize)
-}
diff --git a/internal/worker/enqueue.go b/internal/worker/enqueue.go
index 9471fc4..aaa6f76 100644
--- a/internal/worker/enqueue.go
+++ b/internal/worker/enqueue.go
@@ -10,11 +10,11 @@
"golang.org/x/pkgsite-metrics/internal/config"
"golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/pkgsitedb"
"golang.org/x/pkgsite-metrics/internal/queue"
"golang.org/x/pkgsite-metrics/internal/scan"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
const defaultMinImportedByCount = 10
@@ -73,15 +73,15 @@
return nil
}
-func moduleSpecsToScanRequests(modspecs []scan.ModuleSpec, mode string) []*ivulncheck.Request {
- var sreqs []*ivulncheck.Request
+func moduleSpecsToScanRequests(modspecs []scan.ModuleSpec, mode string) []*govulncheck.Request {
+ var sreqs []*govulncheck.Request
for _, ms := range modspecs {
- sreqs = append(sreqs, &ivulncheck.Request{
+ sreqs = append(sreqs, &govulncheck.Request{
ModuleURLPath: scan.ModuleURLPath{
Module: ms.Path,
Version: ms.Version,
},
- QueryParams: ivulncheck.QueryParams{
+ QueryParams: govulncheck.QueryParams{
ImportedBy: ms.ImportedBy,
Mode: mode,
},
diff --git a/internal/worker/govulncheck.go b/internal/worker/govulncheck.go
index 7561359..9fd8890 100644
--- a/internal/worker/govulncheck.go
+++ b/internal/worker/govulncheck.go
@@ -11,23 +11,23 @@
"runtime/debug"
"golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/log"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
type VulncheckServer struct {
*Server
- storedWorkVersions map[[2]string]*ivulncheck.WorkVersion
- workVersion *ivulncheck.WorkVersion
+ storedWorkVersions map[[2]string]*govulncheck.WorkVersion
+ workVersion *govulncheck.WorkVersion
}
func newVulncheckServer(ctx context.Context, s *Server) (*VulncheckServer, error) {
var (
- swv map[[2]string]*ivulncheck.WorkVersion
+ swv map[[2]string]*govulncheck.WorkVersion
err error
)
if s.bqClient != nil {
- swv, err = ivulncheck.ReadWorkVersions(ctx, s.bqClient)
+ swv, err = govulncheck.ReadWorkVersions(ctx, s.bqClient)
if err != nil {
return nil, err
}
@@ -39,7 +39,7 @@
}, nil
}
-func (h *VulncheckServer) getWorkVersion(ctx context.Context) (_ *ivulncheck.WorkVersion, err error) {
+func (h *VulncheckServer) getWorkVersion(ctx context.Context) (_ *govulncheck.WorkVersion, err error) {
defer derrors.Wrap(&err, "VulncheckServer.getWorkVersion")
h.mu.Lock()
defer h.mu.Unlock()
@@ -53,10 +53,10 @@
if err != nil {
return nil, err
}
- h.workVersion = &ivulncheck.WorkVersion{
+ h.workVersion = &govulncheck.WorkVersion{
VulnDBLastModified: lmt,
WorkerVersion: h.cfg.VersionID,
- SchemaVersion: ivulncheck.SchemaVersion,
+ SchemaVersion: govulncheck.SchemaVersion,
VulnVersion: vulnVersion,
}
log.Infof(ctx, "vulncheck work version: %+v", h.workVersion)
diff --git a/internal/worker/govulncheck_enqueue.go b/internal/worker/govulncheck_enqueue.go
index 1eca9e9..0a6dad5 100644
--- a/internal/worker/govulncheck_enqueue.go
+++ b/internal/worker/govulncheck_enqueue.go
@@ -16,10 +16,10 @@
"golang.org/x/exp/maps"
"golang.org/x/pkgsite-metrics/internal/config"
"golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/queue"
"golang.org/x/pkgsite-metrics/internal/scan"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
"google.golang.org/api/iterator"
)
@@ -35,7 +35,7 @@
func (h *VulncheckServer) enqueue(r *http.Request, allModes bool) error {
ctx := r.Context()
- params := &ivulncheck.EnqueueQueryParams{Min: defaultMinImportedByCount}
+ params := &govulncheck.EnqueueQueryParams{Min: defaultMinImportedByCount}
if err := scan.ParseParams(r, params); err != nil {
return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
}
@@ -67,14 +67,14 @@
return []string{mode}, nil
}
-func createVulncheckQueueTasks(ctx context.Context, cfg *config.Config, params *ivulncheck.EnqueueQueryParams, modes []string) (_ []queue.Task, err error) {
+func createVulncheckQueueTasks(ctx context.Context, cfg *config.Config, params *govulncheck.EnqueueQueryParams, modes []string) (_ []queue.Task, err error) {
defer derrors.Wrap(&err, "createVulncheckQueueTasks(%v)", modes)
var (
tasks []queue.Task
modspecs []scan.ModuleSpec
)
for _, mode := range modes {
- var reqs []*ivulncheck.Request
+ var reqs []*govulncheck.Request
if mode == ModeBinary {
reqs, err = readBinaries(ctx, cfg.BinaryBucket)
if err != nil {
@@ -113,7 +113,7 @@
// gcsBinaryDir is the directory in the GCS bucket that contains binaries that should be scanned.
const gcsBinaryDir = "binaries"
-func readBinaries(ctx context.Context, bucketName string) (reqs []*ivulncheck.Request, err error) {
+func readBinaries(ctx context.Context, bucketName string) (reqs []*govulncheck.Request, err error) {
defer derrors.Wrap(&err, "readBinaries(%q)", bucketName)
if bucketName == "" {
log.Infof(ctx, "binary bucket not configured; not enqueuing binaries")
@@ -136,9 +136,9 @@
if err != nil {
return nil, err
}
- reqs = append(reqs, &ivulncheck.Request{
+ reqs = append(reqs, &govulncheck.Request{
ModuleURLPath: mp,
- QueryParams: ivulncheck.QueryParams{Mode: ModeBinary},
+ QueryParams: govulncheck.QueryParams{Mode: ModeBinary},
})
}
return reqs, nil
diff --git a/internal/worker/govulncheck_enqueue_test.go b/internal/worker/govulncheck_enqueue_test.go
index 6e4535b..65305b0 100644
--- a/internal/worker/govulncheck_enqueue_test.go
+++ b/internal/worker/govulncheck_enqueue_test.go
@@ -12,9 +12,9 @@
"github.com/google/go-cmp/cmp"
"golang.org/x/pkgsite-metrics/internal/config"
+ "golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/queue"
"golang.org/x/pkgsite-metrics/internal/scan"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
var binaryBucket = flag.String("binary-bucket", "", "bucket for scannable binaries")
@@ -27,13 +27,13 @@
if err != nil {
t.Fatal(err)
}
- want := &ivulncheck.Request{
+ want := &govulncheck.Request{
ModuleURLPath: scan.ModuleURLPath{
Module: "golang.org/x/pkgsite",
Version: "v0.0.0-20221004150836-873fb37c2479",
Suffix: "cmd/worker",
},
- QueryParams: ivulncheck.QueryParams{Mode: ModeBinary},
+ QueryParams: govulncheck.QueryParams{Mode: ModeBinary},
}
found := false
for _, sr := range sreqs {
@@ -51,14 +51,14 @@
}
func TestCreateQueueTasks(t *testing.T) {
- vreq := func(path, version, mode string, importedBy int) *ivulncheck.Request {
- return &ivulncheck.Request{
+ vreq := func(path, version, mode string, importedBy int) *govulncheck.Request {
+ return &govulncheck.Request{
ModuleURLPath: scan.ModuleURLPath{Module: path, Version: version},
- QueryParams: ivulncheck.QueryParams{Mode: mode, ImportedBy: importedBy},
+ QueryParams: govulncheck.QueryParams{Mode: mode, ImportedBy: importedBy},
}
}
- params := &ivulncheck.EnqueueQueryParams{Min: 8, File: "testdata/modules.txt"}
+ params := &govulncheck.EnqueueQueryParams{Min: 8, File: "testdata/modules.txt"}
gotTasks, err := createVulncheckQueueTasks(context.Background(), &config.Config{}, params, []string{ModeGovulncheck})
if err != nil {
t.Fatal(err)
@@ -68,7 +68,7 @@
vreq("github.com/pkg/errors", "v0.9.1", ModeGovulncheck, 10),
vreq("golang.org/x/net", "v0.4.0", ModeGovulncheck, 20),
}
- if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(ivulncheck.Request{})); diff != "" {
+ if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(govulncheck.Request{})); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
@@ -86,7 +86,7 @@
vreq("golang.org/x/net", "v0.4.0", ModeGovulncheck, 20),
}
- if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(ivulncheck.Request{})); diff != "" {
+ if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(govulncheck.Request{})); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
}
diff --git a/internal/worker/govulncheck_results.go b/internal/worker/govulncheck_results.go
index c743279..af2f7d3 100644
--- a/internal/worker/govulncheck_results.go
+++ b/internal/worker/govulncheck_results.go
@@ -12,9 +12,9 @@
"cloud.google.com/go/civil"
"golang.org/x/exp/event"
"golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/scan"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
)
var insertResultsCounter = event.NewCounter("insert-results", &event.MetricOptions{Namespace: metricNamespace})
@@ -44,12 +44,12 @@
}
ctx := r.Context()
log.Infof(ctx, "reading results")
- results, err := ivulncheck.FetchResults(ctx, h.bqClient)
+ results, err := govulncheck.FetchResults(ctx, h.bqClient)
if err != nil {
return err
}
log.Infof(ctx, "inserting %d results for %s", len(results), date)
- if err := ivulncheck.InsertResults(ctx, h.bqClient, results, date, allowDuplicates); err != nil {
+ if err := govulncheck.InsertResults(ctx, h.bqClient, results, date, allowDuplicates); err != nil {
return err
}
fmt.Fprintf(w, "%d results for %s inserted successfully.\n", len(results), date)
diff --git a/internal/worker/govulncheck_scan.go b/internal/worker/govulncheck_scan.go
index 92d1f29..6055dc3 100644
--- a/internal/worker/govulncheck_scan.go
+++ b/internal/worker/govulncheck_scan.go
@@ -27,7 +27,6 @@
"golang.org/x/pkgsite-metrics/internal/proxy"
"golang.org/x/pkgsite-metrics/internal/sandbox"
"golang.org/x/pkgsite-metrics/internal/version"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
vulnclient "golang.org/x/vuln/client"
govulncheckapi "golang.org/x/vuln/exp/govulncheck"
)
@@ -76,7 +75,7 @@
}()
ctx := r.Context()
- sreq, err := ivulncheck.ParseRequest(r, "/vulncheck/scan")
+ sreq, err := govulncheck.ParseRequest(r, "/vulncheck/scan")
if err != nil {
return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
}
@@ -117,7 +116,7 @@
return nil
}
var err error
- h.storedWorkVersions, err = ivulncheck.ReadWorkVersions(ctx, h.bqClient)
+ h.storedWorkVersions, err = govulncheck.ReadWorkVersions(ctx, h.bqClient)
return err
}
@@ -126,7 +125,7 @@
proxyClient *proxy.Client
dbClient vulnclient.Client
bqClient *bigquery.Client
- workVersion *ivulncheck.WorkVersion
+ workVersion *govulncheck.WorkVersion
goMemLimit uint64
gcsBucket *storage.BucketHandle
insecure bool
@@ -172,11 +171,11 @@
return s.err
}
-func (s *scanner) ScanModule(ctx context.Context, w http.ResponseWriter, sreq *ivulncheck.Request) error {
+func (s *scanner) ScanModule(ctx context.Context, w http.ResponseWriter, sreq *govulncheck.Request) error {
if sreq.Module == "std" {
return nil // ignore the standard library
}
- row := &ivulncheck.Result{
+ row := &govulncheck.Result{
ModulePath: sreq.Module,
Suffix: sreq.Suffix,
WorkVersion: *s.workVersion,
@@ -227,7 +226,7 @@
}
log.Infof(ctx, "scanner.runScanModule returned %d vulns for %s: row.Vulns=%d err=%v", len(vulns), sreq.Path(), len(row.Vulns), err)
- if err := writeResult(ctx, sreq.Serve, w, s.bqClient, ivulncheck.TableName, row); err != nil {
+ if err := writeResult(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, row); err != nil {
return err
}
@@ -245,7 +244,7 @@
impRow.ScanMemory = 0
impRow.Vulns = vulnsForMode(vulns, modeImports)
log.Infof(ctx, "scanner.runScanModule also storing imports vulns for %s: row.Vulns=%d", sreq.Path(), len(impRow.Vulns))
- return writeResult(ctx, sreq.Serve, w, s.bqClient, ivulncheck.TableName, &impRow)
+ return writeResult(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, &impRow)
}
// vulnsForMode returns vulns that make sense to report for
@@ -256,12 +255,12 @@
// modified to have CallSink=0. For ModeBinary, these are
// exactly the input vulns since binary analysis does not
// distinguish between called and imported vulnerabilities.
-func vulnsForMode(vulns []*ivulncheck.Vuln, mode string) []*ivulncheck.Vuln {
+func vulnsForMode(vulns []*govulncheck.Vuln, mode string) []*govulncheck.Vuln {
if mode == ModeBinary {
return vulns
}
- var vs []*ivulncheck.Vuln
+ var vs []*govulncheck.Vuln
for _, v := range vulns {
if mode == ModeGovulncheck {
// Return only the called vulns for ModeGovulncheck.
@@ -298,7 +297,7 @@
// runScanModule fetches the module version from the proxy, and analyzes it for
// vulnerabilities.
-func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *scanStats) (bvulns []*ivulncheck.Vuln, err error) {
+func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *scanStats) (bvulns []*govulncheck.Vuln, err error) {
err = doScan(ctx, modulePath, version, s.insecure, func() error {
var vulns []*govulncheckapi.Vuln
if s.insecure {
@@ -310,7 +309,7 @@
return err
}
for _, v := range vulns {
- bvulns = append(bvulns, ivulncheck.ConvertGovulncheckOutput(v)...)
+ bvulns = append(bvulns, govulncheck.ConvertGovulncheckOutput(v)...)
}
return nil
})
diff --git a/internal/worker/govulncheck_scan_test.go b/internal/worker/govulncheck_scan_test.go
index 660fa15..1c2e6ec 100644
--- a/internal/worker/govulncheck_scan_test.go
+++ b/internal/worker/govulncheck_scan_test.go
@@ -17,8 +17,8 @@
"golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/config"
"golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/proxy"
- ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
vulnc "golang.org/x/vuln/client"
)
@@ -35,13 +35,13 @@
}
func TestVulnsForMode(t *testing.T) {
- vulns := []*ivulncheck.Vuln{
- &ivulncheck.Vuln{Symbol: "A", CallSink: bigquery.NullInt(0)},
- &ivulncheck.Vuln{Symbol: "B"},
- &ivulncheck.Vuln{Symbol: "C", CallSink: bigquery.NullInt(9)},
+ vulns := []*govulncheck.Vuln{
+ &govulncheck.Vuln{Symbol: "A", CallSink: bigquery.NullInt(0)},
+ &govulncheck.Vuln{Symbol: "B"},
+ &govulncheck.Vuln{Symbol: "C", CallSink: bigquery.NullInt(9)},
}
- vulnsStr := func(vulns []*ivulncheck.Vuln) string {
+ vulnsStr := func(vulns []*govulncheck.Vuln) string {
var vs []string
for _, v := range vulns {
vs = append(vs, fmt.Sprintf("%s:%d", v.Symbol, v.CallSink.Int64))
