internal: add vulncheck metrics pipeline
An initial pipeline is added to measure impact of running functions from
x/vuln/vulncheck on a set of modules downloaded from the Go module
proxy.
Change-Id: I4f3f49653b3e05c5890177955e5b7824af45ec4c
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/465215
Reviewed-by: Julie Qiu <julieqiu@google.com>
Run-TryBot: Julie Qiu <julie@golang.org>
Auto-Submit: Julie Qiu <julieqiu@google.com>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Julie Qiu <julieqiu@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index b9e90d4..6454383 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -22,7 +22,7 @@
workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
devMode = flag.Bool("dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)")
port = flag.String("port", config.GetEnv("PORT", "8080"), "port to listen to")
- dataset = flag.String("dataset", "", "dataset (overrides GO_ECOSYSTEM_METRICS_BIGQUERY_DATASET env var); use 'disable' for no BQ")
+ dataset = flag.String("dataset", "", "dataset (overrides GO_ECOSYSTEM_BIGQUERY_DATASET env var); use 'disable' for no BQ")
insecure = flag.Bool("insecure", false, "bypass sandbox in order to compare with old code")
// flag used in call to safehtml/template.TrustedSourceFromFlag
_ = flag.String("static", "static", "path to folder containing static files served")
diff --git a/go.mod b/go.mod
index a922848..6744930 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@
cloud.google.com/go/errorreporting v0.1.0
cloud.google.com/go/logging v1.6.1
cloud.google.com/go/secretmanager v1.9.0
+ cloud.google.com/go/storage v1.27.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go v1.0.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.26.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.0.0
@@ -26,6 +27,7 @@
golang.org/x/mod v0.7.0
golang.org/x/net v0.5.0
golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2
+ golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1
google.golang.org/api v0.103.0
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c
google.golang.org/grpc v1.50.1
diff --git a/go.sum b/go.sum
index 4bbfa8e..859969e 100644
--- a/go.sum
+++ b/go.sum
@@ -73,6 +73,7 @@
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ=
+cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
cloud.google.com/go/trace v1.4.0 h1:qO9eLn2esajC9sxpqp1YKX37nXC3L4BfGnPS0Cx9dYo=
cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -618,6 +619,8 @@
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2 h1:v0FhRDmSCNH/0EurAT6T8KRY4aNuUhz6/WwBMxG+gvQ=
golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
+golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1 h1:HRexnHfiDA2hkPNMDgf3vxabRMeC+XeS8tCKP9olVbs=
+golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1/go.mod h1:cBP4HMKv0X+x96j8IJWCKk0eqpakBmmHjKGSSC0NaYE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/internal/bigquery/bigquery_test.go b/internal/bigquery/bigquery_test.go
new file mode 100644
index 0000000..34d85fb
--- /dev/null
+++ b/internal/bigquery/bigquery_test.go
@@ -0,0 +1,234 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bigquery
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "sort"
+ "testing"
+ "time"
+
+ bq "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "golang.org/x/pkgsite-metrics/internal/version"
+ "google.golang.org/api/iterator"
+)
+
+var integration = flag.Bool("integration", false, "test against actual service")
+
+func TestIntegration(t *testing.T) {
+ must := func(err error) {
+ t.Helper()
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if !*integration {
+ t.Skip("missing -integration")
+ }
+ ctx := context.Background()
+ const projectID = "go-ecosystem"
+
+ // Create a new dataset ID to avoid problems with re-using existing tables.
+ dsID := fmt.Sprintf("test_%s", time.Now().Format("20060102T030405"))
+ t.Logf("using dataset %s", dsID)
+ client, err := NewClientCreate(ctx, projectID, dsID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ must(client.dataset.Delete(ctx))
+ }()
+
+ if _, err := client.CreateOrUpdateTable(ctx, VulncheckTableName); err != nil {
+ t.Fatal(err)
+ }
+ defer func() { must(client.Table(VulncheckTableName).Delete(ctx)) }()
+
+ tm := time.Date(2022, 7, 21, 0, 0, 0, 0, time.UTC)
+ row := &VulnResult{
+ ModulePath: "m",
+ Version: "v",
+ SortVersion: "sv",
+ ImportedBy: 10,
+ VulncheckWorkVersion: VulncheckWorkVersion{
+ WorkerVersion: "1",
+ SchemaVersion: "s",
+ VulnVersion: "2",
+ VulnDBLastModified: tm,
+ },
+ }
+
+ t.Run("upload", func(t *testing.T) {
+ must(client.Upload(ctx, VulncheckTableName, row))
+ // Round, strip monotonic data and convert to UTC.
+ // Discrepancies of a few microseconds have been seen, so round to seconds
+ // just to be safe.
+ row.CreatedAt = row.CreatedAt.Round(time.Second).UTC()
+ gots, err := readTable[VulnResult](ctx, client.Table(VulncheckTableName), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if g, w := len(gots), 1; g != w {
+ t.Fatalf("got %d, rows, wanted %d", g, w)
+ }
+ got := gots[0]
+ got.CreatedAt = got.CreatedAt.Round(time.Second)
+ if diff := cmp.Diff(row, got); diff != "" {
+ t.Errorf("mismatch (-want, +got):\n%s", diff)
+ }
+ })
+ t.Run("work versions", func(t *testing.T) {
+ wv, err := ReadVulncheckWorkVersions(ctx, client)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wgot := wv[[2]string{"m", "v"}]
+ if wgot == nil {
+ t.Fatal("got nil, wanted work version")
+ }
+ if want := &row.VulncheckWorkVersion; !wgot.Equal(want) {
+ t.Errorf("got %+v, want %+v", wgot, want)
+ }
+
+ if got := wv[[2]string{"m", "v2"}]; got != nil {
+ t.Errorf("got %v; want nil", got)
+ }
+ })
+
+ t.Run("latest", func(t *testing.T) {
+ latestTableID := VulncheckTableName + "-latest"
+ addTable(latestTableID, tableSchema(VulncheckTableName))
+ must(client.CreateTable(ctx, latestTableID))
+ defer func() { must(client.Table(latestTableID).Delete(ctx)) }()
+
+ var want []*VulnResult
+ // Module "a": same work version, should get the latest module version.
+ a1 := &VulnResult{
+ ModulePath: "a",
+ Version: "v1.0.0",
+ ScanMode: "M1",
+ VulncheckWorkVersion: VulncheckWorkVersion{
+ WorkerVersion: "1",
+ SchemaVersion: "s",
+ VulnVersion: "2",
+ VulnDBLastModified: tm,
+ },
+ }
+ a2 := *a1
+ a2.Version = "v1.1.0"
+ want = append(want, &a2)
+
+ // Different scan mode: should get this one too.
+ a3 := a2
+ a3.ScanMode = "M2"
+ want = append(want, &a3)
+
+ // Module "b": same module version, should get the latest work version.
+ b1 := &VulnResult{
+ ModulePath: "b",
+ Version: "v1.0.0",
+ VulncheckWorkVersion: VulncheckWorkVersion{
+ WorkerVersion: "1",
+ SchemaVersion: "s",
+ VulnVersion: "2",
+ VulnDBLastModified: tm,
+ },
+ }
+ b2 := *b1
+ b2.WorkerVersion = "0"
+ want = append(want, b1)
+
+ vrs := []*VulnResult{
+ a1, &a2, &a3,
+ b1, &b2,
+ }
+ for _, vr := range vrs {
+ vr.SortVersion = version.ForSorting(vr.Version)
+ }
+ must(UploadMany(ctx, client, latestTableID, vrs, 20))
+
+ got, err := fetchVulncheckResults(ctx, client, latestTableID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sort.Slice(got, func(i, j int) bool { return got[i].ModulePath < got[j].ModulePath })
+ if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(VulnResult{}, "CreatedAt")); diff != "" {
+ t.Errorf("mismatch (-want, +got):\n%s", diff)
+ }
+
+ // Test InsertVulncheckResults
+ reportTableID := latestTableID + "-report"
+ addTable(reportTableID, tableSchema(VulncheckTableName+"-report"))
+ reportTable := client.dataset.Table(reportTableID)
+ // Table is created by InsertVulncheckResults.
+ defer func() { must(reportTable.Delete(ctx)) }()
+
+ if err := insertVulncheckResults(ctx, client, reportTableID, got, civil.DateOf(time.Now()), false); err != nil {
+ t.Fatal(err)
+ }
+ rgot, err := readTable[ReportVulnResult](ctx, reportTable, func() *ReportVulnResult {
+ return &ReportVulnResult{VulnResult: &VulnResult{}}
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantDate := civil.DateOf(time.Now())
+ for _, r := range rgot {
+ if r.ReportDate != wantDate {
+ t.Errorf("got %s, want %s", r.ReportDate, wantDate)
+ }
+ if d := time.Minute; time.Since(r.InsertedAt) > d {
+ t.Errorf("inserted at %s, more than %s ago", r.InsertedAt, d)
+ }
+ // Sanity check for VulnResult.
+ if r.ModulePath != "a" && r.ModulePath != "b" {
+ t.Errorf("got %q, want 'a' or 'b'", r.ModulePath)
+ }
+
+ }
+
+ })
+}
+
+func readTable[T any](ctx context.Context, table *bq.Table, newT func() *T) ([]*T, error) {
+ var ts []*T
+ if newT == nil {
+ newT = func() *T { return new(T) }
+ }
+ iter := table.Read(ctx)
+ for {
+ tp := newT()
+ err := iter.Next(tp)
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ ts = append(ts, tp)
+ }
+ return ts, nil
+}
+
+func TestIsNotFoundError(t *testing.T) {
+ if !*integration {
+ t.Skip("missing -integration")
+ }
+ client, err := bq.NewClient(context.Background(), "go-ecosystem")
+ if err != nil {
+ t.Fatal(err)
+ }
+ dataset := client.Dataset("nope")
+ _, err = dataset.Metadata(context.Background())
+ if !isNotFoundError(err) {
+ t.Errorf("got false, want true for %v", err)
+ }
+}
diff --git a/internal/bigquery/vulncheck.go b/internal/bigquery/vulncheck.go
new file mode 100644
index 0000000..a583d75
--- /dev/null
+++ b/internal/bigquery/vulncheck.go
@@ -0,0 +1,235 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bigquery
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "time"
+
+ bq "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+ "golang.org/x/exp/maps"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "google.golang.org/api/iterator"
+)
+
+const VulncheckTableName = "vulncheck"
+
+// Note: before modifying VulnResult or Vuln, make sure the change
+// is a valid schema modification.
+// The only supported changes are:
+// - adding a nullable or repeated column
+// - dropping a column
+// - changing a column from required to nullable.
+// See https://cloud.google.com/bigquery/docs/managing-table-schemas for details.
+
+// VulnResult is a row in the BigQuery vulncheck table. It corresponds to a
+// result from the output for vulncheck.Source.
+type VulnResult struct {
+ CreatedAt time.Time `bigquery:"created_at"`
+ ModulePath string `bigquery:"module_path"`
+ Version string `bigquery:"version"`
+ Suffix string `bigquery:"suffix"`
+ SortVersion string `bigquery:"sort_version"`
+ ImportedBy int `bigquery:"imported_by"`
+ Error string `bigquery:"error"`
+ ErrorCategory string `bigquery:"error_category"`
+ CommitTime time.Time `bigquery:"commit_time"`
+ ScanSeconds float64 `bigquery:"scan_seconds"`
+ ScanMemory int64 `bigquery:"scan_memory"`
+ PkgsMemory int64 `bigquery:"pkgs_memory"`
+ ScanMode string `bigquery:"scan_mode"`
+ // Workers is the concurrency limit under which a module is
+ // analyzed. Useful for interpreting memory measurements when
+ // there are multiple modules analyzed in the same process.
+ // 0 if no limit is specified, -1 for potential errors.
+ Workers int `bigquery:"workers"`
+ VulncheckWorkVersion // InferSchema flattens embedded fields
+ Vulns []*Vuln `bigquery:"vulns"`
+}
+
+// VulncheckWorkVersion contains information that can be used to avoid duplicate work.
+// Given two VulncheckWorkVersion values v1 and v2 for the same module path and version,
+// if v1.Equal(v2) then it is not necessary to scan the module.
+type VulncheckWorkVersion struct {
+ // The version of the currently running code. This tracks changes in the
+ // logic of module scanning and processing.
+ WorkerVersion string `bigquery:"worker_version"`
+ // The version of the bigquery schema.
+ SchemaVersion string ` bigquery:"schema_version"`
+ // The version of the golang.org/x/vuln module used by the current module.
+ VulnVersion string `bigquery:"x_vuln_version"`
+ // When the vuln DB was last modified.
+ VulnDBLastModified time.Time `bigquery:"vulndb_last_modified"`
+}
+
+func (v1 *VulncheckWorkVersion) Equal(v2 *VulncheckWorkVersion) bool {
+ if v1 == nil || v2 == nil {
+ return v1 == v2
+ }
+ return v1.WorkerVersion == v2.WorkerVersion &&
+ v1.SchemaVersion == v2.SchemaVersion &&
+ v1.VulnVersion == v2.VulnVersion &&
+ v1.VulnDBLastModified.Equal(v2.VulnDBLastModified)
+}
+
+func (vr *VulnResult) SetUploadTime(t time.Time) { vr.CreatedAt = t }
+
+func (vr *VulnResult) AddError(err error) {
+ if err == nil {
+ return
+ }
+ vr.Error = err.Error()
+ vr.ErrorCategory = derrors.CategorizeError(err)
+}
+
+// Vuln is a record in VulnResult and corresponds to an item in
+// vulncheck.Result.Vulns.
+type Vuln struct {
+ ID string `bigquery:"id"`
+ Symbol string `bigquery:"symbol"`
+ PackagePath string `bigquery:"package_path"`
+ ModulePath string `bigquery:"module_path"`
+ CallSink bq.NullInt64 `bigquery:"call_sink"`
+ ImportSink bq.NullInt64 `bigquery:"import_sink"`
+ RequireSink bq.NullInt64 `bigquery:"require_sink"`
+}
+
+// VulncheckSchemaVersion changes whenever the vulncheck schema changes.
+var VulncheckSchemaVersion string
+
+func init() {
+ s, err := bq.InferSchema(VulnResult{})
+ if err != nil {
+ panic(err)
+ }
+ VulncheckSchemaVersion = schemaVersion(s)
+ addTable(VulncheckTableName, s)
+}
+
+// ReadVulncheckWorkVersions reads the most recent WorkVersions in the vulncheck table.
+func ReadVulncheckWorkVersions(ctx context.Context, c *Client) (_ map[[2]string]*VulncheckWorkVersion, err error) {
+ defer derrors.Wrap(&err, "ReadVulncheckWorkVersions")
+ m := map[[2]string]*VulncheckWorkVersion{}
+ query := partitionQuery(c.FullTableName(VulncheckTableName), "module_path, sort_version", "created_at DESC")
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return nil, err
+ }
+ err = ForEachRow(iter, func(r *VulnResult) bool {
+ m[[2]string{r.ModulePath, r.Version}] = &r.VulncheckWorkVersion
+ return true
+ })
+ if err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+// The module path along with the four sort columns should uniquely specify a
+// row, because we do not generate a new row for a (module, version) if the
+// other three versions are identical. (There is actually a fourth component of
+// the work version, the schema version. But since it is represented by a struct
+// in the worker code and the worker version captures every change to that code,
+// it cannot change independently of worker_version.)
+const orderByClauses = `
+ vulndb_last_modified DESC, -- latest version of database
+ x_vuln_version DESC, -- latest version of x/vuln
+ worker_version DESC, -- latest version of x/pkgsite-metrics
+ sort_version DESC, -- latest version of module
+ created_at DESC -- latest insertion time
+`
+
+func FetchVulncheckResults(ctx context.Context, c *Client) (rows []*VulnResult, err error) {
+ return fetchVulncheckResults(ctx, c, VulncheckTableName)
+}
+
+func fetchVulncheckResults(ctx context.Context, c *Client, tableName string) (rows []*VulnResult, err error) {
+ name := c.FullTableName(tableName)
+ query := partitionQuery(name, "module_path, scan_mode", orderByClauses)
+ log.Infof(ctx, "running latest query on %s", name)
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return nil, err
+ }
+ rows, err = All[VulnResult](iter)
+ if err != nil {
+ return nil, err
+ }
+ log.Infof(ctx, "got %d rows", len(rows))
+
+ // Check for duplicate rows.
+ modvers := map[string]int{}
+ for _, r := range rows {
+ modvers[r.ModulePath+"@"+r.Version+" "+r.ScanMode]++
+ }
+ keys := maps.Keys(modvers)
+ sort.Strings(keys)
+ for _, k := range keys {
+ if n := modvers[k]; n > 1 {
+ return nil, fmt.Errorf("%s has %d rows", k, n)
+ }
+ }
+ return rows, nil
+}
+
+type ReportVulnResult struct {
+ *VulnResult
+ ReportDate civil.Date `bigquery:"report_date"` // for reporting (e.g. dashboard)
+ InsertedAt time.Time `bigquery:"inserted_at"` // to disambiguate if >1 insertion for same date
+}
+
+func init() {
+ s, err := bq.InferSchema(ReportVulnResult{})
+ if err != nil {
+ panic(err)
+ }
+ addTable(VulncheckTableName+"-report", s)
+}
+
+func InsertVulncheckResults(ctx context.Context, c *Client, results []*VulnResult, date civil.Date, allowDuplicates bool) (err error) {
+ return insertVulncheckResults(ctx, c, VulncheckTableName+"-report", results, date, allowDuplicates)
+}
+
+func insertVulncheckResults(ctx context.Context, c *Client, reportTableName string, results []*VulnResult, date civil.Date, allowDuplicates bool) (err error) {
+ derrors.Wrap(&err, "InsertVulncheckResults(%s)", date)
+ // Create the report table if it doesn't exist.
+ if err := c.CreateTable(ctx, reportTableName); err != nil {
+ return err
+ }
+
+ if !allowDuplicates {
+ query := fmt.Sprintf("SELECT COUNT(*) FROM `%s` WHERE report_date = '%s'",
+ c.FullTableName(reportTableName), date)
+ iter, err := c.Query(ctx, query)
+ if err != nil {
+ return err
+ }
+ var count struct {
+ n int
+ }
+ err = iter.Next(&count)
+ if err != nil && err != iterator.Done {
+ return err
+ }
+ if count.n > 0 {
+ return fmt.Errorf("already have %d rows for %s", count.n, date)
+ }
+ }
+
+ now := time.Now()
+ var rows []ReportVulnResult
+ for _, r := range results {
+ rows = append(rows, ReportVulnResult{VulnResult: r, ReportDate: date, InsertedAt: now})
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) // to avoid retrying forever on permanent errors
+ defer cancel()
+ const chunkSize = 1024 // Chunk rows to a void exceeding the maximum allowable request size.
+ return UploadMany(ctx, c, reportTableName, rows, chunkSize)
+}
diff --git a/internal/bigquery/vulncheck_test.go b/internal/bigquery/vulncheck_test.go
new file mode 100644
index 0000000..658ca84
--- /dev/null
+++ b/internal/bigquery/vulncheck_test.go
@@ -0,0 +1,34 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bigquery
+
+import (
+ "testing"
+
+ bq "cloud.google.com/go/bigquery"
+)
+
+func TestSchemaString(t *testing.T) {
+ type nest struct {
+ N []byte
+ M float64
+ }
+
+ type s struct {
+ A string
+ B int
+ C []bool
+ D nest
+ }
+ const want = "A,req:STRING;B,req:INTEGER;C,rep:BOOLEAN;D,req:(N,req:BYTES;M,req:FLOAT)"
+ schema, err := bq.InferSchema(s{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := schemaString(schema)
+ if got != want {
+ t.Errorf("\ngot %q\nwant %q", got, want)
+ }
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index f919027..974fe67 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -74,6 +74,13 @@
// DevMode indicates whether the server is running in development mode.
DevMode bool
+ // VulnDBBucketProjectID is the project ID for the vuln DB bucket and its
+ // associated load balancer.
+ VulnDBBucketProjectID string
+
+ // BinaryBucket holds binaries for vulncheck scanning.
+ BinaryBucket string
+
// The host, port and user of the pkgsite database used to find
// modules to scan.
PkgsiteDBHost string
@@ -86,8 +93,11 @@
// Run analysis binaries without sandbox.
Insecure bool
- // ProxyURL is the URL of the module proxy.
+ // ProxyURL is the url for the Go module proxy.
ProxyURL string
+
+ // VulnDBURL is the url for the Go vulnerability database.
+ VulnDBURL string
}
// Init resolves all configuration values provided by the config package. It
@@ -102,20 +112,23 @@
ts = template.TrustedSourceFromFlag(f.Value)
}
cfg := &Config{
- ProjectID: GetEnv("GOOGLE_CLOUD_PROJECT", "go-ecosystem"),
- ServiceID: "go-ecosystem-worker",
- VersionID: os.Getenv("DOCKER_IMAGE"),
- LocationID: "us-central1",
- StaticPath: ts,
- BigQueryDataset: GetEnv("GO_ECOSYSTEM_BIGQUERY_DATASET", "test"),
- QueueName: os.Getenv("GO_ECOSYSTEM_QUEUE_NAME"),
- QueueURL: os.Getenv("GO_ECOSYSTEM_QUEUE_URL"),
- PkgsiteDBHost: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_HOST", "localhost"),
- PkgsiteDBPort: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_PORT", "5432"),
- PkgsiteDBName: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_NAME", "discovery-db"),
- PkgsiteDBUser: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_USER", "postgres"),
- PkgsiteDBSecret: os.Getenv("GO_ECOSYSTEM_PKGSITE_DB_SECRET"),
- ProxyURL: GetEnv("GO_ECOSYSTEM_PROXY_URL", "https://proxy.golang.org"),
+ ProjectID: os.Getenv("GOOGLE_CLOUD_PROJECT"),
+ ServiceID: "go-ecosystem-worker",
+ VersionID: os.Getenv("DOCKER_IMAGE"),
+ LocationID: "us-central1",
+ StaticPath: ts,
+ BigQueryDataset: GetEnv("GO_ECOSYSTEM_BIGQUERY_DATASET", "test"),
+ QueueName: os.Getenv("GO_ECOSYSTEM_QUEUE_NAME"),
+ QueueURL: os.Getenv("GO_ECOSYSTEM_QUEUE_URL"),
+ VulnDBBucketProjectID: os.Getenv("GO_ECOSYSTEM_VULNDB_BUCKET_PROJECT"),
+ BinaryBucket: os.Getenv("GO_ECOSYSTEM_BINARY_BUCKET"),
+ PkgsiteDBHost: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_HOST", "localhost"),
+ PkgsiteDBPort: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_PORT", "5432"),
+ PkgsiteDBName: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_NAME", "discovery-db"),
+ PkgsiteDBUser: GetEnv("GO_ECOSYSTEM_PKGSITE_DB_USER", "postgres"),
+ PkgsiteDBSecret: os.Getenv("GO_ECOSYSTEM_PKGSITE_DB_SECRET"),
+ ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org"),
+ VulnDBURL: GetEnv("GO_VULNDB_URL", "https://vuln.go.dev"),
}
if OnCloudRun() {
sa, err := gceMetadata(ctx, "instance/service-accounts/default/email")
diff --git a/internal/derrors/derrors.go b/internal/derrors/derrors.go
index f9514b7..2e00c86 100644
--- a/internal/derrors/derrors.go
+++ b/internal/derrors/derrors.go
@@ -1,4 +1,4 @@
-// Copyright 2023 The Go Authors. All rights reserved.
+// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@@ -10,10 +10,13 @@
"errors"
"fmt"
"runtime"
+ "strings"
"cloud.google.com/go/errorreporting"
)
+//lint:file-ignore ST1012 prefixing error values with Err would stutter
+
var (
// NotFound indicates that a requested entity was not found (HTTP 404).
NotFound = errors.New("not found")
@@ -36,9 +39,59 @@
// ProxyError is used to capture non-actionable server errors returned from the proxy.
ProxyError = errors.New("proxy error")
+ // BigQueryError is used to capture server errors returned by BigQuery.
+ BigQueryError = errors.New("BigQuery error")
+
+ // ScanModulePanicError is used to capture panic issues.
+ ScanModulePanicError = errors.New("scan module panic")
+
// ScanModuleOSError is used to capture issues with writing the module zip
// to disk during the scan setup process. This is not an error with vulncheck.
ScanModuleOSError = errors.New("scan module OS error")
+
+ // ScanModuleLoadPackagesError is used to capture general unclassified issues
+ // with load packages during the scan setup process. This is not an error with
+ // vulncheck. There are specific load packages errors that are categorized
+ // separately, e.g., ScanModuleLoadPackagesNoGoModError.
+ ScanModuleLoadPackagesError = errors.New("scan module load packages error")
+
+ // ScanModuleLoadPackagesGoVersionError is used to capture issues with loading
+ // packages where the module is not supported by the current Go version. This
+ // is not an error with any specific scan technique.
+ ScanModuleLoadPackagesGoVersionError = errors.New("scan module load packages error: Go version mismatch")
+
+ // ScanModuleLoadPackagesNoGoModError is used to capture a specific issue
+ // with loading packages during the scan setup process where a go.mod file
+ // is missing. This is not an error with vulncheck.
+ ScanModuleLoadPackagesNoGoModError = errors.New("scan module load packages error: does not have go.mod")
+
+ // ScanModuleLoadPackagesNoGoSumError is used to capture a specific issue
+ // with loading packages during the scan setup process where a go.sum file
+ // is missing. This is not an error with vulncheck.
+ ScanModuleLoadPackagesNoGoSumError = errors.New("scan module load packages error: does not have go.sum")
+
+ // ScanModuleLoadPackagesNoRequiredModuleError is used to capture a specific
+ // issue with loading packages during the scan setup process where a package
+ // is imported but no required module is provided. This is not an error with
+ // vulncheck and is likely happening due to outdated go.sum file.
+ ScanModuleLoadPackagesNoRequiredModuleError = errors.New("scan module load packages error: no required module provided")
+
+ // ScanModuleLoadPackagesMissingGoSumEntryError is used to capture a specific
+ // issue with loading packages during the scan setup process where a package
+ // is imported but some of its go.sum entries are missing. This is not an error
+ // with vulncheck and is likely happening due to outdated go.sum file.
+ ScanModuleLoadPackagesMissingGoSumEntryError = errors.New("scan module load packages error: missing go.sum entry")
+
+ // ScanModuleVulncheckDBConnectionError is used to capture a specific
+ // vulncheck scan error where a connection to vuln db failed.
+ ScanModuleVulncheckDBConnectionError = errors.New("scan module vulncheck error: communication with vuln db failed")
+
+ // ScanModuleVulncheckError is used to capture general issues where
+ // vulncheck.Source fails due to an uncategorized error.
+ ScanModuleVulncheckError = errors.New("scan module vulncheck error")
+
+ // ScanModuleMemoryLimitExceeded occurs when scanning uses too much memory.
+ ScanModuleMemoryLimitExceeded = errors.New("scan module memory limit exceeded")
)
// Wrap adds context to the error and allows
@@ -114,3 +167,40 @@
repClient.Report(errorreporting.Entry{Error: err})
}
}
+
+// CategorizeError returns the category for a given error.
+func CategorizeError(err error) string {
+ switch {
+ case errors.Is(err, ScanModuleVulncheckError):
+ return "VULNCHECK - MISC"
+ case errors.Is(err, ScanModuleVulncheckDBConnectionError):
+ return "VULNCHECK - DB CONNECTION"
+ case errors.Is(err, ScanModuleLoadPackagesError):
+ return "LOAD"
+ case errors.Is(err, ScanModuleLoadPackagesGoVersionError):
+ return "LOAD - WRONG GO VERSION"
+ case errors.Is(err, ScanModuleLoadPackagesNoGoModError):
+ return "LOAD - NO GO.MOD"
+ case errors.Is(err, ScanModuleLoadPackagesNoGoSumError):
+ return "LOAD - NO GO.SUM"
+ case errors.Is(err, ScanModuleLoadPackagesNoRequiredModuleError):
+ return "LOAD - NO REQUIRED MODULE"
+ case errors.Is(err, ScanModuleLoadPackagesMissingGoSumEntryError):
+ return "LOAD - NO GO.SUM ENTRY"
+ case errors.Is(err, ScanModuleOSError):
+ return "OS"
+ case errors.Is(err, ScanModulePanicError):
+ return "PANIC"
+ case errors.Is(err, ScanModuleMemoryLimitExceeded):
+ return "MEM LIMIT EXCEEDED"
+ case errors.Is(err, ProxyError):
+ return "PROXY"
+ case errors.Is(err, BigQueryError):
+ return "BIGQUERY"
+ }
+ return ""
+}
+
+func IsGoVersionMismatchError(msg string) bool {
+ return strings.Contains(msg, "can't be built on Go")
+}
diff --git a/internal/worker/mem_monitor.go b/internal/worker/mem_monitor.go
new file mode 100644
index 0000000..48cf2d7
--- /dev/null
+++ b/internal/worker/mem_monitor.go
@@ -0,0 +1,56 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import "time"
+
+// memMonitor is used to probe memory consumption
+// in a separate Go routine.
+type memMonitor struct {
+ stp chan struct{} // for stopping the monitor
+ res chan uint64 // for communicating results
+}
+
+// newMemMonitor creates and starts new monitor that
+// samples memory consumption.
+//
+// If threshold > 0, then when memory consumption reaches threshold,
+// the monitor will call onThreshold and stop.
+func newMemMonitor(threshold uint64, onThreshold func()) *memMonitor {
+ m := &memMonitor{make(chan struct{}), make(chan uint64)}
+ var max uint64
+ go func() {
+ for {
+ select {
+ case <-m.stp:
+ m.res <- max
+ return
+ default:
+ h := currHeapUsage()
+ if h > max {
+ max = h
+ }
+ if threshold > 0 && h > threshold {
+ onThreshold()
+ m.res <- max
+ return
+ }
+ // We sample memory every 50ms.
+ time.Sleep(50 * time.Millisecond)
+ }
+ }
+ }()
+ return m
+}
+
+// stop stops the monitor and returns the peak
+// memory consumption in bytes.
+func (m *memMonitor) stop() uint64 {
+ if m == nil {
+ return 0
+ }
+ close(m.stp)
+ return <-m.res
+}
diff --git a/internal/worker/server.go b/internal/worker/server.go
index 8ad8b42..980f378 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -1,4 +1,4 @@
-// Copyright 2023 The Go Authors. All rights reserved.
+// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@@ -24,14 +24,19 @@
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/observe"
"golang.org/x/pkgsite-metrics/internal/proxy"
+ "golang.org/x/pkgsite-metrics/internal/queue"
+ "golang.org/x/pkgsite-metrics/internal/scan"
+ vulnc "golang.org/x/vuln/client"
)
type Server struct {
- cfg *config.Config
- observer *observe.Observer
- bqClient *bigquery.Client
- proxyClient *proxy.Client
- staticPath template.TrustedSource
+ cfg *config.Config
+ observer *observe.Observer
+ bqClient *bigquery.Client
+ vulndbClient vulnc.Client
+ proxyClient *proxy.Client
+ queue queue.Queue
+ staticPath template.TrustedSource
devMode bool
mu sync.Mutex
@@ -64,22 +69,38 @@
}
}
+ q, err := queue.New(ctx, cfg,
+ func(ctx context.Context, sreq *scan.Request) (int, error) {
+ // When running locally, only the module path and version are
+ // printed for now.
+ log.Infof(ctx, "enqueuing %s", sreq.URLPathAndParams())
+ return 0, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ dbClient, err := vulnc.NewClient([]string{cfg.VulnDBURL}, vulnc.Options{})
+ if err != nil {
+ return nil, err
+ }
proxyClient, err := proxy.New(cfg.ProxyURL)
if err != nil {
return nil, err
}
s := &Server{
- cfg: cfg,
- bqClient: bq,
- proxyClient: proxyClient,
- devMode: cfg.DevMode,
- staticPath: cfg.StaticPath,
+ cfg: cfg,
+ bqClient: bq,
+ vulndbClient: dbClient,
+ queue: q,
+ proxyClient: proxyClient,
+ devMode: cfg.DevMode,
+ staticPath: cfg.StaticPath,
}
if err := s.loadTemplates(); err != nil {
return nil, err
}
- s.observer, err = observe.NewObserver(ctx, cfg.ProjectID, "go-metrics-worker")
+ s.observer, err = observe.NewObserver(ctx, cfg.ProjectID, "go-ecosystem-worker")
if err != nil {
return nil, err
}
@@ -111,10 +132,18 @@
return nil
})
s.handle("/", s.handleIndexPage)
+
+ if err := s.registerVulncheckHandlers(ctx); err != nil {
+ return nil, err
+ }
+
+ s.handle("/test-vulncheck-sandbox/", s.handleTestVulncheckSandbox)
+ s.handle("/test-db", s.handleTestDB)
+
return s, nil
}
-const metricNamespace = "ecosystem/worker"
+const metricNamespace = "metrics/worker"
type handlerFunc func(w http.ResponseWriter, r *http.Request) error
@@ -138,6 +167,21 @@
http.Handle(pattern, s.observer.Observe(h))
}
+func (s *Server) registerVulncheckHandlers(ctx context.Context) error {
+ h, err := newVulncheckServer(ctx, s)
+ if err != nil {
+ return err
+ }
+ // returns an HTML page displaying information about vulncheck.
+ s.handle("/vulncheck", h.handlePage)
+
+ s.handle("/vulncheck/enqueueall", h.handleEnqueueAll)
+ s.handle("/vulncheck/enqueue", h.handleEnqueue)
+ s.handle("/vulncheck/scan/", h.handleScan)
+ s.handle("/vulncheck/insert-results", h.handleInsertResults)
+ return nil
+}
+
type serverError struct {
status int // HTTP status code
err error // wrapped error
diff --git a/internal/worker/templates.go b/internal/worker/templates.go
index 9753459..71d62d1 100644
--- a/internal/worker/templates.go
+++ b/internal/worker/templates.go
@@ -1,4 +1,4 @@
-// Copyright 2023 The Go Authors. All rights reserved.
+// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@@ -18,7 +18,8 @@
)
const (
- indexTemplate = "worker.tmpl"
+ indexTemplate = "index.tmpl"
+ vulncheckTemplate = "vulncheck.tmpl"
)
type basePage struct {
@@ -33,8 +34,13 @@
if err != nil {
return err
}
+ vulncheck, err := s.parseTemplate(template.TrustedSourceFromConstant(vulncheckTemplate))
+ if err != nil {
+ return err
+ }
s.templates = map[string]*template.Template{
- indexTemplate: index,
+ indexTemplate: index,
+ vulncheckTemplate: vulncheck,
}
return nil
}
diff --git a/internal/worker/vulncheck.go b/internal/worker/vulncheck.go
new file mode 100644
index 0000000..a95530b
--- /dev/null
+++ b/internal/worker/vulncheck.go
@@ -0,0 +1,318 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "runtime/debug"
+
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+)
+
+type VulncheckServer struct {
+ *Server
+ storedWorkVersions map[[2]string]*bigquery.VulncheckWorkVersion
+ workVersion *bigquery.VulncheckWorkVersion
+}
+
+func newVulncheckServer(ctx context.Context, s *Server) (*VulncheckServer, error) {
+ var (
+ swv map[[2]string]*bigquery.VulncheckWorkVersion
+ err error
+ )
+ if s.bqClient != nil {
+ swv, err = bigquery.ReadVulncheckWorkVersions(ctx, s.bqClient)
+ if err != nil {
+ return nil, err
+ }
+ log.Infof(ctx, "read %d work versions", len(swv))
+ }
+ return &VulncheckServer{
+ Server: s,
+ storedWorkVersions: swv,
+ }, nil
+}
+
+func (h *VulncheckServer) getWorkVersion(ctx context.Context) (_ *bigquery.VulncheckWorkVersion, err error) {
+ defer derrors.Wrap(&err, "VulncheckServer.getWorkVersion")
+ h.mu.Lock()
+ defer h.mu.Unlock()
+
+ if h.workVersion == nil {
+ lmt, err := h.vulndbClient.LastModifiedTime(ctx)
+ if err != nil {
+ return nil, err
+ }
+ vulnVersion, err := readVulnVersion()
+ if err != nil {
+ return nil, err
+ }
+ h.workVersion = &bigquery.VulncheckWorkVersion{
+ VulnDBLastModified: lmt,
+ WorkerVersion: h.cfg.VersionID,
+ SchemaVersion: bigquery.VulncheckSchemaVersion,
+ VulnVersion: vulnVersion,
+ }
+ log.Infof(ctx, "vulncheck work version: %+v", h.workVersion)
+ }
+ return h.workVersion, nil
+}
+
+// readVulnVersion returns the version of the golang.org/x/vuln module linked into
+// the current binary.
+func readVulnVersion() (string, error) {
+ const modulePath = "golang.org/x/vuln"
+ info, ok := debug.ReadBuildInfo()
+ if !ok {
+ return "", errors.New("vuln version not available")
+ }
+ for _, mod := range info.Deps {
+ if mod.Path == modulePath {
+ if mod.Replace != nil {
+ mod = mod.Replace
+ }
+ return mod.Version, nil
+ }
+ }
+ return "", fmt.Errorf("module %s not found", modulePath)
+}
+
+func (h *VulncheckServer) handlePage(w http.ResponseWriter, r *http.Request) error {
+ ctx := r.Context()
+ page, err := h.createVulncheckPage(ctx)
+ if err != nil {
+ return err
+ }
+ tmpl, err := h.Server.maybeLoadTemplate(vulncheckTemplate)
+ if err != nil {
+ return err
+ }
+ return renderPage(ctx, w, page, tmpl)
+}
+
+func (h *VulncheckServer) createVulncheckPage(ctx context.Context) (*VulncheckPage, error) {
+ if h.bqClient == nil {
+ return nil, errBQDisabled
+ }
+ table := h.bqClient.FullTableName(bigquery.VulncheckTableName)
+ page := newPage(table)
+ page.basePage = newBasePage()
+
+ rows, err := bigquery.FetchVulncheckResults(ctx, h.bqClient)
+ if err != nil {
+ return nil, err
+ }
+
+ vulnsScanned := handleVulncheckRows(ctx, page, rows)
+ page.NumVulnsScanned = len(vulnsScanned)
+ page.addErrors()
+ return page, nil
+}
+
+type VulncheckPage struct {
+ basePage
+
+ TableName string
+
+ NumVulnsInDatabase int
+ NumVulnsScanned int
+
+ VTAResult *VulncheckResult
+ VTAStacksResult *VulncheckResult
+ ImportsResult *VulncheckResult
+
+ Errors []*ErrorCategory
+}
+
+type ErrorCategory struct {
+ Name string
+ VTANumModules int
+ ImportsNumModules int
+}
+
+func (p *VulncheckPage) PercentVulnsScanned() float64 {
+ return (float64(p.NumVulnsScanned) / float64(p.NumVulnsInDatabase)) * 100
+}
+
+func (p *VulncheckPage) NumModulesSuccess() int {
+ return p.VTAResult.NumModulesSuccess + p.ImportsResult.NumModulesSuccess
+}
+
+type VulncheckResult struct {
+ NumModulesScanned int
+ NumModulesSuccess int
+ NumModulesError int
+ NumModulesVuln int
+
+ ErrorCategory map[string]int
+
+ maxScanSeconds float64
+ sumScanSeconds float64
+
+ maxScanMemory float64
+ sumScanMemory float64
+}
+
+func (v *VulncheckResult) AverageScanSeconds() float64 {
+ return v.sumScanSeconds / float64(v.NumModulesSuccess)
+}
+
+func (v *VulncheckResult) MaxScanSeconds() float64 {
+ return v.maxScanSeconds
+}
+
+// AverageScanMemory in megabytes.
+func (v *VulncheckResult) AverageScanMemory() float64 {
+ return v.sumScanMemory / (float64(v.NumModulesSuccess) * 1024 * 1024)
+}
+
+// MaxScanMemory in megabytes.
+func (v *VulncheckResult) MaxScanMemory() float64 {
+ return v.maxScanMemory / (1024 * 1024)
+}
+
+func (v *VulncheckResult) NumModulesNoVuln() int {
+ return v.NumModulesSuccess - v.NumModulesVuln
+}
+
+func (v *VulncheckResult) PercentSuccess() float64 {
+ return (float64(v.NumModulesSuccess) / float64(v.NumModulesScanned)) * 100
+}
+
+func (v *VulncheckResult) PercentFailed() float64 {
+ return (float64(v.NumModulesError) / float64(v.NumModulesScanned)) * 100
+}
+
+func (v *VulncheckResult) PercentVuln() float64 {
+ return (float64(v.NumModulesVuln) / float64(v.NumModulesSuccess)) * 100
+}
+
+func (v *VulncheckResult) PercentNoVuln() float64 {
+ return (float64(v.NumModulesNoVuln()) / float64(v.NumModulesSuccess)) * 100
+}
+
+func (r *VulncheckResult) update(row *bigquery.VulnResult) {
+ r.NumModulesScanned++
+ if row.Error != "" {
+ r.NumModulesError++
+ r.ErrorCategory[row.ErrorCategory]++
+ return
+ }
+ r.NumModulesSuccess++
+
+ s := row.ScanSeconds
+ if s > r.maxScanSeconds {
+ r.maxScanSeconds = s
+ }
+ r.sumScanSeconds += s
+
+ m := float64(row.ScanMemory)
+ if m > r.maxScanMemory {
+ r.maxScanMemory = m
+ }
+ r.sumScanMemory += m
+
+ if len(row.Vulns) > 0 {
+ if row.ScanMode == ModeImports {
+ r.NumModulesVuln++
+ } else {
+ // VTA and VTA with stacks mode.
+ for _, v := range row.Vulns {
+ if v.CallSink.Int64 > 0 {
+ r.NumModulesVuln++
+ break
+ }
+ }
+ }
+ }
+}
+
+// ReportResults contains aggregate results for a
+// vulnerability reports, such as number of modules
+// in which the vulnerability is found by vulncheck.
+type ReportResult struct {
+ VTANumModules int
+ ImportsNumModules int
+}
+
+// handleVulncheckRows populates page based on vulncheck result rows and
+// returns statistics for each vulnerability detected.
+func handleVulncheckRows(ctx context.Context, page *VulncheckPage, rows []*bigquery.VulnResult) map[string]*ReportResult {
+ vulnsScanned := map[string]*ReportResult{}
+ for _, row := range rows {
+ switch row.ScanMode {
+ case ModeVTA:
+ page.VTAResult.update(row)
+ case ModeVTAStacks:
+ page.VTAStacksResult.update(row)
+ case ModeImports:
+ page.ImportsResult.update(row)
+ default:
+ log.Errorf(ctx, "unexpected mode for %s@%s: %q", row.ModulePath, row.Version, row.ScanMode)
+ continue
+ }
+
+ // For each vuln, count the number of modules in which it
+ // was detected for each mode. Since a vuln in row.Vulns
+ // is defined by a symbol, make sure not to count multiple
+ // symbols of each vuln separately.
+ importsSeen := make(map[string]bool)
+ callsSeen := make(map[string]bool)
+ for _, v := range row.Vulns {
+ if _, ok := vulnsScanned[v.ID]; !ok {
+ vulnsScanned[v.ID] = &ReportResult{}
+ }
+ r := vulnsScanned[v.ID]
+
+ if row.ScanMode == ModeImports {
+ if !importsSeen[v.ID] {
+ r.ImportsNumModules++
+ }
+ importsSeen[v.ID] = true
+ }
+
+ if row.ScanMode == ModeVTA && v.CallSink.Int64 > 0 {
+ if !callsSeen[v.ID] {
+ r.VTANumModules++
+ }
+ callsSeen[v.ID] = true
+ }
+ }
+ }
+ return vulnsScanned
+}
+
+func newPage(table string) *VulncheckPage {
+ return &VulncheckPage{
+ TableName: table,
+ VTAResult: &VulncheckResult{ErrorCategory: make(map[string]int)},
+ VTAStacksResult: &VulncheckResult{ErrorCategory: make(map[string]int)},
+ ImportsResult: &VulncheckResult{ErrorCategory: make(map[string]int)},
+ }
+}
+
+func (page *VulncheckPage) addErrors() {
+ ecs := map[string]*ErrorCategory{}
+ for category, count := range page.VTAResult.ErrorCategory {
+ if _, ok := ecs[category]; !ok {
+ ecs[category] = &ErrorCategory{Name: category}
+ }
+ ecs[category].VTANumModules = count
+ }
+ for category, count := range page.ImportsResult.ErrorCategory {
+ if _, ok := ecs[category]; !ok {
+ ecs[category] = &ErrorCategory{Name: category}
+ }
+ ecs[category].ImportsNumModules = count
+ }
+ for _, ec := range ecs {
+ page.Errors = append(page.Errors, ec)
+ }
+}
diff --git a/internal/worker/vulncheck_enqueue.go b/internal/worker/vulncheck_enqueue.go
new file mode 100644
index 0000000..bf93a8f
--- /dev/null
+++ b/internal/worker/vulncheck_enqueue.go
@@ -0,0 +1,136 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "golang.org/x/pkgsite-metrics/internal/queue"
+ "golang.org/x/pkgsite-metrics/internal/scan"
+ "google.golang.org/api/iterator"
+)
+
+// handleEnqueue enqueues multiple modules for a single vulncheck mode.
+// Query params:
+// - suffix: appended to task queue IDs to generate unique tasks
+// - mode: type of analysis to run; see [modes]
+// - file: path to file containing modules; if missing, use DB
+// - min: minimum import-by count for a module to be included
+func (h *VulncheckServer) handleEnqueue(w http.ResponseWriter, r *http.Request) error {
+ ctx := r.Context()
+ suffix := r.FormValue("suffix")
+ mode, err := vulncheckMode(scan.ParseMode(r))
+ if err != nil {
+ return err
+ }
+
+ var sreqs []*scan.Request
+ if mode == ModeBinary {
+ var err error
+ sreqs, err = readBinaries(ctx, h.cfg.BinaryBucket)
+ if err != nil {
+ return err
+ }
+ } else {
+ minImpCount, err := scan.ParseOptionalIntParam(r, "min", defaultMinImportedByCount)
+ if err != nil {
+ return err
+ }
+ modspecs, err := readModules(ctx, h.cfg, r.FormValue("file"), minImpCount)
+ if err != nil {
+ return err
+ }
+ sreqs = moduleSpecsToScanRequests(modspecs, mode)
+ }
+ return enqueueModules(ctx, sreqs, h.queue, &queue.Options{Namespace: "vulncheck", TaskNameSuffix: suffix})
+}
+
+func vulncheckMode(mode string) (string, error) {
+ if mode == "" {
+ // VTA is the default mode
+ return ModeVTA, nil
+ }
+ mode = strings.ToUpper(mode)
+ if _, ok := modes[mode]; !ok {
+ return "", fmt.Errorf("unsupported mode: %v", mode)
+ }
+ return mode, nil
+}
+
+// handleEnqueueAll enqueues multiple modules for all vulncheck modes.
+// Query params:
+// - suffix: appended to task queue IDs to generate unique tasks
+// - file: path to file containing modules; if missing, use DB
+// - min: minimum import-by count for a module to be included
+func (h *VulncheckServer) handleEnqueueAll(w http.ResponseWriter, r *http.Request) error {
+ ctx := r.Context()
+ suffix := r.FormValue("suffix")
+ minImpCount, err := scan.ParseOptionalIntParam(r, "min", defaultMinImportedByCount)
+ if err != nil {
+ return err
+ }
+ modspecs, err := readModules(ctx, h.cfg, r.FormValue("file"), minImpCount)
+ if err != nil {
+ return err
+ }
+ opts := &queue.Options{Namespace: "vulncheck", TaskNameSuffix: suffix}
+ for mode := range modes {
+ var sreqs []*scan.Request
+ if mode == ModeBinary {
+ sreqs, err = readBinaries(ctx, h.cfg.BinaryBucket)
+ if err != nil {
+ return err
+ }
+ } else {
+ sreqs = moduleSpecsToScanRequests(modspecs, mode)
+ }
+ if err := enqueueModules(ctx, sreqs, h.queue, opts); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// binaryDir is the directory in the GCS bucket that contains binaries that should be scanned.
+const binaryDir = "binaries"
+
+func readBinaries(ctx context.Context, bucketName string) (sreqs []*scan.Request, err error) {
+ defer derrors.Wrap(&err, "readBinaries(%q)", bucketName)
+ if bucketName == "" {
+ log.Infof(ctx, "binary bucket not configured; not enqueuing binaries")
+ return nil, nil
+ }
+ c, err := storage.NewClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ iter := c.Bucket(bucketName).Objects(ctx, &storage.Query{Prefix: binaryDir})
+ for {
+ attrs, err := iter.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ mod, vers, suff, err := scan.ParseModuleVersionSuffix(strings.TrimPrefix(attrs.Name, binaryDir+"/"))
+ if err != nil {
+ return nil, err
+ }
+ sreqs = append(sreqs, &scan.Request{
+ Module: mod,
+ Version: vers,
+ Suffix: suff,
+ Mode: ModeBinary,
+ })
+ }
+ return sreqs, nil
+}
diff --git a/internal/worker/vulncheck_enqueue_test.go b/internal/worker/vulncheck_enqueue_test.go
new file mode 100644
index 0000000..83d2121
--- /dev/null
+++ b/internal/worker/vulncheck_enqueue_test.go
@@ -0,0 +1,44 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "flag"
+ "testing"
+
+ "golang.org/x/pkgsite-metrics/internal/scan"
+)
+
+var binaryBucket = flag.String("binary-bucket", "", "bucket for scannable binaries")
+
+func TestReadBinaries(t *testing.T) {
+ if *binaryBucket == "" {
+ t.Skip("missing -binary-bucket")
+ }
+ sreqs, err := readBinaries(context.Background(), *binaryBucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := &scan.Request{
+ Module: "golang.org/x/pkgsite",
+ Version: "v0.0.0-20221004150836-873fb37c2479",
+ Suffix: "cmd/worker",
+ Mode: ModeBinary,
+ }
+ found := false
+ for _, sr := range sreqs {
+ if *sr == *want {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("did not find %+v in results:", want)
+ for _, r := range sreqs {
+ t.Logf(" %+v", r)
+ }
+ }
+}
diff --git a/internal/worker/vulncheck_results.go b/internal/worker/vulncheck_results.go
new file mode 100644
index 0000000..31e9777
--- /dev/null
+++ b/internal/worker/vulncheck_results.go
@@ -0,0 +1,57 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "fmt"
+ "net/http"
+ "time"
+
+ "cloud.google.com/go/civil"
+ "golang.org/x/exp/event"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "golang.org/x/pkgsite-metrics/internal/scan"
+)
+
+var insertResultsCounter = event.NewCounter("insert-results", &event.MetricOptions{Namespace: metricNamespace})
+
+func (h *VulncheckServer) handleInsertResults(w http.ResponseWriter, r *http.Request) (err error) {
+ defer func() {
+ insertResultsCounter.Record(r.Context(), 1, event.Bool("success", err == nil))
+ }()
+
+ if h.bqClient == nil {
+ return errBQDisabled
+ }
+
+ var date civil.Date
+ if d := r.FormValue("date"); d != "" {
+ var err error
+ date, err = civil.ParseDate(d)
+ if err != nil {
+ return fmt.Errorf("%w: parsing 'date' query param: %v", derrors.InvalidArgument, err)
+ }
+ } else {
+ date = civil.DateOf(time.Now())
+ }
+ allowDuplicates, err := scan.ParseOptionalBoolParam(r, "allow-dups", false)
+ if err != nil {
+ return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
+ }
+ ctx := r.Context()
+ log.Infof(ctx, "reading results")
+ results, err := bigquery.FetchVulncheckResults(ctx, h.bqClient)
+ if err != nil {
+ return err
+ }
+ log.Infof(ctx, "inserting %d results for %s", len(results), date)
+ if err := bigquery.InsertVulncheckResults(ctx, h.bqClient, results, date, allowDuplicates); err != nil {
+ return err
+ }
+ fmt.Fprintf(w, "%d results for %s inserted successfully.\n", len(results), date)
+ return nil
+}
diff --git a/internal/worker/vulncheck_scan.go b/internal/worker/vulncheck_scan.go
new file mode 100644
index 0000000..200bb5c
--- /dev/null
+++ b/internal/worker/vulncheck_scan.go
@@ -0,0 +1,731 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "runtime/debug"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "golang.org/x/exp/event"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+ "golang.org/x/pkgsite-metrics/internal/config"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/load"
+ "golang.org/x/pkgsite-metrics/internal/log"
+ "golang.org/x/pkgsite-metrics/internal/modules"
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+ "golang.org/x/pkgsite-metrics/internal/sandbox"
+ "golang.org/x/pkgsite-metrics/internal/scan"
+ "golang.org/x/pkgsite-metrics/internal/version"
+ vulnc "golang.org/x/vuln/client"
+ "golang.org/x/vuln/vulncheck"
+)
+
+const (
+ // ModeVTA is the default vulncheck mode
+ ModeVTA string = "VTA"
+
+ // modeVTStacks is modeVTA where call stacks are
+ // computed additionally. Closely resembles the
+ // actual logic of govulncheck.
+ ModeVTAStacks string = "VTASTACKS"
+
+ // ModeImports only computes import-level analysis.
+ ModeImports string = "IMPORTS"
+
+ // ModeBinary runs vulncheck.Binary
+ ModeBinary string = "BINARY"
+)
+
+// modes is a set of supported vulncheck modes
+var modes = map[string]bool{
+ ModeImports: true,
+ ModeVTA: true,
+ ModeVTAStacks: true,
+ ModeBinary: true,
+}
+
+func IsValidVulncheckMode(mode string) bool {
+ return modes[mode]
+}
+
+// TODO(b/241402488): shouldSkip is the list of modules that we are not
+// currently scanning due to previous issues that need investigation.
+var shouldSkip = map[string]bool{}
+
+var scanCounter = event.NewCounter("scans", &event.MetricOptions{Namespace: metricNamespace})
+
+// path: /vulncheck/scan/MODULE_VERSION_SUFFIX
+// See scan.ParseRequest for allowed forms.
+// Query params:
+// - mode: type of analysis to run; see [modes]
+// - importedby: number of importers
+func (h *VulncheckServer) handleScan(w http.ResponseWriter, r *http.Request) (err error) {
+ defer derrors.Wrap(&err, "handleScan")
+
+ defer func() {
+ scanCounter.Record(r.Context(), 1, event.Bool("success", err == nil))
+ }()
+
+ ctx := r.Context()
+ sreq, err := scan.ParseRequest(r, "/vulncheck/scan")
+ if err != nil {
+ return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
+ }
+ if sreq.Mode == "" {
+ sreq.Mode = ModeVTA
+ }
+ if shouldSkip[sreq.Module] {
+ log.Infof(ctx, "skipping %s (module in shouldSkip list)", sreq.Path())
+ return nil
+ }
+ if err := h.readVulncheckWorkVersions(ctx); err != nil {
+ return err
+ }
+ scanner, err := newScanner(ctx, h)
+ if err != nil {
+ return err
+ }
+ // An explicit "insecure" query param overrides the default.
+ // This is temporary, for testing running in a sandbox.
+ if r.FormValue("insecure") != "" {
+ scanner.insecure = sreq.Insecure
+ }
+ wv := h.storedWorkVersions[[2]string{sreq.Module, sreq.Version}]
+ if scanner.workVersion.Equal(wv) {
+ log.Infof(ctx, "skipping %s@%s (work version unchanged)", sreq.Module, sreq.Version)
+ return nil
+ }
+
+ log.Infof(ctx, "scanning: %s", sreq.Path())
+ scanner.ScanModule(ctx, sreq)
+ log.Infof(ctx, "fetched and updated %s", sreq.Path())
+ return nil
+}
+
+func (h *VulncheckServer) readVulncheckWorkVersions(ctx context.Context) error {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if h.storedWorkVersions != nil {
+ return nil
+ }
+ if h.bqClient == nil {
+ return nil
+ }
+ var err error
+ h.storedWorkVersions, err = bigquery.ReadVulncheckWorkVersions(ctx, h.bqClient)
+ return err
+}
+
+// A scanner holds state for scanning modules.
+type scanner struct {
+ proxyClient *proxy.Client
+ dbClient vulnc.Client
+ bqClient *bigquery.Client
+ workVersion *bigquery.VulncheckWorkVersion
+ goMemLimit uint64
+ gcsBucket *storage.BucketHandle
+ insecure bool
+ sbox *sandbox.Sandbox
+}
+
+func newScanner(ctx context.Context, h *VulncheckServer) (*scanner, error) {
+ workVersion, err := h.getWorkVersion(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var bucket *storage.BucketHandle
+ if h.cfg.BinaryBucket != "" {
+ c, err := storage.NewClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ bucket = c.Bucket(h.cfg.BinaryBucket)
+ }
+ sbox := sandbox.New("/bundle")
+ sbox.Runsc = "/usr/local/bin/runsc"
+ return &scanner{
+ proxyClient: h.proxyClient,
+ bqClient: h.bqClient,
+ dbClient: h.vulndbClient,
+ workVersion: workVersion,
+ goMemLimit: parseGoMemLimit(os.Getenv("GOMEMLIMIT")),
+ gcsBucket: bucket,
+ insecure: h.cfg.Insecure,
+ sbox: sbox,
+ }, nil
+}
+
+type scanError struct {
+ err error
+}
+
+func (s scanError) Error() string {
+ return s.err.Error()
+}
+
+func (s scanError) Unwrap() error {
+ return s.err
+}
+
+func (s *scanner) ScanModule(ctx context.Context, sreq *scan.Request) {
+ if sreq.Module == "std" {
+ return // ignore the standard library
+ }
+ row := &bigquery.VulnResult{
+ ModulePath: sreq.Module,
+ Suffix: sreq.Suffix,
+ VulncheckWorkVersion: *s.workVersion,
+ }
+ // Scan the version.
+ log.Infof(ctx, "fetching proxy info: %s@%s", sreq.Module, sreq.Version)
+ info, err := s.proxyClient.Info(ctx, sreq.Module, sreq.Version)
+ if err != nil {
+ log.Errorf(ctx, "proxy error: %v", err)
+ row.AddError(fmt.Errorf("%v: %w", err, derrors.ProxyError))
+ return
+ }
+ row.Version = info.Version
+ row.SortVersion = version.ForSorting(row.Version)
+ row.CommitTime = info.Time
+ row.ImportedBy = sreq.ImportedBy
+ row.VulnDBLastModified = s.workVersion.VulnDBLastModified
+ row.ScanMode = sreq.Mode
+
+ log.Infof(ctx, "scanning: %s", sreq.Path())
+ stats := &vulncheckStats{}
+ vulns, err := s.runScanModule(ctx, sreq.Module, info.Version, sreq.Suffix, sreq.Mode, stats)
+ row.ScanSeconds = stats.scanSeconds
+ row.ScanMemory = int64(stats.scanMemory)
+ row.PkgsMemory = int64(stats.pkgsMemory)
+ row.Workers = config.GetEnvInt("CLOUD_RUN_CONCURRENCY", "0", -1)
+ if err != nil {
+ row.AddError(err)
+ log.Infof(ctx, "scanner.runScanModule return error for %s (%v)", sreq.Path(), err)
+ } else {
+ row.Vulns = vulns
+ log.Infof(ctx, "scanner.runScanModule returned %d vulns: %s", len(vulns), sreq.Path())
+ }
+ if s.bqClient == nil {
+ log.Infof(ctx, "bigquery disabled, not uploading")
+ } else {
+ log.Infof(ctx, "uploading to bigquery: %s", sreq.Path())
+ if err := s.bqClient.Upload(ctx, bigquery.VulncheckTableName, row); err != nil {
+ // This is often caused by:
+ // "Upload: googleapi: got HTTP response code 413 with body"
+ // which happens for some modules.
+ row.AddError(fmt.Errorf("%v: %w", err, derrors.BigQueryError))
+ log.Errorf(ctx, "bq.Upload for %s: %v", sreq.Path(), err)
+ }
+ }
+}
+
+type vulncheckStats struct {
+ scanSeconds float64
+ scanMemory uint64
+ pkgsMemory uint64
+}
+
+var activeScans atomic.Int32
+
+// runScanModule fetches the module version from the proxy, and analyzes it for
+// vulnerabilities.
+func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) (bvulns []*bigquery.Vuln, err error) {
+ defer func() {
+ if e := recover(); e != nil {
+ err = fmt.Errorf("%w: %v\n\n%s", derrors.ScanModulePanicError, e, debug.Stack())
+ }
+ }()
+
+ logMemory(ctx, fmt.Sprintf("before scanning %s@%s", modulePath, version))
+ defer logMemory(ctx, fmt.Sprintf("after scanning %s@%s", modulePath, version))
+
+ activeScans.Add(1)
+ defer func() {
+ if activeScans.Add(-1) == 0 {
+ logMemory(ctx, fmt.Sprintf("before 'go clean' for %s@%s", modulePath, version))
+ s.cleanGoCaches(ctx)
+ }
+ }()
+
+ var vulns []*vulncheck.Vuln
+ if s.insecure {
+ vulns, err = s.runScanModuleInsecure(ctx, modulePath, version, binaryDir, mode, stats)
+ } else {
+ vulns, err = s.runScanModuleSandbox(ctx, modulePath, version, binaryDir, mode, stats)
+ }
+
+ // If an error occurred, wrap it accordingly.
+ if err != nil {
+ if isVulnDBConnection(err) {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleVulncheckDBConnectionError)
+ } else if !errors.Is(err, derrors.ScanModuleMemoryLimitExceeded) {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleVulncheckError)
+ }
+ return nil, err
+ }
+ for _, v := range vulns {
+ bvulns = append(bvulns, convertVuln(v))
+ }
+ return bvulns, nil
+}
+
+func (s *scanner) runScanModuleSandbox(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) ([]*vulncheck.Vuln, error) {
+ var (
+ res *vulncheck.Result
+ err error
+ )
+ if mode == ModeBinary {
+ res, err = s.runBinaryScanSandbox(ctx, modulePath, version, binaryDir, stats)
+ } else {
+ res, err = s.runSourceScanSandbox(ctx, modulePath, version, mode, stats)
+ }
+ log.Debugf(ctx, "runScanModuleSandbox %s@%s bin %s, mode %s: got %+v, %v", modulePath, version, binaryDir, mode, res, err)
+ if err != nil {
+ return nil, err
+ }
+ return res.Vulns, nil
+}
+
+func (s *scanner) runSourceScanSandbox(ctx context.Context, modulePath, version, mode string, stats *vulncheckStats) (*vulncheck.Result, error) {
+ stdout, err := runSourceScanSandbox(ctx, modulePath, version, mode, s.proxyClient, s.sbox)
+ if err != nil {
+ return nil, err
+ }
+ return unmarshalVulncheckOutput(stdout)
+}
+
+const sandboxGoModCache = "go/pkg/mod"
+
+func runSourceScanSandbox(ctx context.Context, modulePath, version, mode string, proxyClient *proxy.Client, sbox *sandbox.Sandbox) ([]byte, error) {
+ sandboxDir := "/modules/" + modulePath + "@" + version
+ imageDir := "/bundle/rootfs" + sandboxDir
+ defer os.RemoveAll(imageDir)
+ log.Infof(ctx, "downloading %s@%s to %s", modulePath, version, imageDir)
+ if err := modules.Download(ctx, modulePath, version, imageDir, proxyClient, true); err != nil {
+ log.Debugf(ctx, "download error: %v (%[1]T)", err)
+ return nil, err
+ }
+ // Download all dependencies outside of the sandbox, but use the Go build
+ // cache inside the bundle.
+ log.Infof(ctx, "running go mod download")
+ cmd := exec.Command("go", "mod", "download")
+ cmd.Dir = imageDir
+ cmd.Env = append(cmd.Environ(),
+ "GOPROXY=https://proxy.golang.org",
+ "GOMODCACHE=/bundle/rootfs/"+sandboxGoModCache)
+ _, err := cmd.Output()
+ if err != nil {
+ return nil, fmt.Errorf("%w: 'go mod download' for %s@%s returned %s",
+ derrors.BadModule, modulePath, version, log.IncludeStderr(err))
+ }
+ log.Infof(ctx, "go mod download succeeded")
+ log.Infof(ctx, "%s@%s: running vulncheck in sandbox", modulePath, version)
+ stdout, err := sbox.Run(ctx, "/binaries/vulncheck_sandbox", "-gomodcache", "/"+sandboxGoModCache, mode, sandboxDir)
+ if err != nil {
+ return nil, errors.New(log.IncludeStderr(err))
+ }
+ return stdout, nil
+}
+
+func (s *scanner) runBinaryScanSandbox(ctx context.Context, modulePath, version, binDir string, stats *vulncheckStats) (*vulncheck.Result, error) {
+ if s.gcsBucket == nil {
+ return nil, errors.New("binary bucket not configured; set GO_ECOSYSTEM_BINARY_BUCKET")
+ }
+ // Copy the binary from GCS to the local disk, because vulncheck.Binary
+ // requires a ReaderAt and GCS doesn't provide that.
+ gcsPathname := fmt.Sprintf("%s/%s@%s/%s", binaryDir, modulePath, version, binDir)
+ const destDir = "/bundle/rootfs/binaries"
+ log.With("module", modulePath, "version", version, "dir", binDir).
+ Debugf(ctx, "copying %s to %s", gcsPathname, destDir)
+ destf, err := os.CreateTemp(destDir, "vulncheck-binary-")
+ if err != nil {
+ return nil, err
+ }
+ defer os.Remove(destf.Name())
+ if err := copyFromGCSToWriter(ctx, destf, s.gcsBucket, gcsPathname); err != nil {
+ return nil, err
+ }
+ log.Infof(ctx, "%s@%s/%s: running vulncheck in sandbox on %s", modulePath, version, binDir, destf.Name())
+ stdout, err := s.sbox.Run(ctx, "/binaries/vulncheck_sandbox", ModeBinary, destf.Name())
+ if err != nil {
+ return nil, errors.New(log.IncludeStderr(err))
+ }
+ return unmarshalVulncheckOutput(stdout)
+}
+
+func unmarshalVulncheckOutput(output []byte) (*vulncheck.Result, error) {
+ var e struct {
+ Error string
+ }
+ if err := json.Unmarshal(output, &e); err != nil {
+ return nil, err
+ }
+ if e.Error != "" {
+ return nil, errors.New(e.Error)
+ }
+ var res vulncheck.Result
+ if err := json.Unmarshal(output, &res); err != nil {
+ return nil, err
+ }
+ return &res, nil
+}
+
+func (s *scanner) runScanModuleInsecure(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) (_ []*vulncheck.Vuln, err error) {
+ tempDir, err := os.MkdirTemp("", "runScanModule")
+ if err != nil {
+ return nil, err
+ }
+
+ defer func() {
+ err1 := os.RemoveAll(tempDir)
+ if err == nil {
+ err = err1
+ }
+ }()
+
+ if mode == ModeBinary {
+ return s.runBinaryScanInsecure(ctx, modulePath, version, binaryDir, tempDir, stats)
+ }
+ return s.runSourceScanInsecure(ctx, modulePath, version, mode, tempDir, stats)
+}
+
+func (s *scanner) runSourceScanInsecure(ctx context.Context, modulePath, version, mode, tempDir string, stats *vulncheckStats) ([]*vulncheck.Vuln, error) {
+ log.Debugf(ctx, "fetching module zip: %s@%s", modulePath, version)
+ if err := modules.Download(ctx, modulePath, version, tempDir, s.proxyClient, true); err != nil {
+ return nil, err
+ }
+
+ cctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ cfg := load.DefaultConfig()
+ cfg.Dir = tempDir // filepath.Join(dir, modulePath+"@"+version,
+ cfg.Context = cctx
+
+ runtime.GC()
+ // current memory not related to core (go)vulncheck operations.
+ preScanMemory := currHeapUsage()
+
+ log.Debugf(ctx, "loading packages: %s@%s", modulePath, version)
+ pkgs, pkgErrors, err := load.Packages(cfg, "./...")
+ if err == nil && len(pkgErrors) > 0 {
+ err = fmt.Errorf("%v", pkgErrors)
+ }
+ if err != nil {
+ if !fileExists(filepath.Join(tempDir, "go.mod")) {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleLoadPackagesNoGoModError)
+ } else if !fileExists(filepath.Join(tempDir, "go.sum")) {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleLoadPackagesNoGoSumError)
+ } else if isNoRequiredModule(err) {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleLoadPackagesNoRequiredModuleError)
+ } else if isMissingGoSumEntry(err.Error()) {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleLoadPackagesMissingGoSumEntryError)
+ } else {
+ err = fmt.Errorf("%v: %w", err, derrors.ScanModuleLoadPackagesError)
+ }
+ return nil, err
+ }
+
+ stats.pkgsMemory = memSubtract(currHeapUsage(), preScanMemory)
+
+ // Run vulncheck.Source and collect results.
+ start := time.Now()
+ vcfg := vulncheckConfig(s.dbClient, mode)
+ res, peakMem, err := s.runWithMemoryMonitor(ctx, func() (*vulncheck.Result, error) {
+ log.Debugf(ctx, "running vulncheck.Source: %s@%s", modulePath, version)
+ res, err := vulncheck.Source(cctx, vulncheck.Convert(pkgs), vcfg)
+ log.Debugf(ctx, "completed run for vulncheck.Source: %s@%s, err=%v", modulePath, version, err)
+
+ if err != nil {
+ return res, err
+ }
+ if mode == ModeVTAStacks {
+ log.Debugf(ctx, "running vulncheck.CallStacks: %s@%s", modulePath, version)
+ vulncheck.CallStacks(res)
+ log.Debugf(ctx, "completed run for vulncheck.CallStacks: %s@%s, err=%v", modulePath, version, err)
+ }
+ return res, nil
+ })
+ // scanMemory is peak heap memory used during vulncheck + pkgs.
+ // We subtract any memory not related to these core (go)vulncheck
+ // operations.
+ stats.scanMemory = memSubtract(peakMem, preScanMemory)
+
+ // scanSeconds is the time it took for vulncheck.Source to run.
+ // We want to know this information regardless of whether an error
+ // occurred.
+ stats.scanSeconds = time.Since(start).Seconds()
+ if err != nil {
+ return nil, err
+ }
+ return res.Vulns, nil
+}
+
+// runWithMemoryMonitor runs f in a goroutine with its memory tracked.
+// It returns f's peak memory usage.
+func (s *scanner) runWithMemoryMonitor(ctx context.Context, f func() (*vulncheck.Result, error)) (res *vulncheck.Result, mem uint64, err error) {
+ cctx, cancel := context.WithCancel(ctx)
+ monitor := newMemMonitor(s.goMemLimit, cancel)
+ type sr struct {
+ res *vulncheck.Result
+ err error
+ }
+ srchan := make(chan sr)
+ go func() {
+ res, err := f()
+ srchan <- sr{res, err}
+ }()
+ select {
+ case r := <-srchan:
+ res = r.res
+ err = r.err
+ case <-cctx.Done():
+ err = derrors.ScanModuleMemoryLimitExceeded
+ }
+ return res, monitor.stop(), err
+}
+
+func (s *scanner) runBinaryScanInsecure(ctx context.Context, modulePath, version, binDir, tempDir string, stats *vulncheckStats) ([]*vulncheck.Vuln, error) {
+ if s.gcsBucket == nil {
+ return nil, errors.New("binary bucket not configured; set GO_ECOSYSTEM_BINARY_BUCKET")
+ }
+ // Copy the binary from GCS to the local disk, because vulncheck.Binary
+ // requires a ReaderAt and GCS doesn't provide that.
+ gcsPathname := fmt.Sprintf("%s/%s@%s/%s", binaryDir, modulePath, version, binDir)
+ log.With("module", modulePath, "version", version, "dir", binDir).
+ Debugf(ctx, "copying %s to temp dir", gcsPathname)
+ localPathname := filepath.Join(tempDir, "binary")
+ if err := copyFromGCS(ctx, s.gcsBucket, gcsPathname, localPathname); err != nil {
+ return nil, err
+ }
+
+ binaryFile, err := os.Open(localPathname)
+ if err != nil {
+ return nil, err
+ }
+ defer binaryFile.Close()
+
+ start := time.Now()
+ runtime.GC()
+ // current memory not related to core (go)vulncheck operations.
+ preScanMemory := currHeapUsage()
+ log.Debugf(ctx, "running vulncheck.Binary: %s", gcsPathname)
+ res, err := vulncheck.Binary(ctx, binaryFile, vulncheckConfig(s.dbClient, ModeBinary))
+ log.Debugf(ctx, "completed run for vulncheck.Binary: %s, err=%v", gcsPathname, err)
+ stats.scanSeconds = time.Since(start).Seconds()
+ // TODO: measure peak usage?
+ stats.scanMemory = memSubtract(currHeapUsage(), preScanMemory)
+ if err != nil {
+ return nil, err
+ }
+ return res.Vulns, nil
+}
+
+func copyFromGCS(ctx context.Context, bucket *storage.BucketHandle, srcPath, destPath string) (err error) {
+ defer derrors.Wrap(&err, "copyFromGCS(%q, %q)", srcPath, destPath)
+ destf, err := os.Create(destPath)
+ if err != nil {
+ return err
+ }
+ err1 := copyFromGCSToWriter(ctx, destf, bucket, srcPath)
+ err2 := destf.Close()
+ if err1 != nil {
+ return err1
+ }
+ return err2
+}
+
+func copyFromGCSToWriter(ctx context.Context, w io.Writer, bucket *storage.BucketHandle, srcPath string) error {
+ gcsReader, err := bucket.Object(srcPath).NewReader(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(w, gcsReader)
+ return err
+}
+
+// fileExists checks if file path exists. Returns true
+// if the file exists or it cannot prove that it does
+// not exist. Otherwise, returns false.
+func fileExists(file string) bool {
+ if _, err := os.Stat(file); err == nil {
+ return true
+ } else if errors.Is(err, os.ErrNotExist) {
+ return false
+ }
+ // Conservatively return true if os.Stat fails
+ // for some other reason.
+ return true
+}
+
+func isVulnDBConnection(err error) bool {
+ s := err.Error()
+ return strings.Contains(s, "https://vuln.go.dev") &&
+ strings.Contains(s, "connection")
+}
+
+func isNoRequiredModule(err error) bool {
+ return strings.Contains(err.Error(), "no required module")
+}
+
+func isMissingGoSumEntry(errMsg string) bool {
+ return strings.Contains(errMsg, "missing go.sum entry")
+}
+
+func vulncheckConfig(dbClient vulnc.Client, mode string) *vulncheck.Config {
+ cfg := &vulncheck.Config{Client: dbClient}
+ switch mode {
+ case ModeImports:
+ cfg.ImportsOnly = true
+ default:
+ cfg.ImportsOnly = false
+ }
+ return cfg
+}
+
+func convertVuln(v *vulncheck.Vuln) *bigquery.Vuln {
+ return &bigquery.Vuln{
+ ID: v.OSV.ID,
+ ModulePath: v.ModPath,
+ PackagePath: v.PkgPath,
+ Symbol: v.Symbol,
+ CallSink: bigquery.NullInt(v.CallSink),
+ ImportSink: bigquery.NullInt(v.ImportSink),
+ RequireSink: bigquery.NullInt(v.RequireSink),
+ }
+}
+
+// currHeapUsage computes currently allocate heap bytes.
+func currHeapUsage() uint64 {
+ var stats runtime.MemStats
+ runtime.ReadMemStats(&stats)
+ return stats.Alloc
+}
+
+// memSubtract subtracts memory usage m2 from m1, returning
+// 0 if the result is negative.
+func memSubtract(m1, m2 uint64) uint64 {
+ if m1 <= m2 {
+ return 0
+ }
+ return m1 - m2
+}
+
+// parseGoMemLimit parses the GOMEMLIMIT environment variable.
+// It returns 0 if the variable isn't set or its value is malformed.
+func parseGoMemLimit(s string) uint64 {
+ if len(s) < 2 {
+ return 0
+ }
+ m := uint64(1)
+ if s[len(s)-1] == 'i' {
+ switch s[len(s)-2] {
+ case 'K':
+ m = 1024
+ case 'M':
+ m = 1024 * 1024
+ case 'G':
+ m = 1024 * 1024 * 1024
+ default:
+ return 0
+ }
+ s = s[:len(s)-2]
+ }
+ v, err := strconv.ParseUint(s, 10, 64)
+ if err != nil {
+ return 0
+ }
+ return v * m
+}
+
+func logMemory(ctx context.Context, prefix string) {
+ if !config.OnCloudRun() {
+ return
+ }
+
+ readIntFile := func(filename string) (int, error) {
+ data, err := os.ReadFile(filename)
+ if err != nil {
+ return 0, err
+ }
+ return strconv.Atoi(strings.TrimSpace(string(data)))
+ }
+
+ const (
+ curFilename = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
+ maxFilename = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
+ )
+
+ cur, err := readIntFile(curFilename)
+ if err != nil {
+ log.Errorf(ctx, "reading %s: %v", curFilename, err)
+ }
+ max, err := readIntFile(maxFilename)
+ if err != nil {
+ log.Errorf(ctx, "reading %s: %v", maxFilename, err)
+ }
+
+ const G float64 = 1024 * 1024 * 1024
+
+ log.Infof(ctx, "%s: using %.1fG out of %.1fG", prefix, float64(cur)/G, float64(max)/G)
+}
+
+func (s *scanner) cleanGoCaches(ctx context.Context) error {
+ if !config.OnCloudRun() {
+ log.Infof(ctx, "not on Cloud Run, so not cleaning caches")
+ return nil
+ }
+ var (
+ out []byte
+ err error
+ )
+ if s.insecure {
+ out, err = exec.Command("go", "clean", "-cache", "-modcache").CombinedOutput()
+ } else {
+ out, err = s.sbox.Run(ctx, "/binaries/vulncheck_sandbox", "-gomodcache", "/"+sandboxGoModCache, "-clean")
+ }
+ if err != nil {
+ return fmt.Errorf("cleaning Go caches: %s", log.IncludeStderr(err))
+ }
+ output := ""
+ if len(out) > 0 {
+ output = fmt.Sprintf(" with output %s", out)
+ }
+ log.Infof(ctx, "'go clean' succeeded%s", output)
+ return nil
+}
+
+// Test running vulncheck in the sandbox.
+// This runs a scan but returns the resulting JSON instead of writing it to BigQuery.
+func (s *Server) handleTestVulncheckSandbox(w http.ResponseWriter, r *http.Request) error {
+ ctx := r.Context()
+ sreq, err := scan.ParseRequest(r, "/test-vulncheck-sandbox")
+ if err != nil {
+ return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
+ }
+ sbox := sandbox.New("/bundle")
+ sbox.Runsc = "/usr/local/bin/runsc"
+ out, err := runSourceScanSandbox(ctx, sreq.Module, sreq.Version, ModeVTA, s.proxyClient, sbox)
+ if err != nil {
+ return err
+ }
+ _, err = w.Write(out)
+ return err
+}
diff --git a/internal/worker/vulncheck_scan_test.go b/internal/worker/vulncheck_scan_test.go
new file mode 100644
index 0000000..686b9f1
--- /dev/null
+++ b/internal/worker/vulncheck_scan_test.go
@@ -0,0 +1,146 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "io"
+ "testing"
+
+ "cloud.google.com/go/storage"
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite-metrics/internal/config"
+ "golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/proxy"
+ vulnc "golang.org/x/vuln/client"
+ "golang.org/x/vuln/vulncheck"
+)
+
+var integration = flag.Bool("integration", false, "test against actual service")
+
+func TestAsScanError(t *testing.T) {
+ check := func(err error, want bool) {
+ if got := errors.As(err, new(scanError)); got != want {
+ t.Errorf("%T: got %t, want %t", err, got, want)
+ }
+ }
+ check(io.EOF, false)
+ check(scanError{io.EOF}, true)
+}
+
+func TestRunScanModule(t *testing.T) {
+ t.Skip()
+
+ ctx := context.Background()
+ cfg, err := config.Init(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ dbClient, err := vulnc.NewClient([]string{cfg.VulnDBURL}, vulnc.Options{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ proxyClient, err := proxy.New(cfg.ProxyURL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Run("source", func(t *testing.T) {
+ s := &scanner{proxyClient: proxyClient, dbClient: dbClient, insecure: true}
+ stats := &vulncheckStats{}
+ vulns, err := s.runScanModule(ctx,
+ "golang.org/x/exp/event", "v0.0.0-20220929112958-4a82f8963a65",
+ "", ModeVTA, stats)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantID := "GO-2022-0493"
+ found := false
+ for _, v := range vulns {
+ if v.ID == wantID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("want %s, did not find it in %d vulns", wantID, len(vulns))
+ }
+ if got := stats.scanMemory; got <= 0 {
+ t.Errorf("scan memory not collected or negative: %v", got)
+ }
+ if got := stats.pkgsMemory; got <= 0 {
+ t.Errorf("pkgs memory not collected or negative: %v", got)
+ }
+ })
+ t.Run("memoryLimit", func(t *testing.T) {
+ s := &scanner{proxyClient: proxyClient, dbClient: dbClient, insecure: true, goMemLimit: 2000}
+ _, err := s.runScanModule(ctx, "golang.org/x/mod", "v0.5.1",
+ "", ModeVTA, &vulncheckStats{})
+ if !errors.Is(err, derrors.ScanModuleMemoryLimitExceeded) {
+ t.Errorf("got %v, want MemoryLimitExceeded", err)
+ }
+ })
+ t.Run("binary", func(t *testing.T) {
+ if !*integration { // needs GCS read permission, not available on kokoro
+ t.Skip("missing -integration")
+ }
+ s := &scanner{proxyClient: proxyClient, dbClient: dbClient}
+ gcsClient, err := storage.NewClient(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.gcsBucket = gcsClient.Bucket("go-ecosystem")
+ stats := &vulncheckStats{}
+ vulns, err := s.runScanModule(ctx, "golang.org/x/pkgsite", "v0.0.0-20221004150836-873fb37c2479", "cmd/worker", ModeBinary, stats)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if g, w := len(vulns), 14; g != w {
+ t.Errorf("got %d vulns, want %d", g, w)
+ }
+ })
+}
+
+func TestParseGoMemLimit(t *testing.T) {
+ for _, test := range []struct {
+ in string
+ want uint64
+ }{
+ {"", 0},
+ {"foo", 0},
+ {"23", 23},
+ {"56Ki", 56 * 1024},
+ {"3Mi", 3 * 1024 * 1024},
+ {"8Gi", 8 * 1024 * 1024 * 1024},
+ } {
+ got := parseGoMemLimit(test.in)
+ if got != test.want {
+ t.Errorf("%q: got %d, want %d", test.in, got, test.want)
+ }
+ }
+}
+
+func TestUnmarshalVulncheckOutput(t *testing.T) {
+ _, err := unmarshalVulncheckOutput([]byte(`{"Error": "bad"}`))
+ if got, want := err.Error(), "bad"; got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+ want := &vulncheck.Result{
+ Modules: []*vulncheck.Module{{Path: "m", Version: "v1.2.3"}},
+ }
+ in, err := json.Marshal(want)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, err := unmarshalVulncheckOutput(in)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !cmp.Equal(got, want) {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+}
diff --git a/internal/worker/vulncheck_test.go b/internal/worker/vulncheck_test.go
new file mode 100644
index 0000000..8159c12
--- /dev/null
+++ b/internal/worker/vulncheck_test.go
@@ -0,0 +1,41 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite-metrics/internal/bigquery"
+)
+
+func TestVulnsScanned(t *testing.T) {
+ p := newPage("test")
+
+ vuln1A := &bigquery.Vuln{ID: "1", Symbol: "A", CallSink: bigquery.NullInt(100)}
+ vuln1B := &bigquery.Vuln{ID: "1", Symbol: "B", CallSink: bigquery.NullInt(101)}
+ vuln1C := &bigquery.Vuln{ID: "1", Symbol: "C"}
+ vuln2A := &bigquery.Vuln{ID: "2", Symbol: "A"}
+
+ rows := []*bigquery.VulnResult{
+ {ModulePath: "m1", ScanMode: ModeImports, Vulns: []*bigquery.Vuln{vuln1A, vuln1B, vuln1C, vuln2A}},
+ {ModulePath: "m1", ScanMode: ModeVTA, Vulns: []*bigquery.Vuln{vuln1A, vuln1B}},
+ {ModulePath: "m2", ScanMode: ModeImports, Vulns: []*bigquery.Vuln{vuln2A}},
+ {ModulePath: "m2", ScanMode: ModeVTA, Vulns: []*bigquery.Vuln{}},
+ }
+
+ got := handleVulncheckRows(context.Background(), p, rows)
+ // Vuln 1 is detected in m1 module in both IMPORTS and VTA modes.
+ // Vuln 2 is detected in m1 and m2 in IMPORTS mode, but nowhere
+ // in VTA mode.
+ want := map[string]*ReportResult{
+ "1": &ReportResult{ImportsNumModules: 1, VTANumModules: 1},
+ "2": &ReportResult{ImportsNumModules: 2, VTANumModules: 0},
+ }
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("mismatch (-want, +got):\n%s", diff)
+ }
+}
diff --git a/static/vulncheck.tmpl b/static/vulncheck.tmpl
new file mode 100644
index 0000000..f023c2b
--- /dev/null
+++ b/static/vulncheck.tmpl
@@ -0,0 +1,159 @@
+<!--
+ Copyright 2022 The Go Authors. All rights reserved.
+ Use of this source code is governed by a BSD-style
+ license that can be found in the LICENSE file.
+-->
+
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="utf-8">
+<link href="/static/static.css" rel="stylesheet">
+<title>Go Vulnerability Management - Metrics</title>
+
+<body>
+<main class="go-Container">
+<div class="go-Content">
+ <h1>Go Vulnerability Management - Metrics</h1>
+ <div class="Container">
+ <p>Results are from {{.TableName}}</p>
+ {{template "vulncheck" .}}
+ </div>
+</div>
+</main>
+</body>
+</html>
+
+{{define "vulncheck"}}
+{{/* . is VulncheckPage */}}
+<div>
+ {{template "corpus" .}}
+ {{template "errors" .}}
+ {{template "vulnerabilities" .}}
+</div>
+{{end}}
+
+{{define "corpus"}}
+{{/* . is VulncheckPage */}}
+<div>
+ <h2>Scan Results</h2>
+ <table>
+ <tbody>
+ <tr>
+ <th>Mode</th>
+ <th>Imports Only</th>
+ <th>VTA</th>
+ <th>VTA - Stacks</th>
+ </tr>
+ <tr>
+ <td># Modules Scanned</td>
+ <td>{{.ImportsResult.NumModulesScanned}}</td>
+ <td>{{.VTAResult.NumModulesScanned}}</td>
+ <td>{{.VTAStacksResult.NumModulesScanned}}</td>
+ </tr>
+ <tr>
+ <td># Modules Success</td>
+ <td>{{.ImportsResult.NumModulesSuccess}}</td>
+ <td>{{.VTAResult.NumModulesSuccess}}</td>
+ <td>{{.VTAStacksResult.NumModulesSuccess}}</td>
+ </tr>
+ <tr>
+ <td>% Modules Success</td>
+ <td>{{round .ImportsResult.PercentSuccess}}%</td>
+ <td>{{round .VTAResult.PercentSuccess}}%</td>
+ <td>{{round .VTAStacksResult.PercentSuccess}}%</td>
+ </tr>
+ <tr>
+ <td># Modules Failed</td>
+ <td>{{.ImportsResult.NumModulesError}}</td>
+ <td>{{.VTAResult.NumModulesError}}</td>
+ <td>{{.VTAStacksResult.NumModulesError}}</td>
+ </tr>
+ <tr>
+ <td>Avg Scan Seconds</td>
+ <td>{{round .ImportsResult.AverageScanSeconds}}s</td>
+ <td>{{round .VTAResult.AverageScanSeconds}}s</td>
+ <td>{{round .VTAStacksResult.AverageScanSeconds}}s</td>
+ </tr>
+ <tr>
+ <td>Max Scan Seconds</td>
+ <td>{{round .ImportsResult.MaxScanSeconds}}s</td>
+ <td>{{round .VTAResult.MaxScanSeconds}}s</td>
+ <td>{{round .VTAStacksResult.MaxScanSeconds}}s</td>
+ </tr>
+ <tr>
+ <td>Avg Scan Memory (MB)</td>
+ <td>{{round .ImportsResult.AverageScanMemory}}MB</td>
+ <td>{{round .VTAResult.AverageScanMemory}}MB</td>
+ <td>{{round .VTAStacksResult.AverageScanMemory}}MB</td>
+ </tr>
+ <tr>
+ <td>Max Scan Memory (MB)</td>
+ <td>{{round .ImportsResult.MaxScanMemory}}MB</td>
+ <td>{{round .VTAResult.MaxScanMemory}}MB</td>
+ <td>{{round .VTAStacksResult.MaxScanMemory}}MB</td>
+ </tr>
+ </tbody>
+ </table>
+</div>
+{{end}}
+
+{{define "vulnerabilities"}}
+{{/* . is VulncheckPage */}}
+<div>
+ <h2>Vulnerabilities</h2>
+ <p><strong>VTA: {{.VTAResult.NumModulesSuccess}}</strong> scans succeeded (<strong>{{round .VTAResult.PercentSuccess}}%</strong>)</p>
+ <p><strong>Imports: {{.ImportsResult.NumModulesSuccess}}</strong> scans succeeded (<strong>{{round .ImportsResult.PercentSuccess}}%</strong>)</p>
+ <div>
+ <table>
+ <tbody>
+ <tr>
+ <th>Category</th>
+ <th>VTA</br># modules calls</th>
+ <th>Imports Only</br># modules imports</th>
+ <th>VTA</br>% of success</th>
+ <th>Imports Only</br>% of success</th>
+ </tr>
+ <tr>
+ <td>0 vulns</td>
+ <td>{{.VTAResult.NumModulesNoVuln}}</td>
+ <td>{{.ImportsResult.NumModulesNoVuln}}</td>
+ <td>{{round .VTAResult.PercentNoVuln}}%</td>
+ <td>{{round .ImportsResult.PercentNoVuln}}%</td>
+ </tr>
+ <tr>
+ <td>1+ vulns</td>
+ <td>{{.VTAResult.NumModulesVuln}}</td>
+ <td>{{.ImportsResult.NumModulesVuln}}</td>
+ <td>{{round .VTAResult.PercentVuln}}%</td>
+ <td>{{round .ImportsResult.PercentVuln}}%</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+</div>
+{{end}}
+
+{{define "errors"}}
+{{/* . is VulncheckPage */}}
+<div>
+ <h2>Errors</h2>
+ <p><strong>VTA: {{.VTAResult.NumModulesError}}</strong> scans failed (<strong>{{round .VTAResult.PercentFailed}}%</strong>)</p>
+ <p><strong>Imports: {{.ImportsResult.NumModulesError}}</strong> scans failed (<strong>{{round .ImportsResult.PercentFailed}}%</strong>)</p>
+ <table>
+ <tbody>
+ <tr>
+ <th>Category</th>
+ <th>VTA</br># Modules</th>
+ <th>Imports Only</br># Modules</th>
+ </tr>
+ {{range .Errors}}
+ <tr>
+ <td>{{.Name}}</td>
+ <td>{{.VTANumModules}}</td>
+ <td>{{.ImportsNumModules}}</td>
+ </tr>
+ {{end}}
+ </tbody>
+ </table>
+</div>
+{{end}}
