{cmd,internal}/worker: scan modules
Complete support for scanning modules.
- Add scan-modules command to cmd/worker.
- Remember modules we scanned in the Store.
- Track the last-modified vuln DB time to avoid unnecessary re-scanning.
Change-Id: Id2b6d3b2d91c6617d31f4fe6997babba2db220bd
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/393695
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Damien Neil <dneil@google.com>
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index d31d9cb..198ff38 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -69,6 +69,7 @@
fmt.Fprintln(out, " list-cves TRIAGE_STATE: display info about CVE records")
fmt.Fprintln(out, " create-issues: create issues for CVEs that need them")
fmt.Fprintln(out, " show ID1 ID2 ...: display CVE records")
+ fmt.Fprintln(out, " scan-modules: scan modules for vulnerabilities")
fmt.Fprintln(out, "flags:")
flag.PrintDefaults()
}
@@ -138,6 +139,8 @@
return createIssuesCommand(ctx)
case "show":
return showCommand(ctx, flag.Args()[1:])
+ case "scan-modules":
+ return scanModulesCommand(ctx)
default:
return fmt.Errorf("unknown command: %q", flag.Arg(1))
}
@@ -277,6 +280,10 @@
return nil
}
+func scanModulesCommand(ctx context.Context) error {
+ return worker.ScanModules(ctx, cfg.Store)
+}
+
func die(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, format, args...)
fmt.Fprintln(os.Stderr)
diff --git a/go.mod b/go.mod
index 8e3d688..40c1311 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@
require (
cloud.google.com/go/errorreporting v0.1.0
cloud.google.com/go/firestore v1.6.1
+ cloud.google.com/go/storage v1.10.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go v1.0.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.26.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.0.0
@@ -32,7 +33,7 @@
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
golang.org/x/tools v0.1.8
- google.golang.org/api v0.68.0
+ google.golang.org/api v0.70.0
google.golang.org/grpc v1.44.0
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
honnef.co/go/tools v0.2.2
@@ -41,7 +42,8 @@
require (
cloud.google.com/go v0.100.2 // indirect
- cloud.google.com/go/compute v1.2.0 // indirect
+ cloud.google.com/go/compute v1.3.0 // indirect
+ cloud.google.com/go/iam v0.3.0 // indirect
cloud.google.com/go/monitoring v1.2.0 // indirect
cloud.google.com/go/trace v1.0.0 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
@@ -69,10 +71,10 @@
go.opentelemetry.io/otel/sdk/metric v0.26.0 // indirect
go.opentelemetry.io/otel/trace v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
- golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
+ golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20220207185906-7721543eae58 // indirect
+ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)
diff --git a/go.sum b/go.sum
index 386b967..5ef9680 100644
--- a/go.sum
+++ b/go.sum
@@ -37,14 +37,17 @@
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
-cloud.google.com/go/compute v1.2.0 h1:EKki8sSdvDU0OO9mAXGwPXOTOgPz2l08R0/IutDH11I=
cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw=
+cloud.google.com/go/compute v1.3.0 h1:mPL/MzDDYHsh5tHRS9mhmhWlcgClCrCa6ApQCU6wnHI=
+cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/errorreporting v0.1.0 h1:z40EhrjRspplwbpO+9DSnC4kgDokBi94T/gYwtdKL5Q=
cloud.google.com/go/errorreporting v0.1.0/go.mod h1:cZSiBMvrnl0X13pD9DwKf9sQ8Eqy3EzHqkyKBZxiIrM=
cloud.google.com/go/firestore v1.6.1 h1:8rBq3zRjnHx8UtBvaOWqBB1xq9jH6/wltfQLlTMh2Fw=
cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
+cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc=
+cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
cloud.google.com/go/monitoring v1.2.0 h1:fEvQITrhVcPM6vuDQcgPMbU5kZFeQFwZmE7v6+S8BPo=
cloud.google.com/go/monitoring v1.2.0/go.mod h1:tE8I08OzjWmXLhCopnPaUDpfGOEJOonfWXGR9E9SsFo=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@@ -55,6 +58,7 @@
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/trace v1.0.0 h1:laKx2y7IWMjguCe5zZx6n7qLtREk4kyE69SXVC0VSN8=
cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A=
@@ -193,9 +197,11 @@
github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -510,9 +516,11 @@
golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -624,8 +632,10 @@
google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
google.golang.org/api v0.66.0/go.mod h1:I1dmXYpX7HGwz/ejRxwQp2qj5bFAz93HiCU1C1oYd9M=
-google.golang.org/api v0.68.0 h1:9eJiHhwJKIYX6sX2fUZxQLi7pDRA/MYu8c12q6WbJik=
+google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
google.golang.org/api v0.68.0/go.mod h1:sOM8pTpwgflXRhz+oC8H2Dr+UcbMqkPPWNJo88Q7TH8=
+google.golang.org/api v0.70.0 h1:67zQnAE0T2rB0A3CwLSas0K+SbVzSxP+zTLkQLexeiw=
+google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -701,10 +711,14 @@
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220201184016-50beb8ab5c44/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220204002441-d6cc3cc0770e/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220207185906-7721543eae58 h1:i67FGOy2/zGfhE3YgHdrOrcFbOBhqdcRoBrsDqSQrOI=
+google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220207185906-7721543eae58/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf h1:SVYXkUz2yZS9FWb2Gm8ivSlbNQzL2Z/NpPKE3RG2jWk=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
diff --git a/internal/worker/scan_modules.go b/internal/worker/scan_modules.go
index b5384d4..e3ed344 100644
--- a/internal/worker/scan_modules.go
+++ b/internal/worker/scan_modules.go
@@ -11,14 +11,17 @@
"io"
"os"
"path/filepath"
+ "sort"
"strings"
"time"
+ "cloud.google.com/go/storage"
"golang.org/x/exp/vulncheck"
"golang.org/x/tools/go/packages"
vulnc "golang.org/x/vuln/client"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/worker/log"
+ "golang.org/x/vulndb/internal/worker/store"
)
// Selected repos under golang.org/x.
@@ -33,9 +36,9 @@
"golang.org/x/vuln", "golang.org/x/vulndb", "golang.org/x/website",
}
-// scanModules scans a list of Go modules for vulnerabilities.
+// ScanModules scans a list of Go modules for vulnerabilities.
// It assumes the root of each repo is a module, and there are no nested modules.
-func scanModules(ctx context.Context) error {
+func ScanModules(ctx context.Context, st store.Store) error {
dbClient, err := vulnc.NewClient([]string{vulnDBURL}, vulnc.Options{})
if err != nil {
return err
@@ -46,7 +49,7 @@
if err != nil {
return err
}
- if err := processModule(ctx, modulePath, latest, dbClient); err != nil {
+ if err := processModule(ctx, modulePath, latest, dbClient, st); err != nil {
return err
}
latestTagged, err := latestTaggedVersion(ctx, modulePath)
@@ -54,7 +57,7 @@
return err
}
if latestTagged != "" && latestTagged != latest {
- if err := processModule(ctx, modulePath, latestTagged, dbClient); err != nil {
+ if err := processModule(ctx, modulePath, latestTagged, dbClient, st); err != nil {
return err
}
}
@@ -62,8 +65,27 @@
return nil
}
-func processModule(ctx context.Context, modulePath, version string, dbClient vulnc.Client) error {
+func processModule(ctx context.Context, modulePath, version string, dbClient vulnc.Client, st store.Store) (err error) {
+ defer derrors.Wrap(&err, "processModule(%q, %q)", modulePath, version)
+
+ dbTime, err := vulnDBTime(ctx)
+ if err != nil {
+ return err
+ }
+ r, err := st.GetModuleScanRecord(ctx, modulePath, version, dbTime)
+ if err != nil {
+ return err
+ }
+ if r != nil {
+ // Already done.
+ log.Debugf(ctx, "already scanned %s@%s at DB time %s", modulePath, version, dbTime)
+ return nil
+ }
+
res, err := scanModule(ctx, modulePath, version, dbClient)
+ if err2 := createModuleScanRecord(ctx, st, modulePath, version, dbTime, res, err); err2 != nil {
+ return err2
+ }
if err != nil {
return err
}
@@ -75,6 +97,32 @@
return nil
}
+func createModuleScanRecord(ctx context.Context, st store.Store, path, version string, dbTime time.Time, res *vulncheck.Result, err error) error {
+ var errstr string
+ var vulnIDs []string
+ if err != nil {
+ errstr = err.Error()
+ } else {
+ m := map[string]bool{}
+ for _, v := range res.Vulns {
+ m[v.OSV.ID] = true
+ }
+ for id := range m {
+ vulnIDs = append(vulnIDs, id)
+ }
+ sort.Strings(vulnIDs)
+ }
+
+ return st.CreateModuleScanRecord(ctx, &store.ModuleScanRecord{
+ Path: path,
+ Version: version,
+ DBTime: dbTime,
+ Error: errstr,
+ VulnIDs: vulnIDs,
+ FinishedAt: time.Now(),
+ })
+}
+
// scanRepo clones the given repo and analyzes it for vulnerabilities. If commit
// is "HEAD", the head commit is scanned. Otherwise, commit must be a hex string
// corresponding to a commit, and that commit is checked out and scanned.
@@ -165,3 +213,19 @@
}
return nil
}
+
+// vulnDBTime returns the time that the vuln DB was last updated.
+func vulnDBTime(ctx context.Context) (_ time.Time, err error) {
+ // Until the vuln DB client supports this, use the update time
+ // of the index file.
+ defer derrors.Wrap(&err, "vulnDBTime")
+ c, err := storage.NewClient(ctx)
+ if err != nil {
+ return time.Time{}, err
+ }
+ attrs, err := c.Bucket(vulnDBBucket).Object("index.json").Attrs(ctx)
+ if err != nil {
+ return time.Time{}, err
+ }
+ return attrs.Updated, nil
+}
diff --git a/internal/worker/scan_modules_test.go b/internal/worker/scan_modules_test.go
index 9cca86c..fdeb4ae 100644
--- a/internal/worker/scan_modules_test.go
+++ b/internal/worker/scan_modules_test.go
@@ -13,6 +13,7 @@
"golang.org/x/exp/event"
vulnc "golang.org/x/vuln/client"
"golang.org/x/vulndb/internal/worker/log"
+ "golang.org/x/vulndb/internal/worker/store"
)
// TestScanModules is slow, so put it behind a flag.
@@ -22,10 +23,10 @@
if !*runScanModulesTest {
t.Skip("-scan flag missing")
}
- // Verify only that scanRepos works (doesn't return an error).
+ // Verify only that scanModules works (doesn't return an error).
ctx := event.WithExporter(context.Background(),
event.NewExporter(log.NewLineHandler(os.Stderr), nil))
- if err := scanModules(ctx); err != nil {
+ if err := ScanModules(ctx, store.NewMemStore()); err != nil {
t.Fatal(err)
}
}
diff --git a/internal/worker/server.go b/internal/worker/server.go
index 628ac7e..9039847 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -375,7 +375,7 @@
}
func (s *Server) handleScanModules(w http.ResponseWriter, r *http.Request) error {
- return scanModules(r.Context())
+ return ScanModules(r.Context(), s.cfg.Store)
}
func initOpenTelemetry(projectID string) (tp *sdktrace.TracerProvider, mp metric.MeterProvider, err error) {
diff --git a/internal/worker/worker.go b/internal/worker/worker.go
index 02fc986..f29175f 100644
--- a/internal/worker/worker.go
+++ b/internal/worker/worker.go
@@ -126,7 +126,10 @@
return c.msg
}
-const vulnDBURL = "https://storage.googleapis.com/go-vulndb"
+const (
+ vulnDBBucket = "go-vulndb"
+ vulnDBURL = "https://storage.googleapis.com/" + vulnDBBucket
+)
// readVulnDB returns a list of all CVE IDs in the Go vuln DB.
func readVulnDB(ctx context.Context) ([]string, error) {
