internal, cmd: pin govulncheck to new version
This change updates govulncheck to the latest version @master, and
changes logic to reflect the new json output. This includes updating the
local vulndb test data to reflect the new vulndb schema and updating
various tests related to running govulncheck in the ecosystem metrics
pipeline.
Change-Id: I7c0394e6d78844f9deb19485040b8030748109de
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/496887
Reviewed-by: Zvonimir Pavlinovic <zpavlinovic@google.com>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Maceo Thompson <maceothompson@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/cmd/govulncheck_sandbox/govulncheck_sandbox.go b/cmd/govulncheck_sandbox/govulncheck_sandbox.go
index a8ef94c..90220ea 100644
--- a/cmd/govulncheck_sandbox/govulncheck_sandbox.go
+++ b/cmd/govulncheck_sandbox/govulncheck_sandbox.go
@@ -17,9 +17,6 @@
"fmt"
"io"
"os"
- "os/exec"
- "syscall"
- "time"
"golang.org/x/pkgsite-metrics/internal/govulncheck"
"golang.org/x/pkgsite-metrics/internal/worker"
@@ -69,34 +66,13 @@
}
func runGovulncheck(govulncheckPath, mode, filePath, vulnDBDir string) (*govulncheck.SandboxResponse, error) {
- pattern := "./..."
- dir := ""
-
- if mode == worker.ModeBinary {
- pattern = filePath
- } else {
- dir = filePath
+ response := govulncheck.SandboxResponse{
+ Stats: govulncheck.ScanStats{},
}
-
- govulncheckCmd := exec.Command(govulncheckPath, "-json", pattern)
- govulncheckCmd.Dir = dir
- govulncheckCmd.Env = append(govulncheckCmd.Environ(), "GOVULNDB=file://"+vulnDBDir)
- start := time.Now()
- output, err := govulncheckCmd.CombinedOutput()
- if err != nil {
- // Temporary check because govulncheck currently exits code 3 if any vulns
- // are found but no other errors occurred.
- if e := (&exec.ExitError{}); !errors.As(err, &e) || e.ProcessState.ExitCode() != 3 {
- return nil, fmt.Errorf("govulncheck error: err=%v out=%s", err, output)
- }
- }
- response := govulncheck.SandboxResponse{}
- response.Stats.ScanSeconds = time.Since(start).Seconds()
- result, err := govulncheck.UnmarshalGovulncheckResult(output)
+ findings, err := govulncheck.RunGovulncheckCmd(govulncheckPath, mode, filePath, vulnDBDir, &response.Stats)
if err != nil {
return nil, err
}
- response.Res = *result
- response.Stats.ScanMemory = uint64(govulncheckCmd.ProcessState.SysUsage().(*syscall.Rusage).Maxrss)
+ response.Findings = findings
return &response, nil
}
diff --git a/cmd/govulncheck_sandbox/govulncheck_sandbox_test.go b/cmd/govulncheck_sandbox/govulncheck_sandbox_test.go
index 6e1de92..9ff5a7d 100644
--- a/cmd/govulncheck_sandbox/govulncheck_sandbox_test.go
+++ b/cmd/govulncheck_sandbox/govulncheck_sandbox_test.go
@@ -17,8 +17,8 @@
"golang.org/x/pkgsite-metrics/internal/buildtest"
"golang.org/x/pkgsite-metrics/internal/derrors"
"golang.org/x/pkgsite-metrics/internal/govulncheck"
+ "golang.org/x/pkgsite-metrics/internal/govulncheckapi"
"golang.org/x/pkgsite-metrics/internal/worker"
- govulncheckapi "golang.org/x/vuln/exp/govulncheck"
)
func Test(t *testing.T) {
@@ -31,13 +31,13 @@
t.Fatal(err)
}
- checkVuln := func(t *testing.T, res *govulncheckapi.Result) {
+ checkVuln := func(t *testing.T, findings []*govulncheckapi.Finding) {
wantID := "GO-2021-0113"
- i := slices.IndexFunc(res.Vulns, func(v *govulncheckapi.Vuln) bool {
- return v.OSV.ID == wantID
+ i := slices.IndexFunc(findings, func(f *govulncheckapi.Finding) bool {
+ return f.OSV == wantID
})
if i < 0 {
- t.Fatalf("no vuln with ID %s. Result:\n%+v", wantID, res)
+ t.Fatalf("no vuln with ID %s. Result:\n%+v", wantID, findings)
}
}
@@ -56,7 +56,7 @@
t.Fatal(err)
}
- checkVuln(t, &resp.Res)
+ checkVuln(t, resp.Findings)
if resp.Stats.ScanSeconds <= 0 {
t.Errorf("got %f; want >0 scan seconds", resp.Stats.ScanSeconds)
}
@@ -78,7 +78,7 @@
if err != nil {
t.Fatal(err)
}
- checkVuln(t, &resp.Res)
+ checkVuln(t, resp.Findings)
})
// Errors
@@ -94,8 +94,8 @@
},
{
name: "no vulndb",
- args: []string{govulncheckPath, worker.ModeGovulncheck, module, "does not exist"},
- want: "does not exist",
+ args: []string{govulncheckPath, worker.ModeGovulncheck, module, "DNE"},
+ want: "URL missing path",
},
{
name: "no mode",
@@ -105,7 +105,9 @@
{
name: "no module",
args: []string{govulncheckPath, worker.ModeGovulncheck, "nosuchmodule", vulndb},
- want: "no such file",
+ // Once govulncheck destinguishes this issue from no .mod file,
+ // update want to reflect govulncheck's new output
+ want: "no go.mod",
},
} {
t.Run(test.name, func(t *testing.T) {
diff --git a/go.mod b/go.mod
index ab648c9..d30d426 100644
--- a/go.mod
+++ b/go.mod
@@ -25,16 +25,16 @@
go.opentelemetry.io/otel/sdk v1.4.0
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
golang.org/x/exp/event v0.0.0-20220218215828-6cf2b201936e
- golang.org/x/mod v0.7.0
- golang.org/x/net v0.7.0
- golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2
- golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1
+ golang.org/x/mod v0.10.0
+ golang.org/x/net v0.9.0
+ golang.org/x/tools v0.8.1-0.20230421161920-b9619ee54b47
+ golang.org/x/vuln v0.1.1-0.20230519043451-0e42683d7808
google.golang.org/api v0.110.0
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec
google.golang.org/grpc v1.53.0
google.golang.org/protobuf v1.28.1
honnef.co/go/tools v0.4.3
- mvdan.cc/unparam v0.0.0-20220926085101-66de63301820
+ mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8
)
require (
@@ -59,8 +59,8 @@
golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
golang.org/x/oauth2 v0.5.0 // indirect
golang.org/x/sync v0.1.0 // indirect
- golang.org/x/sys v0.5.0 // indirect
- golang.org/x/text v0.7.0 // indirect
+ golang.org/x/sys v0.7.0 // indirect
+ golang.org/x/text v0.9.0 // indirect
golang.org/x/time v0.1.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index d68fb22..9edef40 100644
--- a/go.sum
+++ b/go.sum
@@ -351,8 +351,8 @@
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -391,8 +391,8 @@
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -477,8 +477,8 @@
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -490,8 +490,8 @@
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -549,10 +549,10 @@
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2 h1:v0FhRDmSCNH/0EurAT6T8KRY4aNuUhz6/WwBMxG+gvQ=
-golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
-golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1 h1:HRexnHfiDA2hkPNMDgf3vxabRMeC+XeS8tCKP9olVbs=
-golang.org/x/vuln v0.0.0-20230201222900-4c848edceff1/go.mod h1:cBP4HMKv0X+x96j8IJWCKk0eqpakBmmHjKGSSC0NaYE=
+golang.org/x/tools v0.8.1-0.20230421161920-b9619ee54b47 h1:fQlOhMJ24apqitZX8S4hbCbHU1Z9AvyWkN3BYI55Le4=
+golang.org/x/tools v0.8.1-0.20230421161920-b9619ee54b47/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
+golang.org/x/vuln v0.1.1-0.20230519043451-0e42683d7808 h1:wONxNPbHtFp0Ern9o6AIePAWZchvw16H5ZA2t3AkOFY=
+golang.org/x/vuln v0.1.1-0.20230519043451-0e42683d7808/go.mod h1:V0eyhHwaAaHrt42J9bgrN6rd12f6GU4T0Lu0ex2wDg4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -734,8 +734,8 @@
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw=
honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA=
-mvdan.cc/unparam v0.0.0-20220926085101-66de63301820 h1:fggBTMFbBz7CMny3mWZphe0B/6D8ILBunvvB1cNNHi8=
-mvdan.cc/unparam v0.0.0-20220926085101-66de63301820/go.mod h1:7fKhD/gH+APJ9Y27S2PYO7+oVWtb3XPrw9W5ayxVq2A=
+mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8 h1:VuJo4Mt0EVPychre4fNlDWDuE5AjXtPJpRUWqZDQhaI=
+mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8/go.mod h1:Oh/d7dEtzsNHGOq1Cdv8aMm3KdKhVvPbRQcM8WFpBR8=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/internal/govulncheck/govulncheck.go b/internal/govulncheck/govulncheck.go
index be46da6..035ea42 100644
--- a/internal/govulncheck/govulncheck.go
+++ b/internal/govulncheck/govulncheck.go
@@ -7,17 +7,28 @@
package govulncheck
import (
+ "bytes"
"context"
"encoding/json"
"errors"
"net/http"
+ "os/exec"
"strings"
+ "syscall"
"time"
"golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/derrors"
+ "golang.org/x/pkgsite-metrics/internal/govulncheckapi"
"golang.org/x/pkgsite-metrics/internal/scan"
- "golang.org/x/vuln/exp/govulncheck"
+)
+
+const (
+ // ModeBinary runs the govulncheck binary in binary mode.
+ ModeBinary string = "BINARY"
+
+ // ModeGovulncheck runs the govulncheck binary in default (source) mode.
+ ModeGovulncheck = "GOVULNCHECK"
)
// EnqueueQueryParams for govulncheck/enqueue.
@@ -80,22 +91,22 @@
}, nil
}
-func ConvertGovulncheckOutput(v *govulncheck.Vuln) (vulns []*Vuln) {
- for _, module := range v.Modules {
- for _, pkg := range module.Packages {
- vuln := &Vuln{
- ID: v.OSV.ID,
- ModulePath: module.Path,
- PackagePath: pkg.Path,
- Version: module.FoundVersion,
- }
- if len(pkg.CallStacks) > 0 {
- vuln.Called = true
- }
- vulns = append(vulns, vuln)
- }
+// ConvertGovulncheckFinding takes a finding from govulncheck and converts it to
+// a bigquery vuln.
+func ConvertGovulncheckFinding(f *govulncheckapi.Finding) *Vuln {
+ vulnerableFrame := f.Trace[0]
+ vuln := &Vuln{
+ ID: f.OSV,
+ PackagePath: vulnerableFrame.Package,
+ ModulePath: vulnerableFrame.Module,
+ Version: vulnerableFrame.Version,
+ Called: false,
}
- return vulns
+ if vulnerableFrame.Function != "" {
+ vuln.Called = true
+ }
+
+ return vuln
}
const TableName = "govulncheck"
@@ -233,8 +244,8 @@
// and statistics about memory usage and run time. Used
// for capturing result of govulncheck run in a sandbox.
type SandboxResponse struct {
- Res govulncheck.Result
- Stats ScanStats
+ Findings []*govulncheckapi.Finding
+ Stats ScanStats
}
func UnmarshalSandboxResponse(output []byte) (*SandboxResponse, error) {
@@ -252,19 +263,34 @@
return &res, nil
}
-func UnmarshalGovulncheckResult(output []byte) (*govulncheck.Result, error) {
- var e struct {
- Error string
+func RunGovulncheckCmd(govulncheckPath, mode, modulePath, vulndbDir string, stats *ScanStats) ([]*govulncheckapi.Finding, error) {
+ pattern := "./..."
+ dir := ""
+
+ if mode == ModeBinary {
+ pattern = modulePath
+ } else {
+ dir = modulePath
}
- if err := json.Unmarshal(output, &e); err != nil {
+
+ stdOut := bytes.Buffer{}
+ stdErr := bytes.Buffer{}
+ govulncheckCmd := exec.Command(govulncheckPath, "-json", "-db=file://"+vulndbDir, "-C="+dir, pattern)
+
+ govulncheckCmd.Stdout = &stdOut
+ govulncheckCmd.Stderr = &stdErr
+
+ start := time.Now()
+ if err := govulncheckCmd.Run(); err != nil {
+ return nil, errors.New(stdErr.String())
+ }
+ stats.ScanSeconds = time.Since(start).Seconds()
+ stats.ScanMemory = uint64(govulncheckCmd.ProcessState.SysUsage().(*syscall.Rusage).Maxrss)
+
+ handler := NewMetricsHandler()
+ err := govulncheckapi.HandleJSON(&stdOut, handler)
+ if err != nil {
return nil, err
}
- if e.Error != "" {
- return nil, errors.New(e.Error)
- }
- var res govulncheck.Result
- if err := json.Unmarshal(output, &res); err != nil {
- return nil, err
- }
- return &res, nil
+ return handler.Findings(), nil
}
diff --git a/internal/govulncheck/govulncheck_test.go b/internal/govulncheck/govulncheck_test.go
index e7b515a..b9afc83 100644
--- a/internal/govulncheck/govulncheck_test.go
+++ b/internal/govulncheck/govulncheck_test.go
@@ -12,134 +12,72 @@
bq "cloud.google.com/go/bigquery"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/pkgsite-metrics/internal/bigquery"
- "golang.org/x/pkgsite-metrics/internal/osv"
+ "golang.org/x/pkgsite-metrics/internal/govulncheckapi"
test "golang.org/x/pkgsite-metrics/internal/testing"
- "golang.org/x/vuln/exp/govulncheck"
- oldOsv "golang.org/x/vuln/osv"
"google.golang.org/api/iterator"
)
-func TestConvertGovulncheckOutput(t *testing.T) {
+func TestConvertGovulncheckFinding(t *testing.T) {
var (
- newOsvEntry = &osv.Entry{
- ID: "GO-YYYY-1234",
- Affected: []osv.Affected{
+ osvID = "GO-YYYY-XXXX"
+ vuln1 = &govulncheckapi.Finding{
+ OSV: osvID,
+ Trace: []*govulncheckapi.Frame{
{
- Module: osv.Module{
- Path: "example.com/repo/module",
- Ecosystem: "Go",
- },
- EcosystemSpecific: osv.EcosystemSpecific{
- Packages: []osv.Package{
- {
- Path: "example.com/repo/module/package",
- Symbols: []string{
- "Symbol",
- "Another",
- },
- },
- },
- },
- },
- },
- }
- osvEntry = &oldOsv.Entry{
- ID: newOsvEntry.ID,
- Affected: []oldOsv.Affected{
- {
- Package: oldOsv.Package{
- Name: "example.com/repo/module",
- Ecosystem: "Go",
- },
- EcosystemSpecific: oldOsv.EcosystemSpecific{
- Imports: []oldOsv.EcosystemSpecificImport{
- {
- Path: "example.com/repo/module/package",
- Symbols: []string{
- "Symbol",
- "Another",
- },
- },
- },
- },
+ Module: "example.com/repo/module",
+ Version: "v0.0.1",
+ Package: "example.com/repo/module/package",
+ Function: "func",
+ Position: &govulncheckapi.Position{},
},
},
}
- vuln1 = &govulncheck.Vuln{
- OSV: osvEntry,
- Modules: []*govulncheck.Module{
+ vuln2 = &govulncheckapi.Finding{
+ OSV: osvID,
+ FixedVersion: "",
+ Trace: []*govulncheckapi.Frame{
{
- FoundVersion: "v0.0.1",
- Path: "example.com/repo/module",
- Packages: []*govulncheck.Package{
- {
- Path: "example.com/repo/module/package",
- CallStacks: []govulncheck.CallStack{
- {
- Symbol: "Symbol",
- Summary: "example.go:1:1 xyz.func calls pkgPath.Symbol",
- Frames: []*govulncheck.StackFrame{},
- },
- },
- },
- },
- },
- },
- }
-
- vuln2 = &govulncheck.Vuln{
- OSV: osvEntry,
- Modules: []*govulncheck.Module{
- {
- FoundVersion: "v1.0.0",
- Path: "example.com/repo/module",
- Packages: []*govulncheck.Package{
- {
- Path: "example.com/repo/module/package",
- },
- },
+ Module: "example.com/repo/module",
+ Version: "v1.0.0",
+ Package: "example.com/repo/module/package",
+ Position: nil,
},
},
}
)
tests := []struct {
- name string
- vuln *govulncheck.Vuln
- wantVulns []*Vuln
+ name string
+ vuln *govulncheckapi.Finding
+ wantVuln *Vuln
}{
{
- name: "call one symbol but not all",
+ name: "called",
vuln: vuln1,
- wantVulns: []*Vuln{
- {
- ID: "GO-YYYY-1234",
- PackagePath: "example.com/repo/module/package",
- ModulePath: "example.com/repo/module",
- Version: "v0.0.1",
- Called: true,
- },
+ wantVuln: &Vuln{
+ ID: "GO-YYYY-XXXX",
+ PackagePath: "example.com/repo/module/package",
+ ModulePath: "example.com/repo/module",
+ Version: "v0.0.1",
+ Called: true,
},
},
{
- name: "call no symbols",
+ name: "Not called",
vuln: vuln2,
- wantVulns: []*Vuln{
- {
- ID: "GO-YYYY-1234",
- PackagePath: "example.com/repo/module/package",
- ModulePath: "example.com/repo/module",
- Version: "v1.0.0",
- Called: false,
- },
+ wantVuln: &Vuln{
+ ID: "GO-YYYY-XXXX",
+ PackagePath: "example.com/repo/module/package",
+ ModulePath: "example.com/repo/module",
+ Version: "v1.0.0",
+ Called: false,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if diff := cmp.Diff(ConvertGovulncheckOutput(tt.vuln), tt.wantVulns, cmpopts.EquateEmpty(), cmp.AllowUnexported(Vuln{})); diff != "" {
+ if diff := cmp.Diff(ConvertGovulncheckFinding(tt.vuln), tt.wantVuln, cmp.AllowUnexported(Vuln{})); diff != "" {
t.Errorf("mismatch (-got, +want): %s", diff)
}
})
diff --git a/internal/govulncheck/handler.go b/internal/govulncheck/handler.go
new file mode 100644
index 0000000..44bf307
--- /dev/null
+++ b/internal/govulncheck/handler.go
@@ -0,0 +1,51 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package govulncheck
+
+import (
+ "golang.org/x/exp/maps"
+ "golang.org/x/pkgsite-metrics/internal/govulncheckapi"
+ "golang.org/x/pkgsite-metrics/internal/osv"
+)
+
+// NewMetricsHandler returns a handler that returns a set of all findings.
+// For use in the ecosystem metrics pipeline.
+func NewMetricsHandler() *MetricsHandler {
+ m := make(map[string]*govulncheckapi.Finding)
+ return &MetricsHandler{
+ byOSV: m,
+ }
+}
+
+type MetricsHandler struct {
+ byOSV map[string]*govulncheckapi.Finding
+}
+
+func (h *MetricsHandler) Config(c *govulncheckapi.Config) error {
+ return nil
+}
+
+func (h *MetricsHandler) Progress(p *govulncheckapi.Progress) error {
+ return nil
+}
+
+func (h *MetricsHandler) OSV(e *osv.Entry) error {
+ return nil
+}
+
+func (h *MetricsHandler) Finding(finding *govulncheckapi.Finding) error {
+ f, found := h.byOSV[finding.OSV]
+ if !found || f.Trace[0].Function == "" {
+ // If the vuln wasn't called in the first trace, replace it with
+ // the new finding (that way if the vuln is called at any point
+ // it's trace will reflect that, which is needed when converting to bq)
+ h.byOSV[finding.OSV] = finding
+ }
+ return nil
+}
+
+func (h *MetricsHandler) Findings() []*govulncheckapi.Finding {
+ return maps.Values(h.byOSV)
+}
diff --git a/internal/govulncheck/handler_test.go b/internal/govulncheck/handler_test.go
new file mode 100644
index 0000000..cfba8ca
--- /dev/null
+++ b/internal/govulncheck/handler_test.go
@@ -0,0 +1,50 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package govulncheck
+
+import (
+ "testing"
+
+ "golang.org/x/pkgsite-metrics/internal/govulncheckapi"
+)
+
+func TestMetricsHandler(t *testing.T) {
+ osvID := "GO-YYYY-XXXX"
+ calledFinding := &govulncheckapi.Finding{
+ OSV: osvID,
+ Trace: []*govulncheckapi.Frame{
+ {
+ Module: "example.com/repo/module",
+ Version: "v0.0.1",
+ Package: "example.com/repo/module/package",
+ Function: "func",
+ Position: &govulncheckapi.Position{},
+ },
+ },
+ }
+
+ uncalledFinding := &govulncheckapi.Finding{
+ OSV: osvID,
+ FixedVersion: "",
+ Trace: []*govulncheckapi.Frame{
+ {
+ Module: "example.com/repo/module",
+ Version: "v0.0.1",
+ Package: "example.com/repo/module/package",
+ Position: nil,
+ },
+ },
+ }
+
+ t.Run("Called finding overwrites uncalled w/ same ID", func(t *testing.T) {
+ h := NewMetricsHandler()
+ h.Finding(uncalledFinding)
+ h.Finding(calledFinding)
+ findings := h.Findings()
+ if len(findings) != 1 || findings[0] != calledFinding {
+ t.Errorf("MetricsHandler.Finding() error: expected %v, got %v", calledFinding, findings[0])
+ }
+ })
+}
diff --git a/internal/govulncheckapi/handler.go b/internal/govulncheckapi/handler.go
new file mode 100644
index 0000000..d5d672f
--- /dev/null
+++ b/internal/govulncheckapi/handler.go
@@ -0,0 +1,61 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The govulncheckapi package is copied from x/vuln/internal/govulncheck
+// and matches the output structure of govulncheck when ran in -json mode.
+package govulncheckapi
+
+import (
+ "encoding/json"
+ "io"
+
+ "golang.org/x/pkgsite-metrics/internal/osv"
+)
+
+// Handler handles messages to be presented in a vulnerability scan output
+// stream.
+type Handler interface {
+ // Config communicates introductory message to the user.
+ Config(config *Config) error
+
+ // Progress is called to display a progress message.
+ Progress(progress *Progress) error
+
+ // OSV is invoked for each osv Entry in the stream.
+ OSV(entry *osv.Entry) error
+
+ // Finding is called for each vulnerability finding in the stream.
+ Finding(finding *Finding) error
+}
+
+// HandleJSON reads the json from the supplied stream and hands the decoded
+// output to the handler.
+func HandleJSON(from io.Reader, to Handler) error {
+ dec := json.NewDecoder(from)
+ for dec.More() {
+ msg := Message{}
+ // decode the next message in the stream
+ if err := dec.Decode(&msg); err != nil {
+ return err
+ }
+ // dispatch the message
+ var err error
+ if msg.Config != nil {
+ err = to.Config(msg.Config)
+ }
+ if msg.Progress != nil {
+ err = to.Progress(msg.Progress)
+ }
+ if msg.OSV != nil {
+ err = to.OSV(msg.OSV)
+ }
+ if msg.Finding != nil {
+ err = to.Finding(msg.Finding)
+ }
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/internal/govulncheckapi/result.go b/internal/govulncheckapi/result.go
new file mode 100644
index 0000000..8853792
--- /dev/null
+++ b/internal/govulncheckapi/result.go
@@ -0,0 +1,140 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The govulncheckapi package is copied from x/vuln/internal/govulncheck
+// and matches the output structure of govulncheck when ran in -json mode.
+package govulncheckapi
+
+import (
+ "time"
+
+ "golang.org/x/pkgsite-metrics/internal/osv"
+)
+
+// Message is an entry in the output stream. It will always have exactly one
+// field filled in.
+type Message struct {
+ Config *Config `json:"config,omitempty"`
+ Progress *Progress `json:"progress,omitempty"`
+ OSV *osv.Entry `json:"osv,omitempty"`
+ Finding *Finding `json:"finding,omitempty"`
+}
+
+// ProtocolVersion is the current protocol version this file implements
+const ProtocolVersion = "v0.1.0"
+
+type Config struct {
+ // ProtocolVersion specifies the version of the JSON protocol.
+ ProtocolVersion string `json:"protocol_version,omitempty"`
+
+ // ScannerName is the name of the tool, for example, govulncheck.
+ //
+ // We expect this JSON format to be used by other tools that wrap
+ // govulncheck, which will have a different name.
+ ScannerName string `json:"scanner_name,omitempty"`
+
+ // ScannerVersion is the version of the tool.
+ ScannerVersion string `json:"scanner_version,omitempty"`
+
+ // DB is the database used by the tool, for example,
+ // vuln.go.dev.
+ DB string `json:"db,omitempty"`
+
+ // LastModified is the last modified time of the data source.
+ DBLastModified *time.Time `json:"db_last_modified,omitempty"`
+
+ // GoVersion is the version of Go used for analyzing standard library
+ // vulnerabilities.
+ GoVersion string `json:"go_version,omitempty"`
+
+ // Consider only vulnerabilities that apply to this OS.
+ GOOS string `json:"goos,omitempty"`
+
+ // Consider only vulnerabilities that apply to this architecture.
+ GOARCH string `json:"goarch,omitempty"`
+
+ // ImportsOnly instructs vulncheck to analyze import chains only.
+ // Otherwise, call chains are analyzed too.
+ ImportsOnly bool `json:"imports_only,omitempty"`
+}
+
+type Progress struct {
+ // A time stamp for the message.
+ Timestamp *time.Time `json:"time,omitempty"`
+
+ // Message is the progress message.
+ Message string `json:"message,omitempty"`
+}
+
+// Vuln represents a single OSV entry.
+type Finding struct {
+ // OSV is the id of the detected vulnerability.
+ OSV string `json:"osv,omitempty"`
+
+ // FixedVersion is the module version where the vulnerability was
+ // fixed. This is empty if a fix is not available.
+ //
+ // If there are multiple fixed versions in the OSV report, this will
+ // be the fixed version in the latest range event for the OSV report.
+ //
+ // For example, if the range events are
+ // {introduced: 0, fixed: 1.0.0} and {introduced: 1.1.0}, the fixed version
+ // will be empty.
+ //
+ // For the stdlib, we will show the fixed version closest to the
+ // Go version that is used. For example, if a fix is available in 1.17.5 and
+ // 1.18.5, and the GOVERSION is 1.17.3, 1.17.5 will be returned as the
+ // fixed version.
+ FixedVersion string `json:"fixed_version,omitempty"`
+
+ // Trace contains an entry for each frame in the trace.
+ //
+ // Frames are sorted starting from the imported vulnerable symbol
+ // until the entry point. The first frame in Frames should match
+ // Symbol.
+ //
+ // In binary mode, trace will contain a single-frame with no position
+ // information.
+ //
+ // When a package is imported but no vulnerable symbol is called, the trace
+ // will contain a single-frame with no symbol or position information.
+ Trace []*Frame `json:"trace,omitempty"`
+}
+
+// Frame represents an entry in a finding trace.
+type Frame struct {
+ // Module is the module path of the module containing this symbol.
+ //
+ // Importable packages in the standard library will have the path "stdlib".
+ Module string `json:"module"`
+
+ // Version is the module version from the build graph.
+ Version string `json:"version,omitempty"`
+
+ // Package is the import path.
+ Package string `json:"package,omitempty"`
+
+ // Function is the function name.
+ Function string `json:"function,omitempty"`
+
+ // Receiver is the receiver type if the called symbol is a method.
+ //
+ // The client can create the final symbol name by
+ // prepending Receiver to FuncName.
+ Receiver string `json:"receiver,omitempty"`
+
+ // Position describes an arbitrary source position
+ // including the file, line, and column location.
+ // A Position is valid if the line number is > 0.
+ Position *Position `json:"position,omitempty"`
+}
+
+// Position is a copy of token.Position used to marshal/unmarshal
+// JSON correctly.
+type Position struct {
+ Filename string `json:"filename,omitempty"` // filename, if any
+ Offset int `json:"offset"` // offset, starting at 0
+ Line int `json:"line"` // line number, starting at 1
+ Column int `json:"column"` // column number, starting at 1 (byte count)
+}
diff --git a/internal/testdata/vulndb/ID/GO-2020-0015.json b/internal/testdata/vulndb/ID/GO-2020-0015.json
new file mode 100644
index 0000000..9eb429d
--- /dev/null
+++ b/internal/testdata/vulndb/ID/GO-2020-0015.json
@@ -0,0 +1,77 @@
+{
+ "id": "GO-2020-0015",
+ "published": "2021-04-14T20:04:52Z",
+ "modified": "2021-06-07T12:00:00Z",
+ "aliases": [
+ "CVE-2020-14040",
+ "GHSA-5rcv-m4m3-hfh7"
+ ],
+ "details": "An attacker could provide a single byte to a UTF16 decoder instantiated with\nUseBOM or ExpectBOM to trigger an infinite loop if the String function on\nthe Decoder is called, or the Decoder is passed to transform.String.\nIf used to parse user supplied input, this may be used as a denial of service\nvector.\n",
+ "affected": [
+ {
+ "package": {
+ "name": "golang.org/x/text",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0"
+ },
+ {
+ "fixed": "0.3.3"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://pkg.go.dev/vuln/GO-2020-0015"
+ },
+ "ecosystem_specific": {
+ "imports": [
+ {
+ "path": "golang.org/x/text/encoding/unicode",
+ "symbols": [
+ "bomOverride.Transform",
+ "utf16Decoder.Transform"
+ ]
+ },
+ {
+ "path": "golang.org/x/text/transform",
+ "symbols": [
+ "Transform"
+ ]
+ }
+ ]
+ }
+ }
+ ],
+ "references": [
+ {
+ "type": "FIX",
+ "url": "https://go.dev/cl/238238"
+ },
+ {
+ "type": "FIX",
+ "url": "https://go.googlesource.com/text/+/23ae387dee1f90d29a23c0e87ee0b46038fbed0e"
+ },
+ {
+ "type": "WEB",
+ "url": "https://go.dev/issue/39491"
+ },
+ {
+ "type": "WEB",
+ "url": "https://groups.google.com/g/golang-announce/c/bXVeAmGOqz0"
+ },
+ {
+ "type": "WEB",
+ "url": "https://nvd.nist.gov/vuln/detail/CVE-2020-14040"
+ },
+ {
+ "type": "WEB",
+ "url": "https://github.com/advisories/GHSA-5rcv-m4m3-hfh7"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/internal/testdata/vulndb/ID/GO-2021-0113.json b/internal/testdata/vulndb/ID/GO-2021-0113.json
new file mode 100644
index 0000000..31a193f
--- /dev/null
+++ b/internal/testdata/vulndb/ID/GO-2021-0113.json
@@ -0,0 +1,60 @@
+{
+ "id": "GO-2021-0113",
+ "published": "2021-10-06T17:51:21Z",
+ "modified": "2021-10-06T17:51:21Z",
+ "aliases": [
+ "CVE-2021-38561"
+ ],
+ "details": "Due to improper index calculation, an incorrectly formatted language tag can cause Parse\nto panic via an out of bounds read. If Parse is used to process untrusted user inputs,\nthis may be used as a vector for a denial of service attack.\n",
+ "affected": [
+ {
+ "package": {
+ "name": "golang.org/x/text",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0"
+ },
+ {
+ "fixed": "0.3.7"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://pkg.go.dev/vuln/GO-2021-0113"
+ },
+ "ecosystem_specific": {
+ "imports": [
+ {
+ "path": "golang.org/x/text/language",
+ "symbols": [
+ "MatchStrings",
+ "MustParse",
+ "Parse",
+ "ParseAcceptLanguage"
+ ]
+ }
+ ]
+ }
+ }
+ ],
+ "references": [
+ {
+ "type": "FIX",
+ "url": "https://go.dev/cl/340830"
+ },
+ {
+ "type": "FIX",
+ "url": "https://go.googlesource.com/text/+/383b2e75a7a4198c42f8f87833eefb772868a56f"
+ },
+ {
+ "type": "WEB",
+ "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-38561"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/internal/testdata/vulndb/golang.org/x/text.json b/internal/testdata/vulndb/golang.org/x/text.json
deleted file mode 100644
index e9d55a3..0000000
--- a/internal/testdata/vulndb/golang.org/x/text.json
+++ /dev/null
@@ -1 +0,0 @@
-[{"id":"GO-2020-0015","published":"2021-04-14T20:04:52Z","modified":"2021-06-07T12:00:00Z","aliases":["CVE-2020-14040","GHSA-5rcv-m4m3-hfh7"],"details":"An attacker could provide a single byte to a UTF16 decoder instantiated with\nUseBOM or ExpectBOM to trigger an infinite loop if the String function on\nthe Decoder is called, or the Decoder is passed to transform.String.\nIf used to parse user supplied input, this may be used as a denial of service\nvector.\n","affected":[{"package":{"name":"golang.org/x/text","ecosystem":"Go"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"0.3.3"}]}],"database_specific":{"url":"https://pkg.go.dev/vuln/GO-2020-0015"},"ecosystem_specific":{"imports":[{"path":"golang.org/x/text/encoding/unicode","symbols":["bomOverride.Transform","utf16Decoder.Transform"]},{"path":"golang.org/x/text/transform","symbols":["Transform"]}]}}],"references":[{"type":"FIX","url":"https://go.dev/cl/238238"},{"type":"FIX","url":"https://go.googlesource.com/text/+/23ae387dee1f90d29a23c0e87ee0b46038fbed0e"},{"type":"WEB","url":"https://go.dev/issue/39491"},{"type":"WEB","url":"https://groups.google.com/g/golang-announce/c/bXVeAmGOqz0"},{"type":"WEB","url":"https://nvd.nist.gov/vuln/detail/CVE-2020-14040"},{"type":"WEB","url":"https://github.com/advisories/GHSA-5rcv-m4m3-hfh7"}]},{"id":"GO-2021-0113","published":"2021-10-06T17:51:21Z","modified":"2021-10-06T17:51:21Z","aliases":["CVE-2021-38561"],"details":"Due to improper index calculation, an incorrectly formatted language tag can cause Parse\nto panic via an out of bounds read. If Parse is used to process untrusted user inputs,\nthis may be used as a vector for a denial of service attack.\n","affected":[{"package":{"name":"golang.org/x/text","ecosystem":"Go"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"0.3.7"}]}],"database_specific":{"url":"https://pkg.go.dev/vuln/GO-2021-0113"},"ecosystem_specific":{"imports":[{"path":"golang.org/x/text/language","symbols":["MatchStrings","MustParse","Parse","ParseAcceptLanguage"]}]}}],"references":[{"type":"FIX","url":"https://go.dev/cl/340830"},{"type":"FIX","url":"https://go.googlesource.com/text/+/383b2e75a7a4198c42f8f87833eefb772868a56f"},{"type":"WEB","url":"https://nvd.nist.gov/vuln/detail/CVE-2021-38561"}]}]
\ No newline at end of file
diff --git a/internal/testdata/vulndb/index.json b/internal/testdata/vulndb/index.json
deleted file mode 100644
index 9a8f819..0000000
--- a/internal/testdata/vulndb/index.json
+++ /dev/null
@@ -1 +0,0 @@
-{"golang.org/x/text":"2022-09-20T15:16:04Z"}
diff --git a/internal/testdata/vulndb/index/db.json b/internal/testdata/vulndb/index/db.json
new file mode 100644
index 0000000..dceaf71
--- /dev/null
+++ b/internal/testdata/vulndb/index/db.json
@@ -0,0 +1 @@
+{"modified":"2023-05-18T20:38:56Z"}
diff --git a/internal/testdata/vulndb/index/modules.json b/internal/testdata/vulndb/index/modules.json
new file mode 100644
index 0000000..aa354d9
--- /dev/null
+++ b/internal/testdata/vulndb/index/modules.json
@@ -0,0 +1,15 @@
+[
+ {
+ "path": "golang.org/x/text",
+ "vulns": [
+ {
+ "id": "GO-2020-0015",
+ "modified": "2021-06-07T12:00:00Z"
+ },
+ {
+ "id": "GO-2021-0113",
+ "modified": "2021-10-06T17:51:21Z"
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/internal/testdata/vulndb/index/vulns.json b/internal/testdata/vulndb/index/vulns.json
new file mode 100644
index 0000000..af591a6
--- /dev/null
+++ b/internal/testdata/vulndb/index/vulns.json
@@ -0,0 +1,17 @@
+[
+ {
+ "id": "GO-2020-0015",
+ "modified": "2021-06-07T12:00:00Z",
+ "aliases": [
+ "CVE-2020-14040",
+ "GHSA-5rcv-m4m3-hfh7"
+ ]
+ },
+ {
+ "id": "GO-2021-0113",
+ "modified": "2021-10-06T17:51:21Z",
+ "aliases": [
+ "CVE-2021-38561"
+ ]
+ }
+]
\ No newline at end of file
diff --git a/internal/worker/govulncheck.go b/internal/worker/govulncheck.go
index 7196e59..9b09ae9 100644
--- a/internal/worker/govulncheck.go
+++ b/internal/worker/govulncheck.go
@@ -6,6 +6,7 @@
import (
"context"
+ "time"
"golang.org/x/pkgsite-metrics/internal"
"golang.org/x/pkgsite-metrics/internal/derrors"
@@ -43,7 +44,7 @@
defer h.mu.Unlock()
if h.workVersion == nil {
- lmt, err := h.vulndbClient.LastModifiedTime(ctx)
+ lmt := time.Now() // TODO: Implement this
if err != nil {
return nil, err
}
diff --git a/internal/worker/govulncheck_scan.go b/internal/worker/govulncheck_scan.go
index 0dc864e..821c08b 100644
--- a/internal/worker/govulncheck_scan.go
+++ b/internal/worker/govulncheck_scan.go
@@ -10,24 +10,20 @@
"fmt"
"net/http"
"os"
- "os/exec"
"path/filepath"
"regexp"
"strings"
- "syscall"
- "time"
"cloud.google.com/go/storage"
"golang.org/x/exp/event"
"golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/derrors"
"golang.org/x/pkgsite-metrics/internal/govulncheck"
+ "golang.org/x/pkgsite-metrics/internal/govulncheckapi"
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/proxy"
"golang.org/x/pkgsite-metrics/internal/sandbox"
"golang.org/x/pkgsite-metrics/internal/version"
- vulnclient "golang.org/x/vuln/client"
- govulncheckapi "golang.org/x/vuln/exp/govulncheck"
)
const (
@@ -150,7 +146,6 @@
// A scanner holds state for scanning modules.
type scanner struct {
proxyClient *proxy.Client
- dbClient vulnclient.Client
bqClient *bigquery.Client
workVersion *govulncheck.WorkVersion
gcsBucket *storage.BucketHandle
@@ -180,7 +175,6 @@
return &scanner{
proxyClient: h.proxyClient,
bqClient: h.bqClient,
- dbClient: h.vulndbClient,
workVersion: workVersion,
gcsBucket: bucket,
insecure: h.cfg.Insecure,
@@ -228,10 +222,10 @@
row.ScanMode = sreq.Mode
log.Infof(ctx, "running scanner.runScanModule: %s@%s", sreq.Path(), sreq.Version)
- stats := &scanStats{}
+ stats := &govulncheck.ScanStats{}
vulns, err := s.runScanModule(ctx, sreq.Module, info.Version, sreq.Suffix, sreq.Mode, stats)
- row.ScanSeconds = stats.scanSeconds
- row.ScanMemory = int64(stats.scanMemory)
+ row.ScanSeconds = stats.ScanSeconds
+ row.ScanMemory = int64(stats.ScanMemory)
if err != nil {
switch {
case isMissingGoMod(err) || isNoModulesSpecified(err):
@@ -310,14 +304,9 @@
return vs
}
-type scanStats struct {
- scanSeconds float64
- scanMemory uint64
-}
-
// runScanModule fetches the module version from the proxy, and analyzes it for
// vulnerabilities.
-func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *scanStats) (bvulns []*govulncheck.Vuln, err error) {
+func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *govulncheck.ScanStats) (bvulns []*govulncheck.Vuln, err error) {
err = doScan(ctx, modulePath, version, s.insecure, func() (err error) {
// In ModeBinary, path is a file path to the input binary.
// Otherwise, it is a path to the input module directory.
@@ -332,26 +321,26 @@
}
}
- var vulns []*govulncheckapi.Vuln
+ var findings []*govulncheckapi.Finding
if s.insecure {
- vulns, err = s.runGovulncheckScanInsecure(ctx, modulePath, version, inputPath, mode, stats)
+ findings, err = s.runGovulncheckScanInsecure(ctx, modulePath, version, inputPath, mode, stats)
} else {
- vulns, err = s.runGovulncheckScanSandbox(ctx, modulePath, version, inputPath, mode, stats)
+ findings, err = s.runGovulncheckScanSandbox(ctx, modulePath, version, inputPath, mode, stats)
}
if err != nil {
return err
}
- log.Debugf(ctx, "govulncheck stats: %dkb | %vs", stats.scanMemory, stats.scanSeconds)
+ log.Debugf(ctx, "govulncheck stats: %dkb | %vs", stats.ScanMemory, stats.ScanSeconds)
- for _, v := range vulns {
- bvulns = append(bvulns, govulncheck.ConvertGovulncheckOutput(v)...)
+ for _, v := range findings {
+ bvulns = append(bvulns, govulncheck.ConvertGovulncheckFinding(v))
}
return nil
})
return bvulns, err
}
-func (s *scanner) runGovulncheckScanSandbox(ctx context.Context, modulePath, version, inputPath, mode string, stats *scanStats) (_ []*govulncheckapi.Vuln, err error) {
+func (s *scanner) runGovulncheckScanSandbox(ctx context.Context, modulePath, version, inputPath, mode string, stats *govulncheck.ScanStats) (_ []*govulncheckapi.Finding, err error) {
if mode == ModeBinary {
return s.runBinaryScanSandbox(ctx, modulePath, version, inputPath, stats)
}
@@ -364,12 +353,12 @@
if err != nil {
return nil, err
}
- stats.scanMemory = response.Stats.ScanMemory
- stats.scanSeconds = response.Stats.ScanSeconds
- return response.Res.Vulns, nil
+ stats.ScanMemory = response.Stats.ScanMemory
+ stats.ScanSeconds = response.Stats.ScanSeconds
+ return response.Findings, nil
}
-func (s *scanner) runBinaryScanSandbox(ctx context.Context, modulePath, version, binDir string, stats *scanStats) ([]*govulncheckapi.Vuln, error) {
+func (s *scanner) runBinaryScanSandbox(ctx context.Context, modulePath, version, binDir string, stats *govulncheck.ScanStats) ([]*govulncheckapi.Finding, error) {
if s.gcsBucket == nil {
return nil, errors.New("binary bucket not configured; set GO_ECOSYSTEM_BINARY_BUCKET")
}
@@ -400,9 +389,9 @@
if err != nil {
return nil, err
}
- stats.scanMemory = response.Stats.ScanMemory
- stats.scanSeconds = response.Stats.ScanSeconds
- return response.Res.Vulns, nil
+ stats.ScanMemory = response.Stats.ScanMemory
+ stats.ScanSeconds = response.Stats.ScanSeconds
+ return response.Findings, nil
}
func (s *scanner) runGovulncheckSandbox(ctx context.Context, mode, arg string) (*govulncheck.SandboxResponse, error) {
@@ -422,19 +411,19 @@
return govulncheck.UnmarshalSandboxResponse(stdout)
}
-func (s *scanner) runGovulncheckScanInsecure(ctx context.Context, modulePath, version, inputPath, mode string, stats *scanStats) (_ []*govulncheckapi.Vuln, err error) {
+func (s *scanner) runGovulncheckScanInsecure(ctx context.Context, modulePath, version, inputPath, mode string, stats *govulncheck.ScanStats) (_ []*govulncheckapi.Finding, err error) {
if mode == ModeBinary {
return s.runBinaryScanInsecure(ctx, modulePath, version, inputPath, os.TempDir(), stats)
}
- vulns, err := s.runGovulncheckCmd("./...", inputPath, stats)
+ findings, err := govulncheck.RunGovulncheckCmd(s.govulncheckPath, ModeGovulncheck, inputPath, s.vulnDBDir, stats)
if err != nil {
return nil, err
}
- return vulns, nil
+ return findings, nil
}
-func (s *scanner) runBinaryScanInsecure(ctx context.Context, modulePath, version, binDir, tempDir string, stats *scanStats) ([]*govulncheckapi.Vuln, error) {
+func (s *scanner) runBinaryScanInsecure(ctx context.Context, modulePath, version, binDir, tempDir string, stats *govulncheck.ScanStats) ([]*govulncheckapi.Finding, error) {
if s.gcsBucket == nil {
return nil, errors.New("binary bucket not configured; set GO_ECOSYSTEM_BINARY_BUCKET")
}
@@ -447,32 +436,11 @@
if err := copyToLocalFile(localPathname, false, gcsPathname, gcsOpenFileFunc(ctx, s.gcsBucket)); err != nil {
return nil, err
}
- vulns, err := s.runGovulncheckCmd(localPathname, "", stats)
+ findings, err := govulncheck.RunGovulncheckCmd(s.govulncheckPath, ModeBinary, localPathname, s.vulnDBDir, stats)
if err != nil {
return nil, err
}
- return vulns, nil
-}
-
-func (s *scanner) runGovulncheckCmd(pattern, tempDir string, stats *scanStats) ([]*govulncheckapi.Vuln, error) {
- start := time.Now()
- govulncheckCmd := exec.Command(s.govulncheckPath, "-json", pattern)
- govulncheckCmd.Env = append(govulncheckCmd.Environ(), "GOVULNDB=file://"+s.vulnDBDir)
- govulncheckCmd.Dir = tempDir
- output, err := govulncheckCmd.CombinedOutput()
- if err != nil {
- if e := (&exec.ExitError{}); !errors.As(err, &e) || e.ProcessState.ExitCode() != 3 {
- return nil, fmt.Errorf("govulncheck error: err=%v out=%s", err, output)
- }
- }
- stats.scanSeconds = time.Since(start).Seconds()
- stats.scanMemory = uint64(govulncheckCmd.ProcessState.SysUsage().(*syscall.Rusage).Maxrss)
-
- res, err := govulncheck.UnmarshalGovulncheckResult(output)
- if err != nil {
- return nil, err
- }
- return res.Vulns, nil
+ return findings, nil
}
func isNoModulesSpecified(err error) bool {
diff --git a/internal/worker/govulncheck_scan_test.go b/internal/worker/govulncheck_scan_test.go
index 4ec3aaa..cb30677 100644
--- a/internal/worker/govulncheck_scan_test.go
+++ b/internal/worker/govulncheck_scan_test.go
@@ -118,8 +118,8 @@
s.gcsBucket = gcsClient.Bucket("go-ecosystem")
}
- stats := &scanStats{}
- vulns, err := s.runGovulncheckScanInsecure(ctx,
+ stats := &govulncheck.ScanStats{}
+ findings, err := s.runGovulncheckScanInsecure(ctx,
"golang.org/vuln", "v0.0.0",
tc.input, tc.mode, stats)
if err != nil {
@@ -127,19 +127,19 @@
}
wantID := "GO-2021-0113"
found := false
- for _, v := range vulns {
- if v.OSV.ID == wantID {
+ for _, v := range findings {
+ if v.OSV == wantID {
found = true
break
}
}
if !found {
- t.Errorf("want %s, did not find it in %d vulns", wantID, len(vulns))
+ t.Errorf("want %s, did not find it in %d vulns", wantID, len(findings))
}
- if got := stats.scanSeconds; got <= 0 {
+ if got := stats.ScanSeconds; got <= 0 {
t.Errorf("scan time not collected or negative: %v", got)
}
- if got := stats.scanMemory; got <= 0 {
+ if got := stats.ScanMemory; got <= 0 {
t.Errorf("scan memory not collected or negative: %v", got)
}
})
diff --git a/internal/worker/server.go b/internal/worker/server.go
index aa3efe3..33d842e 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -26,17 +26,15 @@
"golang.org/x/pkgsite-metrics/internal/observe"
"golang.org/x/pkgsite-metrics/internal/proxy"
"golang.org/x/pkgsite-metrics/internal/queue"
- vulnc "golang.org/x/vuln/client"
)
type Server struct {
- cfg *config.Config
- observer *observe.Observer
- bqClient *bigquery.Client
- vulndbClient vulnc.Client
- proxyClient *proxy.Client
- queue queue.Queue
- jobDB *jobs.DB
+ cfg *config.Config
+ observer *observe.Observer
+ bqClient *bigquery.Client
+ proxyClient *proxy.Client
+ queue queue.Queue
+ jobDB *jobs.DB
devMode bool
mu sync.Mutex
@@ -66,8 +64,6 @@
if err != nil {
return nil, err
}
- dbClient, err := vulnc.NewClient([]string{cfg.VulnDBURL}, vulnc.Options{})
- log.Debugf(ctx, "vulnc.NewClient returned err %v", err)
if err != nil {
return nil, err
}
@@ -86,13 +82,12 @@
}
}
s := &Server{
- cfg: cfg,
- bqClient: bq,
- vulndbClient: dbClient,
- queue: q,
- proxyClient: proxyClient,
- devMode: cfg.DevMode,
- jobDB: jdb,
+ cfg: cfg,
+ bqClient: bq,
+ queue: q,
+ proxyClient: proxyClient,
+ devMode: cfg.DevMode,
+ jobDB: jdb,
}
if cfg.ProjectID != "" && cfg.ServiceID != "" {
