blob: a71050c7fa393df317981991f8c09301c56a40ed [file] [log] [blame]
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package worker
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
"cloud.google.com/go/storage"
"golang.org/x/exp/event"
"golang.org/x/pkgsite-metrics/internal/bigquery"
"golang.org/x/pkgsite-metrics/internal/config"
"golang.org/x/pkgsite-metrics/internal/derrors"
"golang.org/x/pkgsite-metrics/internal/load"
"golang.org/x/pkgsite-metrics/internal/log"
"golang.org/x/pkgsite-metrics/internal/modules"
"golang.org/x/pkgsite-metrics/internal/proxy"
"golang.org/x/pkgsite-metrics/internal/sandbox"
"golang.org/x/pkgsite-metrics/internal/version"
ivulncheck "golang.org/x/pkgsite-metrics/internal/vulncheck"
vulnclient "golang.org/x/vuln/client"
"golang.org/x/vuln/exp/govulncheck"
"golang.org/x/vuln/vulncheck"
)
const (
// ModeImports only computes import-level analysis.
ModeImports string = "IMPORTS"
// ModeBinary runs vulncheck.Binary
ModeBinary string = "BINARY"
// ModeGovulncheck runs the govulncheck binary
ModeGovulncheck = "GOVULNCHECK"
)
// modes is a set of supported vulncheck modes
var modes = map[string]bool{
ModeImports: true,
ModeBinary: true,
ModeGovulncheck: true,
}
func IsValidVulncheckMode(mode string) bool {
return modes[mode]
}
// TODO(b/241402488): shouldSkip is the list of modules that we are not
// currently scanning due to previous issues that need investigation.
var shouldSkip = map[string]bool{}
var scanCounter = event.NewCounter("scans", &event.MetricOptions{Namespace: metricNamespace})
// path: /vulncheck/scan/MODULE_VERSION_SUFFIX?params
// See internal/vulncheck.ParseRequest for allowed path forms and query params.
func (h *VulncheckServer) handleScan(w http.ResponseWriter, r *http.Request) (err error) {
defer derrors.Wrap(&err, "handleScan")
defer func() {
scanCounter.Record(r.Context(), 1, event.Bool("success", err == nil))
}()
ctx := r.Context()
sreq, err := ivulncheck.ParseRequest(r, "/vulncheck/scan")
if err != nil {
return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
}
if sreq.Mode == "" {
sreq.Mode = ModeGovulncheck
}
if shouldSkip[sreq.Module] {
log.Infof(ctx, "skipping %s (module in shouldSkip list)", sreq.Path())
return nil
}
if err := h.readVulncheckWorkVersions(ctx); err != nil {
return err
}
scanner, err := newScanner(ctx, h)
if err != nil {
return err
}
// An explicit "insecure" query param overrides the default.
if sreq.Insecure {
scanner.insecure = sreq.Insecure
}
wv := h.storedWorkVersions[[2]string{sreq.Module, sreq.Version}]
if scanner.workVersion.Equal(wv) {
log.Infof(ctx, "skipping %s@%s (work version unchanged)", sreq.Module, sreq.Version)
return nil
}
log.Infof(ctx, "scanning: %s", sreq.Path())
return scanner.ScanModule(ctx, w, sreq)
}
func (h *VulncheckServer) readVulncheckWorkVersions(ctx context.Context) error {
h.mu.Lock()
defer h.mu.Unlock()
if h.storedWorkVersions != nil {
return nil
}
if h.bqClient == nil {
return nil
}
var err error
h.storedWorkVersions, err = ivulncheck.ReadWorkVersions(ctx, h.bqClient)
return err
}
// A scanner holds state for scanning modules.
type scanner struct {
proxyClient *proxy.Client
dbClient vulnclient.Client
bqClient *bigquery.Client
workVersion *ivulncheck.WorkVersion
goMemLimit uint64
gcsBucket *storage.BucketHandle
insecure bool
sbox *sandbox.Sandbox
}
func newScanner(ctx context.Context, h *VulncheckServer) (*scanner, error) {
workVersion, err := h.getWorkVersion(ctx)
if err != nil {
return nil, err
}
var bucket *storage.BucketHandle
if h.cfg.BinaryBucket != "" {
c, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
bucket = c.Bucket(h.cfg.BinaryBucket)
}
sbox := sandbox.New("/bundle")
sbox.Runsc = "/usr/local/bin/runsc"
return &scanner{
proxyClient: h.proxyClient,
bqClient: h.bqClient,
dbClient: h.vulndbClient,
workVersion: workVersion,
goMemLimit: parseGoMemLimit(os.Getenv("GOMEMLIMIT")),
gcsBucket: bucket,
insecure: h.cfg.Insecure,
sbox: sbox,
}, nil
}
type scanError struct {
err error
}
func (s scanError) Error() string {
return s.err.Error()
}
func (s scanError) Unwrap() error {
return s.err
}
func (s *scanner) ScanModule(ctx context.Context, w http.ResponseWriter, sreq *ivulncheck.Request) error {
if sreq.Module == "std" {
return nil // ignore the standard library
}
row := &ivulncheck.Result{
ModulePath: sreq.Module,
Suffix: sreq.Suffix,
WorkVersion: *s.workVersion,
}
// Scan the version.
log.Infof(ctx, "fetching proxy info: %s@%s", sreq.Module, sreq.Version)
info, err := s.proxyClient.Info(ctx, sreq.Module, sreq.Version)
if err != nil {
log.Errorf(ctx, err, "proxy error")
row.AddError(fmt.Errorf("%v: %w", err, derrors.ProxyError))
return nil
}
row.Version = info.Version
row.SortVersion = version.ForSorting(row.Version)
row.CommitTime = info.Time
row.ImportedBy = sreq.ImportedBy
row.VulnDBLastModified = s.workVersion.VulnDBLastModified
row.ScanMode = sreq.Mode
log.Infof(ctx, "scanning: %s", sreq.Path())
stats := &vulncheckStats{}
vulns, err := s.runScanModule(ctx, sreq.Module, info.Version, sreq.Suffix, sreq.Mode, stats)
row.ScanSeconds = stats.scanSeconds
row.ScanMemory = int64(stats.scanMemory)
row.PkgsMemory = int64(stats.pkgsMemory)
row.Workers = config.GetEnvInt("CLOUD_RUN_CONCURRENCY", "0", -1)
if err != nil {
// If an error occurred, wrap it accordingly
if isVulnDBConnection(err) {
err = fmt.Errorf("%v: %w", err, derrors.ScanModuleVulncheckDBConnectionError)
} else if !errors.Is(err, derrors.ScanModuleMemoryLimitExceeded) && sreq.Mode != ModeGovulncheck {
err = fmt.Errorf("%v: %w", err, derrors.ScanModuleVulncheckError)
}
row.AddError(err)
log.Infof(ctx, "scanner.runScanModule return error for %s (%v)", sreq.Path(), err)
} else {
row.Vulns = vulns
log.Infof(ctx, "scanner.runScanModule returned %d vulns: %s", len(vulns), sreq.Path())
}
return writeResult(ctx, sreq.Serve, w, s.bqClient, ivulncheck.TableName, row)
}
type vulncheckStats struct {
scanSeconds float64
scanMemory uint64
pkgsMemory uint64
}
// runScanModule fetches the module version from the proxy, and analyzes it for
// vulnerabilities.
func (s *scanner) runScanModule(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) (bvulns []*ivulncheck.Vuln, err error) {
err = doScan(ctx, modulePath, version, s.insecure, func() error {
if mode != ModeGovulncheck {
var vulns []*vulncheck.Vuln
if s.insecure {
vulns, err = s.runScanModuleInsecure(ctx, modulePath, version, binaryDir, mode, stats)
} else {
vulns, err = s.runScanModuleSandbox(ctx, modulePath, version, binaryDir, mode, stats)
}
if err != nil {
return err
}
for _, v := range vulns {
bvulns = append(bvulns, ivulncheck.ConvertVulncheckOutput(v))
}
} else { // Govulncheck mode
var vulns []*govulncheck.Vuln
if s.insecure {
vulns, err = s.runGovulncheckScanInsecure(ctx, modulePath, version, stats)
} else {
res, err := s.runGovulncheckScanSandbox(ctx, modulePath, version, stats)
if err != nil {
return err
}
vulns = res.Vulns
}
if err != nil {
return err
}
for _, v := range vulns {
bvulns = append(bvulns, ivulncheck.ConvertGovulncheckOutput(v)...)
}
}
return nil
})
return bvulns, err
}
func (s *scanner) runGovulncheckScanSandbox(ctx context.Context, modulePath, version string, stats *vulncheckStats) (*govulncheck.Result, error) {
sandboxDir := "/modules/" + modulePath + "@" + version
imageDir := "/bundle/rootfs" + sandboxDir
defer os.RemoveAll(imageDir)
log.Infof(ctx, "downloading %s@%s to %s", modulePath, version, imageDir)
if err := modules.Download(ctx, modulePath, version, imageDir, s.proxyClient, true); err != nil {
log.Debugf(ctx, "download error: %v (%[1]T)", err)
return nil, err
}
// Download all dependencies outside of the sandbox, but use the Go build
// cache inside the bundle.
log.Infof(ctx, "running go mod download")
cmd := exec.Command("go", "mod", "download")
cmd.Dir = imageDir
cmd.Env = append(cmd.Environ(),
"GOPROXY=https://proxy.golang.org",
"GOMODCACHE=/bundle/rootfs/"+sandboxGoModCache)
_, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("%w: 'go mod download' for %s@%s returned %s",
derrors.BadModule, modulePath, version, derrors.IncludeStderr(err))
}
log.Infof(ctx, "go mod download succeeded")
log.Infof(ctx, "%s@%s: running govulncheck in sandbox", modulePath, version)
govulncheckcmd := s.sbox.Command("/binaries/vulncheck_sandbox", ModeGovulncheck, sandboxDir)
stdout, err := govulncheckcmd.Output()
if err != nil {
return nil, errors.New(derrors.IncludeStderr(err))
}
return unmarshalGovulncheckOutput(stdout)
}
func (s *scanner) runScanModuleSandbox(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) ([]*vulncheck.Vuln, error) {
var (
res *vulncheck.Result
err error
)
if mode == ModeBinary {
res, err = s.runBinaryScanSandbox(ctx, modulePath, version, binaryDir, stats)
} else {
res, err = s.runSourceScanSandbox(ctx, modulePath, version, mode, stats)
}
log.Debugf(ctx, "runScanModuleSandbox %s@%s bin %s, mode %s: got %+v, %v", modulePath, version, binaryDir, mode, res, err)
if err != nil {
return nil, err
}
return res.Vulns, nil
}
func (s *scanner) runSourceScanSandbox(ctx context.Context, modulePath, version, mode string, stats *vulncheckStats) (*vulncheck.Result, error) {
stdout, err := runSourceScanSandbox(ctx, modulePath, version, mode, s.proxyClient, s.sbox)
if err != nil {
return nil, err
}
return unmarshalVulncheckOutput(stdout)
}
// Inside the sandbox, the user is root and their $HOME directory is /root.
const (
// The Go module cache resides in its default location, $HOME/go/pkg/mod.
sandboxGoModCache = "root/go/pkg/mod"
// The Go cache resides in its default location, $HOME/.cache/go-build.
sandboxGoCache = "root/.cache/go-build"
)
func runSourceScanSandbox(ctx context.Context, modulePath, version, mode string, proxyClient *proxy.Client, sbox *sandbox.Sandbox) ([]byte, error) {
sandboxDir, cleanup, err := downloadModuleSandbox(ctx, modulePath, version, proxyClient)
if err != nil {
return nil, err
}
defer cleanup()
log.Infof(ctx, "%s@%s: running vulncheck in sandbox", modulePath, version)
stdout, err := sbox.Command("/binaries/vulncheck_sandbox", mode, sandboxDir).Output()
if err != nil {
return nil, errors.New(derrors.IncludeStderr(err))
}
return stdout, nil
}
func downloadModuleSandbox(ctx context.Context, modulePath, version string, proxyClient *proxy.Client) (string, func(), error) {
sandboxDir := "/modules/" + modulePath + "@" + version
imageDir := "/bundle/rootfs" + sandboxDir
log.Infof(ctx, "downloading %s@%s to %s", modulePath, version, imageDir)
if err := modules.Download(ctx, modulePath, version, imageDir, proxyClient, true); err != nil {
log.Debugf(ctx, "download error: %v (%[1]T)", err)
return "", nil, err
}
// Download all dependencies outside of the sandbox, but use the Go build
// cache ("/bundle/rootfs/" + sandboxGoCache) inside the bundle.
log.Infof(ctx, "running go mod download")
cmd := exec.Command("go", "mod", "download")
cmd.Dir = imageDir
cmd.Env = append(cmd.Environ(),
"GOPROXY=https://proxy.golang.org",
"GOMODCACHE=/bundle/rootfs/"+sandboxGoModCache)
_, err := cmd.Output()
if err != nil {
return "", nil, fmt.Errorf("%w: 'go mod download' for %s@%s returned %s",
derrors.BadModule, modulePath, version, derrors.IncludeStderr(err))
}
log.Infof(ctx, "go mod download succeeded")
return sandboxDir, func() { os.RemoveAll(imageDir) }, nil
}
func (s *scanner) runBinaryScanSandbox(ctx context.Context, modulePath, version, binDir string, stats *vulncheckStats) (*vulncheck.Result, error) {
if s.gcsBucket == nil {
return nil, errors.New("binary bucket not configured; set GO_ECOSYSTEM_BINARY_BUCKET")
}
// Copy the binary from GCS to the local disk, because vulncheck.Binary
// requires a ReaderAt and GCS doesn't provide that.
gcsPathname := fmt.Sprintf("%s/%s@%s/%s", binaryDir, modulePath, version, binDir)
const destDir = "/bundle/rootfs/binaries"
log.Debug(ctx, "copying",
"from", gcsPathname,
"to", destDir,
"module", modulePath, "version", version,
"dir", binDir)
destf, err := os.CreateTemp(destDir, "vulncheck-binary-")
if err != nil {
return nil, err
}
defer os.Remove(destf.Name())
rc, err := s.gcsBucket.Object(gcsPathname).NewReader(ctx)
if err != nil {
return nil, err
}
defer rc.Close()
if err := copyAndClose(destf, rc); err != nil {
return nil, err
}
log.Infof(ctx, "%s@%s/%s: running vulncheck in sandbox on %s", modulePath, version, binDir, destf.Name())
stdout, err := s.sbox.Command("/binaries/vulncheck_sandbox", ModeBinary, destf.Name()).Output()
if err != nil {
return nil, errors.New(derrors.IncludeStderr(err))
}
return unmarshalVulncheckOutput(stdout)
}
func unmarshalVulncheckOutput(output []byte) (*vulncheck.Result, error) {
var e struct {
Error string
}
if err := json.Unmarshal(output, &e); err != nil {
return nil, err
}
if e.Error != "" {
return nil, errors.New(e.Error)
}
var res vulncheck.Result
if err := json.Unmarshal(output, &res); err != nil {
return nil, err
}
return &res, nil
}
func unmarshalGovulncheckOutput(output []byte) (*govulncheck.Result, error) {
var e struct {
Error string
}
if err := json.Unmarshal(output, &e); err != nil {
return nil, err
}
if e.Error != "" {
return nil, errors.New(e.Error)
}
var res govulncheck.Result
if err := json.Unmarshal(output, &res); err != nil {
return nil, err
}
return &res, nil
}
func (s *scanner) runGovulncheckScanInsecure(ctx context.Context, modulePath, version string, stats *vulncheckStats) (_ []*govulncheck.Vuln, err error) {
tempDir, err := os.MkdirTemp("", "runGovulncheckScan")
if err != nil {
return nil, err
}
defer func() {
err1 := os.RemoveAll(tempDir)
if err == nil {
err = err1
}
}()
log.Debugf(ctx, "fetching module zip: %s@%s", modulePath, version)
if err := modules.Download(ctx, modulePath, version, tempDir, s.proxyClient, true); err != nil {
return nil, err
}
start := time.Now()
vulns, err := runGovulncheckCmd(ctx, modulePath, tempDir, stats)
if err != nil {
return nil, err
}
stats.scanSeconds = time.Since(start).Seconds()
return vulns, nil
}
func runGovulncheckCmd(ctx context.Context, modulePath, tempDir string, stats *vulncheckStats) ([]*govulncheck.Vuln, error) {
govulncheckName := "/bundle/rootfs/binaries/govulncheck"
if !fileExists(govulncheckName) {
govulncheckName = "govulncheck"
}
govulncheckCmd := exec.Command(govulncheckName, "-json", "./...")
govulncheckCmd.Dir = tempDir
output, err := govulncheckCmd.Output()
if err != nil {
return nil, err
}
res, err := unmarshalGovulncheckOutput(output)
if err != nil {
return nil, err
}
return res.Vulns, nil
}
func (s *scanner) runScanModuleInsecure(ctx context.Context, modulePath, version, binaryDir, mode string, stats *vulncheckStats) (_ []*vulncheck.Vuln, err error) {
tempDir, err := os.MkdirTemp("", "runScanModule")
if err != nil {
return nil, err
}
defer func() {
err1 := os.RemoveAll(tempDir)
if err == nil {
err = err1
}
}()
if mode == ModeBinary {
return s.runBinaryScanInsecure(ctx, modulePath, version, binaryDir, tempDir, stats)
}
return s.runSourceScanInsecure(ctx, modulePath, version, mode, tempDir, stats)
}
func (s *scanner) runSourceScanInsecure(ctx context.Context, modulePath, version, mode, tempDir string, stats *vulncheckStats) ([]*vulncheck.Vuln, error) {
log.Debugf(ctx, "fetching module zip: %s@%s", modulePath, version)
if err := modules.Download(ctx, modulePath, version, tempDir, s.proxyClient, true); err != nil {
return nil, err
}
cctx, cancel := context.WithCancel(ctx)
defer cancel()
cfg := load.DefaultConfig()
cfg.Dir = tempDir // filepath.Join(dir, modulePath+"@"+version,
cfg.Context = cctx
runtime.GC()
// current memory not related to core (go)vulncheck operations.
preScanMemory := currHeapUsage()
log.Debugf(ctx, "loading packages: %s@%s", modulePath, version)
pkgs, pkgErrors, err := load.Packages(cfg, "./...")
if err == nil && len(pkgErrors) > 0 {
err = fmt.Errorf("%v", pkgErrors)
}
if err != nil {
return nil, err
}
stats.pkgsMemory = memSubtract(currHeapUsage(), preScanMemory)
// Run vulncheck.Source and collect results.
start := time.Now()
vcfg := vulncheckConfig(s.dbClient, mode)
res, peakMem, err := s.runWithMemoryMonitor(ctx, func() (*vulncheck.Result, error) {
log.Debugf(ctx, "running vulncheck.Source: %s@%s", modulePath, version)
res, err := vulncheck.Source(cctx, vulncheck.Convert(pkgs), vcfg)
log.Debugf(ctx, "completed run for vulncheck.Source: %s@%s, err=%v", modulePath, version, err)
if err != nil {
return res, err
}
return res, nil
})
// scanMemory is peak heap memory used during vulncheck + pkgs.
// We subtract any memory not related to these core (go)vulncheck
// operations.
stats.scanMemory = memSubtract(peakMem, preScanMemory)
// scanSeconds is the time it took for vulncheck.Source to run.
// We want to know this information regardless of whether an error
// occurred.
stats.scanSeconds = time.Since(start).Seconds()
if err != nil {
return nil, err
}
return res.Vulns, nil
}
// runWithMemoryMonitor runs f in a goroutine with its memory tracked.
// It returns f's peak memory usage.
func (s *scanner) runWithMemoryMonitor(ctx context.Context, f func() (*vulncheck.Result, error)) (res *vulncheck.Result, mem uint64, err error) {
cctx, cancel := context.WithCancel(ctx)
monitor := newMemMonitor(s.goMemLimit, cancel)
type sr struct {
res *vulncheck.Result
err error
}
srchan := make(chan sr)
go func() {
res, err := f()
srchan <- sr{res, err}
}()
select {
case r := <-srchan:
res = r.res
err = r.err
case <-cctx.Done():
err = derrors.ScanModuleMemoryLimitExceeded
}
return res, monitor.stop(), err
}
func (s *scanner) runBinaryScanInsecure(ctx context.Context, modulePath, version, binDir, tempDir string, stats *vulncheckStats) ([]*vulncheck.Vuln, error) {
if s.gcsBucket == nil {
return nil, errors.New("binary bucket not configured; set GO_ECOSYSTEM_BINARY_BUCKET")
}
// Copy the binary from GCS to the local disk, because vulncheck.Binary
// requires a ReaderAt and GCS doesn't provide that.
gcsPathname := fmt.Sprintf("%s/%s@%s/%s", binaryDir, modulePath, version, binDir)
log.Debug(ctx, "copying to temp dir",
"from", gcsPathname, "module", modulePath, "version", version, "dir", binDir)
localPathname := filepath.Join(tempDir, "binary")
if err := copyToLocalFile(localPathname, false, gcsPathname, gcsOpenFileFunc(ctx, s.gcsBucket)); err != nil {
return nil, err
}
binaryFile, err := os.Open(localPathname)
if err != nil {
return nil, err
}
defer binaryFile.Close()
start := time.Now()
runtime.GC()
// current memory not related to core (go)vulncheck operations.
preScanMemory := currHeapUsage()
log.Debugf(ctx, "running vulncheck.Binary: %s", gcsPathname)
res, err := vulncheck.Binary(ctx, binaryFile, vulncheckConfig(s.dbClient, ModeBinary))
log.Debugf(ctx, "completed run for vulncheck.Binary: %s, err=%v", gcsPathname, err)
stats.scanSeconds = time.Since(start).Seconds()
// TODO: measure peak usage?
stats.scanMemory = memSubtract(currHeapUsage(), preScanMemory)
if err != nil {
return nil, err
}
return res.Vulns, nil
}
func copyFromGCSToWriter(ctx context.Context, w io.Writer, bucket *storage.BucketHandle, srcPath string) error {
gcsReader, err := bucket.Object(srcPath).NewReader(ctx)
if err != nil {
return err
}
_, err = io.Copy(w, gcsReader)
return err
}
func isVulnDBConnection(err error) bool {
s := err.Error()
return strings.Contains(s, "https://vuln.go.dev") &&
strings.Contains(s, "connection")
}
func vulncheckConfig(dbClient vulnclient.Client, mode string) *vulncheck.Config {
cfg := &vulncheck.Config{Client: dbClient}
switch mode {
case ModeImports:
cfg.ImportsOnly = true
default:
cfg.ImportsOnly = false
}
return cfg
}
// currHeapUsage computes currently allocate heap bytes.
func currHeapUsage() uint64 {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
return stats.Alloc
}
// memSubtract subtracts memory usage m2 from m1, returning
// 0 if the result is negative.
func memSubtract(m1, m2 uint64) uint64 {
if m1 <= m2 {
return 0
}
return m1 - m2
}
// parseGoMemLimit parses the GOMEMLIMIT environment variable.
// It returns 0 if the variable isn't set or its value is malformed.
func parseGoMemLimit(s string) uint64 {
if len(s) < 2 {
return 0
}
m := uint64(1)
if s[len(s)-1] == 'i' {
switch s[len(s)-2] {
case 'K':
m = 1024
case 'M':
m = 1024 * 1024
case 'G':
m = 1024 * 1024 * 1024
default:
return 0
}
s = s[:len(s)-2]
}
v, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return 0
}
return v * m
}
// fileExists checks if file path exists. Returns true
// if the file exists or it cannot prove that it does
// not exist. Otherwise, returns false.
func fileExists(file string) bool {
if _, err := os.Stat(file); err == nil {
return true
} else if errors.Is(err, os.ErrNotExist) {
return false
}
// Conservatively return true if os.Stat fails
// for some other reason.
return true
}