internal/worker: add query param to serve vulncheck/scan
Add a "serve" query param to the vulncheck/scan endpoint.
A URL with "serve=true" will scan as usual, but write the
result back to the client as JSON instead of uploading it
to BigQuery.
This makes it easier to test scanning without writing any data.
Previously we had the test-vulncheck-sandbox endpoint for this,
but it followed different logic so it wasn't as good a test.
Change-Id: If7037ff4fe3a11a082ac3c03c752a16339031c1e
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/471235
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
Reviewed-by: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/Makefile b/Makefile
index 3a34eb5..bff1134 100644
--- a/Makefile
+++ b/Makefile
@@ -58,7 +58,8 @@
# Test by scanning a small module.
test: docker-run-bg
- curl -s 'http://localhost:8080/test-vulncheck-sandbox/github.com/fossas/fossa-cli@v1.1.10?importedby=1' | grep GO-2020-0016
+
+ curl -s 'http://localhost:8080/vulncheck/scan/github.com/fossas/fossa-cli@v1.1.10?importedby=1&serve=true' | grep GO-2020-0016
docker container stop `cat /tmp/ecosystem-docker-container-id`
clean:
diff --git a/internal/log/line_handler.go b/internal/log/line_handler.go
index d5598f8..65a48b4 100644
--- a/internal/log/line_handler.go
+++ b/internal/log/line_handler.go
@@ -42,7 +42,7 @@
func (h *LineHandler) Handle(ctx context.Context, r slog.Record) error {
var buf bytes.Buffer
- fmt.Fprintf(&buf, "%s %s %s", r.Time.Format("2006/01/02 15:04:05"), r.Level, r.Message)
+ fmt.Fprintf(&buf, "%s %-5s %s", r.Time.Format("2006/01/02 15:04:05"), r.Level, r.Message)
prefix := ""
for ga := h.gora; ga != nil; ga = ga.Next {
diff --git a/internal/worker/server.go b/internal/worker/server.go
index 23262c2..c1f6f93 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -134,7 +134,6 @@
return nil, err
}
- s.handle("/test-vulncheck-sandbox/", s.handleTestVulncheckSandbox)
s.handle("/test-db", s.handleTestDB)
return s, nil
diff --git a/internal/worker/vulncheck_scan.go b/internal/worker/vulncheck_scan.go
index 53462b1..7b664c5 100644
--- a/internal/worker/vulncheck_scan.go
+++ b/internal/worker/vulncheck_scan.go
@@ -72,11 +72,8 @@
var scanCounter = event.NewCounter("scans", &event.MetricOptions{Namespace: metricNamespace})
-// path: /vulncheck/scan/MODULE_VERSION_SUFFIX
-// See scan.ParseRequest for allowed forms.
-// Query params:
-// - mode: type of analysis to run; see [modes]
-// - importedby: number of importers
+// path: /vulncheck/scan/MODULE_VERSION_SUFFIX?params
+// See parseVulncheckRequest for allowed path forms and query params.
func (h *VulncheckServer) handleScan(w http.ResponseWriter, r *http.Request) (err error) {
defer derrors.Wrap(&err, "handleScan")
@@ -104,8 +101,7 @@
return err
}
// An explicit "insecure" query param overrides the default.
- // This is temporary, for testing running in a sandbox.
- if r.FormValue("insecure") != "" {
+ if sreq.Insecure {
scanner.insecure = sreq.Insecure
}
wv := h.storedWorkVersions[[2]string{sreq.Module, sreq.Version}]
@@ -115,9 +111,7 @@
}
log.Infof(ctx, "scanning: %s", sreq.Path())
- scanner.ScanModule(ctx, sreq)
- log.Infof(ctx, "fetched and updated %s", sreq.Path())
- return nil
+ return scanner.ScanModule(ctx, w, sreq)
}
func (h *VulncheckServer) readVulncheckWorkVersions(ctx context.Context) error {
@@ -185,9 +179,9 @@
return s.err
}
-func (s *scanner) ScanModule(ctx context.Context, sreq *vulncheckRequest) {
+func (s *scanner) ScanModule(ctx context.Context, w http.ResponseWriter, sreq *vulncheckRequest) error {
if sreq.Module == "std" {
- return // ignore the standard library
+ return nil // ignore the standard library
}
row := &bigquery.VulnResult{
ModulePath: sreq.Module,
@@ -200,7 +194,7 @@
if err != nil {
log.Errorf(ctx, err, "proxy error")
row.AddError(fmt.Errorf("%v: %w", err, derrors.ProxyError))
- return
+ return nil
}
row.Version = info.Version
row.SortVersion = version.ForSorting(row.Version)
@@ -223,7 +217,15 @@
row.Vulns = vulns
log.Infof(ctx, "scanner.runScanModule returned %d vulns: %s", len(vulns), sreq.Path())
}
- if s.bqClient == nil {
+ if sreq.Serve {
+ // Write the result to the client instead of uploading to BigQuery.
+ log.Infof(ctx, "serving result to client")
+ data, err := json.MarshalIndent(row, "", " ")
+ if err != nil {
+ return fmt.Errorf("marshaling result: %w", err)
+ }
+ w.Write(data)
+ } else if s.bqClient == nil {
log.Infof(ctx, "bigquery disabled, not uploading")
} else {
log.Infof(ctx, "uploading to bigquery: %s", sreq.Path())
@@ -235,6 +237,7 @@
log.Errorf(ctx, err, "bq.Upload for %s", sreq.Path())
}
}
+ return nil
}
type vulncheckStats struct {
@@ -763,24 +766,6 @@
return nil
}
-// Test running vulncheck in the sandbox.
-// This runs a scan but returns the resulting JSON instead of writing it to BigQuery.
-func (s *Server) handleTestVulncheckSandbox(w http.ResponseWriter, r *http.Request) error {
- ctx := r.Context()
- sreq, err := parseVulncheckRequest(r, "/test-vulncheck-sandbox")
- if err != nil {
- return fmt.Errorf("%w: %v", derrors.InvalidArgument, err)
- }
- sbox := sandbox.New("/bundle")
- sbox.Runsc = "/usr/local/bin/runsc"
- out, err := runSourceScanSandbox(ctx, sreq.Module, sreq.Version, ModeVTA, s.proxyClient, sbox)
- if err != nil {
- return err
- }
- _, err = w.Write(out)
- return err
-}
-
// vulncheckRequest contains information passed
// to a scan endpoint.
type vulncheckRequest struct {
@@ -793,6 +778,7 @@
ImportedBy int // imported-by count
Mode string // vulncheck mode (VTA, etc)
Insecure bool // if true, run outside sandbox
+ Serve bool // serve results back to client instead of writing them to BigQuery
}
// These methods implement queue.Task.
