internal/worker: use batch upload for compare mode
This is expected to resolve some bigquery "too much write" errors and it
refactors some code.
Change-Id: I69d8f4fdf5b449773bc37b98aaa2042cb971952d
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/521055
Reviewed-by: Maceo Thompson <maceothompson@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/internal/worker/govulncheck_scan.go b/internal/worker/govulncheck_scan.go
index dd16d14..8f49a85 100644
--- a/internal/worker/govulncheck_scan.go
+++ b/internal/worker/govulncheck_scan.go
@@ -229,6 +229,8 @@
return nil
}
log.Infof(ctx, "scanner.runGovulncheckCompare found %d compilable binaries in %s:", len(response.FindingsForMod), sreq.Path())
+
+ var rows []bigquery.Row
for pkg, results := range response.FindingsForMod {
if results.Error != nil {
// Just log error if binary failed to build. Otherwise, we'd have
@@ -239,17 +241,13 @@
binRow := createComparisonRow(pkg, &results.BinaryResults, baseRow, ModeBinary)
srcRow := createComparisonRow(pkg, &results.SourceResults, baseRow, ModeGovulncheck)
-
log.Infof(ctx, "found %d vulns in binary mode and %d vulns in source mode for package %s (module: %s)", len(binRow.Vulns), len(srcRow.Vulns), pkg, sreq.Path())
-
- if err := writeResult(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, binRow); err != nil {
- return err
- }
- if err := writeResult(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, srcRow); err != nil {
- return err
- }
+ rows = append(rows, binRow, srcRow)
}
+ if len(rows) > 0 {
+ return writeResults(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, rows)
+ }
return nil
}
diff --git a/internal/worker/scan.go b/internal/worker/scan.go
index 06f1186..caec509 100644
--- a/internal/worker/scan.go
+++ b/internal/worker/scan.go
@@ -158,16 +158,7 @@
if serve {
// Write the result to the client instead of uploading to BigQuery.
- log.Infof(ctx, "serving result to client")
- data, err := json.MarshalIndent(row, "", " ")
- if err != nil {
- return fmt.Errorf("marshaling result: %w", err)
- }
- _, err = w.Write(data)
- if err != nil {
- log.Errorf(ctx, err, "writing to client")
- }
- return nil // No point serving an error, the write already happened.
+ return serveJSON(ctx, row, w)
}
// Upload to BigQuery.
if client == nil {
@@ -177,6 +168,34 @@
return client.Upload(ctx, table, row)
}
+func writeResults(ctx context.Context, serve bool, w http.ResponseWriter, client *bigquery.Client, table string, rows []bigquery.Row) (err error) {
+ defer derrors.Wrap(&err, "writeResults")
+
+ if serve {
+ // Write the results to the client instead of uploading to BigQuery.
+ return serveJSON(ctx, rows, w)
+ }
+ // Upload to BigQuery.
+ if client == nil {
+ log.Infof(ctx, "bigquery disabled, not uploading")
+ return nil
+ }
+ return bigquery.UploadMany(ctx, client, table, rows, 0)
+}
+
+func serveJSON(ctx context.Context, content interface{}, w http.ResponseWriter) error {
+ log.Infof(ctx, "serving result to client")
+ data, err := json.MarshalIndent(content, "", " ")
+ if err != nil {
+ return fmt.Errorf("marshaling result: %w", err)
+ }
+ _, err = w.Write(data)
+ if err != nil {
+ log.Errorf(ctx, err, "writing to client")
+ }
+ return nil // No point serving an error, the write already happened.
+}
+
type openFileFunc func(filename string) (io.ReadCloser, error)
// copyToLocalFile opens destPath for writing locally, making it executable if specified.
