internal/worker: store govulncheck rows in a single transaction
The idea is to hopefully reduce the number of update operations
to the table, making it less likely to hit table.write quotas.
Change-Id: I7de75020b247c82194202396b5c67ee034bffee4
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/523715
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Maceo Thompson <maceothompson@google.com>
diff --git a/internal/worker/govulncheck_scan.go b/internal/worker/govulncheck_scan.go
index 206b2f6..c125868 100644
--- a/internal/worker/govulncheck_scan.go
+++ b/internal/worker/govulncheck_scan.go
@@ -355,25 +355,22 @@
}
log.Infof(ctx, "scanner.runScanModule returned %d vulns for %s: row.Vulns=%d err=%v", len(vulns), sreq.Path(), len(row.Vulns), err)
- if err := writeResult(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, row); err != nil {
- return err
+ rows := []bigquery.Row{row}
+ if sreq.Mode == ModeGovulncheck {
+ // For ModeGovulncheck, add the copy of row and report
+ // each vulnerability as imported. We set the performance
+ // numbers to 0 since we don't actually perform a scan
+ // at the level of import chains. Also makes a copy if
+ // the original row has an error and no vulns.
+ impRow := *row
+ impRow.ScanMode = modeImports
+ impRow.ScanSeconds = 0
+ impRow.ScanMemory = 0
+ impRow.Vulns = vulnsForMode(vulns, modeImports)
+ log.Infof(ctx, "scanner.runScanModule also storing imports vulns for %s: row.Vulns=%d", sreq.Path(), len(impRow.Vulns))
+ rows = append(rows, &impRow)
}
-
- if sreq.Mode != ModeGovulncheck {
- return nil
- }
- // For ModeGovulncheck, add the copy of row and report
- // each vulnerability as imported. We set the performance
- // numbers to 0 since we don't actually perform a scan
- // at the level of import chains. Also makes a copy if
- // the original row has an error and no vulns.
- impRow := *row
- impRow.ScanMode = modeImports
- impRow.ScanSeconds = 0
- impRow.ScanMemory = 0
- impRow.Vulns = vulnsForMode(vulns, modeImports)
- log.Infof(ctx, "scanner.runScanModule also storing imports vulns for %s: row.Vulns=%d", sreq.Path(), len(impRow.Vulns))
- return writeResult(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, &impRow)
+ return writeResults(ctx, sreq.Serve, w, s.bqClient, govulncheck.TableName, rows)
}
// vulnsForMode returns vulns that make sense to report for
diff --git a/internal/worker/scan.go b/internal/worker/scan.go
index 65e2f12..52306ef 100644
--- a/internal/worker/scan.go
+++ b/internal/worker/scan.go
@@ -168,6 +168,7 @@
return client.Upload(ctx, table, row)
}
+// writeResults is like writeResult but stores the rows in a single transaction.
func writeResults(ctx context.Context, serve bool, w http.ResponseWriter, client *bigquery.Client, table string, rows []bigquery.Row) (err error) {
defer derrors.Wrap(&err, "writeResults")
