internal/worker: reduce skipped-directory log spam
Change-Id: I0e46cc0069d9905d72ce464ca3e24c378152fa9b
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/376995
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Damien Neil <dneil@google.com>
diff --git a/internal/worker/update.go b/internal/worker/update.go
index f48f4d4..f2bebd0 100644
--- a/internal/worker/update.go
+++ b/internal/worker/update.go
@@ -35,6 +35,7 @@
}
type updateStats struct {
+ skipped bool // directory skipped because hashes match
numProcessed, numAdded, numModified int
}
@@ -108,6 +109,8 @@
return ur, err
}
+ var skippedDirs []string
+ const logSkippedEvery = 20 // Log a message every this many skipped directories.
for _, dirFiles := range filesByDir {
stats, err := u.updateDirectory(ctx, dirFiles)
// Change the CommitUpdateRecord in the Store to reflect the results of the directory update.
@@ -117,6 +120,14 @@
return ur, fmt.Errorf("update failed with %w, could not set update record: %v", err, err2)
}
}
+ if stats.skipped {
+ skippedDirs = append(skippedDirs, dirFiles[0].DirPath)
+ if len(skippedDirs) > logSkippedEvery {
+ log.Infof(ctx, "skipping directory %s and %d others because the hashes match",
+ skippedDirs[0], len(skippedDirs)-1)
+ skippedDirs = nil
+ }
+ }
ur.NumProcessed += stats.numProcessed
ur.NumAdded += stats.numAdded
ur.NumModified += stats.numModified
@@ -144,8 +155,7 @@
return updateStats{}, err
}
if dirHash == dbHash {
- log.Infof(ctx, "skipping directory %s because the hashes match", dirPath)
- return updateStats{}, nil
+ return updateStats{skipped: true}, nil
}
// Set the hash to something that can't match, until we fully process this directory.
if err := u.st.SetDirectoryHash(ctx, dirPath, "in progress"); err != nil {
