internal/worker: add/remove negative prefix patterns
Add several negative patterns discovered while running an update.
Remove a couple of patterns that were too general.
Also, add a Debug log level and make some log messages be Debug.
Change-Id: I93079054a53ef7de62893b4449466196d5588d0f
Reviewed-on: https://go-review.googlesource.com/c/vuln/+/370834
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index 313831a..f2a8eda 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -18,6 +18,7 @@
"strings"
"text/tabwriter"
+ "golang.org/x/exp/event"
"golang.org/x/vuln/internal/gitrepo"
"golang.org/x/vuln/internal/worker"
"golang.org/x/vuln/internal/worker/log"
@@ -77,6 +78,11 @@
}
ctx := log.WithLineLogger(context.Background())
+ log.Info(ctx, "config",
+ event.String("Project", cfg.Project),
+ event.String("Namespace", cfg.Namespace),
+ event.String("IssueRepo", cfg.IssueRepo))
+
var err error
cfg.Store, err = store.NewFireStore(ctx, cfg.Project, cfg.Namespace)
if err != nil {
diff --git a/internal/worker/log/log.go b/internal/worker/log/log.go
index 3b02053..de0883c 100644
--- a/internal/worker/log/log.go
+++ b/internal/worker/log/log.go
@@ -59,6 +59,11 @@
return ctx
}
+// Debug emits one log event at the Debug severity.
+func Debug(ctx context.Context, message string, labels ...event.Label) {
+ event.Log(ctx, message, append(labels, severity.Debug.Label())...)
+}
+
// Info emits one log event at the Info severity.
func Info(ctx context.Context, message string, labels ...event.Label) {
event.Log(ctx, message, append(labels, severity.Info.Label())...)
@@ -74,6 +79,11 @@
event.Log(ctx, message, append(labels, severity.Error.Label())...)
}
+// Debugf logs a formatted message with no labels at the Debug severity.
+func Debugf(ctx context.Context, format string, args ...interface{}) {
+ Debug(ctx, fmt.Sprintf(format, args...))
+}
+
// Infof logs a formatted message with no labels at the Info severity.
func Infof(ctx context.Context, format string, args ...interface{}) {
Info(ctx, fmt.Sprintf(format, args...))
diff --git a/internal/worker/paths.go b/internal/worker/paths.go
index dc5f4cd..7f2c7eb 100644
--- a/internal/worker/paths.go
+++ b/internal/worker/paths.go
@@ -40,32 +40,42 @@
"*.blogspot.com",
"*.blogspot.dk",
"*.readthedocs.org",
+ "*.slashdot.org",
+ "advisories.mageia.org",
"archives.neohapsis.com",
- "archives.neohapsis.com/archives/bugtraq",
- "blog.*",
+ "arstechnica.com/security",
"blog.python.org",
"blogs.oracle.com",
"blogs.technet.com",
"bugs.*",
"bugzilla.*",
+ "cert.uni-stuttgart.de/archive",
+ "community.rapid7.com/community/*/blog",
"cr.yp.to/talks",
"crbug.com",
+ "dev2dev.bea.com/pub/advisory",
"developer.mozilla.org/docs",
"developer.mozilla.org/en-US/docs",
"docs.google.com",
"docs.microsoft.com",
+ "downloads.securityfocus.com/vulnerabilities",
"drupal.org/node",
"erpscan.com/advisories",
"exchange.xforce.ibmcloud.com",
+ "fedoranews.org",
+ "ftp.caldera.com/pub/security",
+ "ftp.netbsd.org/pub",
+ "ftp.sco.com/pub",
"github.com/*/*/blob",
"github.com/*/*/commit",
"github.com/*/*/issues",
- "github.com/torvalds/linux/commit",
"groups.google.com",
"helpx.adobe.com/security",
"hg.openjdk.java.net",
"ics-cert.us-cert.gov",
"issues.apache.org",
+ "issues.rpath.com",
+ "java.net",
"jira.*",
"jvn.jp",
"jvndb.jvn.jp",
@@ -74,19 +84,26 @@
"lists.*/archive",
"lists.*/archives",
"lists.*/pipermail",
+ "lists.apache.org",
+ "lists.apple.com",
+ "lists.debian.org",
+ "lists.mysql.com",
"lists.opensuse.org",
"lists.ubuntu.com",
"mail-archives.*",
+ "mail.*.org/archive",
"mail.*.org/archives",
"mail.*/pipermail",
"mailman.*.org/archives",
"mailman.*.org/pipermail",
"nodesecurity.io/advisories",
+ "online.securityfocus.com/advisories",
"openwall.com/lists",
"oss.oracle.com/pipermail",
"osvdb.org",
"owncloud.org/about/security",
"packetstormsecurity.com/files",
+ "patches.sgi.com/support/free/security/advisories",
"plus.google.com",
"puppetlabs.com/security",
"raw.github.com",
@@ -94,46 +111,77 @@
"seclists.org",
"secunia.com/advisories",
"secunia.com/secunia_research",
+ "security.e-matters.de/advisories",
"security.gentoo.org/glsa",
+ "securityreason.com/securityalert",
+ "securityreason.com/securityalert/",
+ "securityresponse.symantec.com",
+ "securitytracker.com/alerts",
"service.sap.com",
"subversion.apache.org/security",
- "support.*",
"technet.microsoft.com/en-us/security",
"technet.microsoft.com/security",
"tools.cisco.com/security/center",
"twitter.com",
"ubuntu.com/usn",
- "weblog.*",
+ "usn.ubuntu.com",
+ "www.adobe.com/support",
"www.adobe.com/support/security",
+ "www.atstake.com/research/advisories",
"www.bugzilla.org/security",
+ "www.cert.org/advisories",
+ "www.ciac.org/ciac/bulletins",
+ "www.cisco.com/warp/public/707",
"www.coresecurity.com/advisories",
"www.debian.org/security",
+ "www.derkeiler.com/Mailing-Lists",
"www.drupal.org/node",
"www.exploit-db.com",
+ "www.gentoo.org/security",
"www.htbridge.com/advisory",
"www.ibm.com/developerworks/java",
+ "www.iss.net/security_center",
"www.kb.cert.org",
+ "www.kde.org/info/security",
+ "www.kernel.org/pub",
"www.kernel.org/pub/linux/kernel/v3*/ChangeLog*",
+ "www.linux-mandrake.com/en/security",
+ "www.linuxsecurity.com/advisories",
+ "www.microsoft.com/technet/security",
"www.mozilla.org/security",
+ "www.netvigilance.com/advisory*",
+ "www.novell.com/linux/security",
"www.openwall.com/lists",
"www.oracle.com/technetwork",
"www.osvdb.org",
"www.phpmyadmin.net/home_page/security",
"www.portcullis-security.com/security-research-and-downloads",
"www.postgresql.org/docs",
+ "www.red-database-security.com/advisory",
"www.redhat.com/archives",
+ "www.redhat.com/support/errata",
"www.samba.org/samba/security",
+ "www.secunia.com/advisories",
+ "www.securiteam.com/exploits",
+ "www.securiteam.com/securitynews",
+ "www.securiteam.com/unixfocus",
+ "www.securiteam.com/windowsntfocus",
"www.security-assessment.com/files",
"www.securityfocus.com",
"www.securitytracker.com",
"www.sophos.com/en-us/support",
"www.suse.com/support",
+ "www.symantec.com/avcenter/security",
+ "www.trustix.org/errata",
"www.ubuntu.com/usn",
"www.us-cert.gov/cas",
"www.us-cert.gov/ncas",
+ "www.us.debian.org/security",
"www.vmware.com/security/advisories",
+ "www.vupen.com/english/advisories",
"www.wireshark.org/security",
"www.zerodayinitiative.com/advisories",
+ "xforce.iss.net/alerts",
"zerodayinitiative.com/advisories",
}
diff --git a/internal/worker/update.go b/internal/worker/update.go
index 33ebb0f..b9724f3 100644
--- a/internal/worker/update.go
+++ b/internal/worker/update.go
@@ -194,6 +194,7 @@
endID := idFromFilename(batch[len(batch)-1].filename)
defer derrors.Wrap(&err, "updateBatch(%s-%s)", startID, endID)
+ log.Debug(ctx, "update transaction started", event.String("startID", startID), event.String("endID", endID))
err = u.st.RunTransaction(ctx, func(ctx context.Context, tx store.Transaction) error {
numAdds = 0
numMods = 0
@@ -232,7 +233,7 @@
if err != nil {
return 0, 0, err
}
- log.Info(ctx, "update transaction",
+ log.Debug(ctx, "update transaction finished",
event.String("startID", startID),
event.String("endID", endID),
event.Int64("adds", int64(numAdds)),
