internal/dcensus: add debugging for trace exporter
We occasionally see errors when exporting traces. For example:
rpc error: code = Internal desc = grpc: error while marshaling: string field contains invalid UTF-8
This CL adds a wrapper around the trace exporter to help figure out
what the bad string is.
Change-Id: I1d4918da6a08333f6bb4fea672238dfff83e3264
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/256766
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/go.mod b/go.mod
index 4dc11de..4b8cddb 100644
--- a/go.mod
+++ b/go.mod
@@ -21,12 +21,10 @@
github.com/golang-migrate/migrate/v4 v4.6.2
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
github.com/gomodule/redigo v2.0.0+incompatible // indirect
- github.com/google/addlicense v0.0.0-20190510175307-22550fa7c1b0 // indirect
github.com/google/go-cmp v0.5.2
github.com/google/go-replayers/httpreplay v0.1.0
github.com/google/licensecheck v0.0.0-20200805042302-c54f297c3b57
github.com/google/safehtml v0.0.1
- github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/lib/pq v1.2.0
github.com/microcosm-cc/bluemonday v1.0.2
github.com/russross/blackfriday/v2 v2.0.1
diff --git a/go.sum b/go.sum
index e02765c..d0d1c58 100644
--- a/go.sum
+++ b/go.sum
@@ -51,8 +51,6 @@
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg=
contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A=
-contrib.go.opencensus.io/exporter/stackdriver v0.12.7 h1:XWDDoMSlZchLyQZw8HKE+7vn3FpfaVR5Yz9E4ifxiU0=
-contrib.go.opencensus.io/exporter/stackdriver v0.12.7/go.mod h1:ZOhmSfHIoyVaQ+bKN+lR4h7K2olTIJsrdOwWHsNGw4w=
contrib.go.opencensus.io/exporter/stackdriver v0.13.4 h1:ksUxwH3OD5sxkjzEqGxNTl+Xjsmu3BnC/300MhSVTSc=
contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc=
contrib.go.opencensus.io/integrations/ocsql v0.1.4 h1:kfg5Yyy1nYUrqzyfW5XX+dzMASky8IJXhtHe0KTYNS4=
@@ -85,8 +83,6 @@
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.22.1 h1://WJvJi9iq/i5TWHuK3hIC23xCZYH7Qv7SIN2vZVqxY=
-github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.23.20 h1:2CBuL21P0yKdZN5urf2NxKa1ha8fhnY+A3pBCHFeZoA=
github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.34.29 h1:4Yw8eC4nCXiIVmHJO5PD4oh0vI/df5o6cYTVzFV7vWA=
@@ -222,7 +218,6 @@
github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
-github.com/google/addlicense v0.0.0-20190510175307-22550fa7c1b0/go.mod h1:QtPG26W17m+OIQgE6gQ24gC1M6pUaMBAbFrTIDtwG/E=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -275,7 +270,6 @@
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -290,6 +284,7 @@
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -346,6 +341,7 @@
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -536,7 +532,6 @@
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -621,6 +616,7 @@
golang.org/x/tools v0.0.0-20200828161849-5deb26317202 h1:DrWbY9UUFi/sl/3HkNVoBjDbGfIPZZfgoGsGxOL1EU8=
golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c h1:AQsh/7arPVFDBraQa8x7GoVnwnGg1kM7J2ySI0kF5WU=
golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -711,7 +707,6 @@
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
diff --git a/internal/dcensus/dcensus.go b/internal/dcensus/dcensus.go
index a1e1134..8de6fbd 100644
--- a/internal/dcensus/dcensus.go
+++ b/internal/dcensus/dcensus.go
@@ -133,6 +133,7 @@
}
view.RegisterExporter(viewExporter)
+ dte := &debugTraceExporter{}
// We want traces to be associated with the *app*, not the instance.
// TraceSpansBufferMaxBytes is increased from the default of 8MiB, though we
// can't increase *too* much because this is still running in GAE, which is
@@ -142,11 +143,13 @@
MonitoredResource: (*monitoredResource)(cfg.MonitoredResource),
TraceSpansBufferMaxBytes: 32 * 1024 * 1024, // 32 MiB
DefaultMonitoringLabels: stackdriverLabels(cfg),
+ OnError: dte.onError,
})
if err != nil {
log.Fatalf(ctx, "error creating trace exporter: %v", err)
}
- trace.RegisterExporter(traceExporter)
+ dte.exp = traceExporter
+ trace.RegisterExporter(dte)
}
// NewViewExporter creates a StackDriver exporter for stats.
@@ -174,6 +177,9 @@
ProjectID: cfg.ProjectID,
MonitoredResource: mr,
DefaultMonitoringLabels: stackdriverLabels(cfg),
+ OnError: func(err error) {
+ log.Errorf(context.Background(), "Stackdriver view exporter: %v", err)
+ },
})
}
diff --git a/internal/dcensus/debug.go b/internal/dcensus/debug.go
new file mode 100644
index 0000000..8bac6f1
--- /dev/null
+++ b/internal/dcensus/debug.go
@@ -0,0 +1,63 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dcensus
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "sync"
+
+ "go.opencensus.io/trace"
+ "golang.org/x/pkgsite/internal/log"
+)
+
+type debugTraceExporter struct {
+ exp trace.Exporter
+ mu sync.Mutex
+ err error
+}
+
+func (d *debugTraceExporter) onError(err error) {
+ log.Debugf(context.Background(), "trace exporter: onError called with %v", err)
+ d.err = err
+}
+
+// ExportSpan implements the trace.Exporter interface.
+func (d *debugTraceExporter) ExportSpan(s *trace.SpanData) {
+ ctx := context.Background()
+ d.mu.Lock()
+ d.exp.ExportSpan(s)
+ err := d.err
+ d.err = nil
+ d.mu.Unlock()
+ if err != nil {
+ log.Errorf(ctx, "trace exporter: %v", err)
+ log.Debugf(ctx, "trace exporter SpanData:\n%s", dumpSpanData(s))
+ }
+}
+
+func dumpSpanData(s *trace.SpanData) string {
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "Name: %q\n", s.Name)
+ dumpAttributes(&buf, s.Attributes)
+ for _, a := range s.Annotations {
+ fmt.Fprintf(&buf, " annotation: %q\n", a.Message)
+ dumpAttributes(&buf, a.Attributes)
+ }
+ fmt.Fprintf(&buf, "Status.Message: %q\n", s.Status.Message)
+ fmt.Fprintln(&buf, "link attrs:")
+ for _, l := range s.Links {
+ dumpAttributes(&buf, l.Attributes)
+ }
+ return buf.String()
+}
+
+func dumpAttributes(w io.Writer, m map[string]interface{}) {
+ for k, v := range m {
+ fmt.Fprintf(w, " %q: %#v\n", k, v)
+ }
+}
diff --git a/internal/worker/fetch.go b/internal/worker/fetch.go
index ccdb889..d060b26 100644
--- a/internal/worker/fetch.go
+++ b/internal/worker/fetch.go
@@ -11,6 +11,7 @@
"sort"
"strings"
"time"
+ "unicode/utf8"
"go.opencensus.io/trace"
"golang.org/x/mod/semver"
@@ -50,6 +51,12 @@
tctx, span := trace.StartSpan(ctx, "FetchAndUpdateState")
ctx = experiment.NewContext(tctx, experiment.FromContext(ctx).Active()...)
ctx = log.NewContextWithLabel(ctx, "fetch", modulePath+"@"+requestedVersion)
+ if !utf8.ValidString(modulePath) {
+ log.Errorf(ctx, "module path %q is not valid UTF-8", modulePath)
+ }
+ if !utf8.ValidString(requestedVersion) {
+ log.Errorf(ctx, "requested version %q is not valid UTF-8", requestedVersion)
+ }
span.AddAttributes(
trace.StringAttribute("modulePath", modulePath),
trace.StringAttribute("version", requestedVersion))
