sandbox: add container creation metrics
This change measures the latency and success of container creation.
These metrics will help capacity planning and investigating production
issues.
Updates golang/go#25224
Updates golang/go#38530
Change-Id: Id7f373acb8741d4465c6e632badb188b6e855787
Reviewed-on: https://go-review.googlesource.com/c/playground/+/229980
Run-TryBot: Alexander Rakoczy <alex@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/sandbox/metrics.go b/sandbox/metrics.go
index 82b4329..075be96 100644
--- a/sandbox/metrics.go
+++ b/sandbox/metrics.go
@@ -21,13 +21,12 @@
mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
)
-// Customizations of ochttp views. Views are updated as follows:
-// * The views are prefixed with go-playground-sandbox.
-// * ochttp.KeyServerRoute is added as a tag to label metrics per-route.
var (
- mContainers = stats.Int64("go-playground/sandbox/container_count", "number of sandbox containers", stats.UnitDimensionless)
- mUnwantedContainers = stats.Int64("go-playground/sandbox/unwanted_container_count", "number of sandbox containers that are unexpectedly running", stats.UnitDimensionless)
- mMaxContainers = stats.Int64("go-playground/sandbox/max_container_count", "target number of sandbox containers", stats.UnitDimensionless)
+ kContainerCreateSuccess = tag.MustNewKey("go-playground/sandbox/container_create_success")
+ mContainers = stats.Int64("go-playground/sandbox/container_count", "number of sandbox containers", stats.UnitDimensionless)
+ mUnwantedContainers = stats.Int64("go-playground/sandbox/unwanted_container_count", "number of sandbox containers that are unexpectedly running", stats.UnitDimensionless)
+ mMaxContainers = stats.Int64("go-playground/sandbox/max_container_count", "target number of sandbox containers", stats.UnitDimensionless)
+ mContainerCreateLatency = stats.Float64("go-playground/sandbox/container_create_latency", "", stats.UnitMilliseconds)
containerCount = &view.View{
Name: "go-playground/sandbox/container_count",
@@ -50,7 +49,25 @@
Measure: mMaxContainers,
Aggregation: view.LastValue(),
}
+ containerCreateCount = &view.View{
+ Name: "go-playground/sandbox/container_create_count",
+ Description: "Number of containers created",
+ Measure: mContainerCreateLatency,
+ TagKeys: []tag.Key{kContainerCreateSuccess},
+ Aggregation: view.Count(),
+ }
+ containerCreationLatency = &view.View{
+ Name: "go-playground/sandbox/container_create_latency",
+ Description: "Latency distribution of container creation",
+ Measure: mContainerCreateLatency,
+ Aggregation: ochttp.DefaultLatencyDistribution,
+ }
+)
+// Customizations of ochttp views. Views are updated as follows:
+// * The views are prefixed with go-playground-sandbox.
+// * ochttp.KeyServerRoute is added as a tag to label metrics per-route.
+var (
ServerRequestCountView = &view.View{
Name: "go-playground-sandbox/http/server/request_count",
Description: "Count of HTTP requests started",
@@ -104,6 +121,8 @@
containerCount,
unwantedContainerCount,
maxContainerCount,
+ containerCreateCount,
+ containerCreationLatency,
ServerRequestCountView,
ServerRequestBytesView,
ServerResponseBytesView,
diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go
index 59ba51a..350eeea 100644
--- a/sandbox/sandbox.go
+++ b/sandbox/sandbox.go
@@ -33,6 +33,7 @@
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/stats"
+ "go.opencensus.io/tag"
"go.opencensus.io/trace"
"golang.org/x/playground/internal"
"golang.org/x/playground/sandbox/sandboxtypes"
@@ -419,6 +420,17 @@
}
func startContainer(ctx context.Context) (c *Container, err error) {
+ start := time.Now()
+ defer func() {
+ status := "success"
+ if err != nil {
+ status = "error"
+ }
+ // Ignore error. The only error can be invalid tag key or value length, which we know are safe.
+ _ = stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(kContainerCreateSuccess, status)},
+ mContainerCreateLatency.M(float64(time.Since(start))/float64(time.Millisecond)))
+ }()
+
name := "play_run_" + randHex(8)
setContainerWanted(name, true)
cmd := exec.Command("docker", "run",