content,internal: add experiments and excluded to worker homepage
Information about experiments and excluded prefixes are added to the
worker homepage.
Change-Id: I7bb7fd1eece434bd4da12e1af384b141c8a0ed41
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/239181
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/content/static/html/worker/index.tmpl b/content/static/html/worker/index.tmpl
index 162b4dc..a0a8866 100644
--- a/content/static/html/worker/index.tmpl
+++ b/content/static/html/worker/index.tmpl
@@ -1,90 +1,91 @@
<!--
- Copyright 2019 The Go Authors. All rights reserved.
- Use of this source code is governed by a BSD-style
- license that can be found in the LICENSE file.
+ Copyright 2019 The Go Authors. All rights reserved.
+ Use of this source code is governed by a BSD-style
+ license that can be found in the LICENSE file.
-->
{{define "versionTable"}}
- <table>
- <thead>
- {{if .}}
- <tr>
- <th>Module Version</th>
- <th>Index Timestamp</th>
- <th>Status</th>
- <th>Error</th>
- <th>Attempts</th>
- <th>LastAttempt</th>
- <th>NextAttempt</th>
- </tr>
- </thead>
- <tbody>
- {{range .}}
- <tr>
- <td>{{.ModulePath}}/@v/{{.Version}}</td>
- <td>{{.IndexTimestamp | timefmt}}</td>
- <td>{{.Status}}</td>
- <td>{{.Error | truncate 500}}</td>
- <td>{{.TryCount}}</td>
- <td>{{.LastProcessedAt | timefmt}}</td>
- <td>{{.NextProcessedAfter | timefmt}}</td>
- </tr>
- {{end}}
- {{else}}
- <p>No versions.</p>
- {{end}}
- </tbody>
- </table>
+ {{if .}}
+ <table>
+ <thead>
+ <tr>
+ <th>Module Version</th>
+ <th>Index Timestamp</th>
+ <th>Status</th>
+ <th>Error</th>
+ <th>Attempts</th>
+ <th>LastAttempt</th>
+ <th>NextAttempt</th>
+ </tr>
+ </thead>
+ <tbody>
+ {{range .}}
+ <tr>
+ <td>{{.ModulePath}}/@v/{{.Version}}</td>
+ <td>{{.IndexTimestamp | timefmt}}</td>
+ <td>{{.Status}}</td>
+ <td>{{.Error | truncate 500}}</td>
+ <td>{{.TryCount}}</td>
+ <td>{{.LastProcessedAt | timefmt}}</td>
+ <td>{{.NextProcessedAfter | timefmt}}</td>
+ </tr>
+ {{end}}
+ </tbody>
+ </table>
+ {{else}}
+ <p>No versions.</p>
+ {{end}}
{{end -}}
<!DOCTYPE html>
<script>
function submitForm(formName, reload) {
- let form = document[formName];
- form.result.value = "request pending...";
- let xhr = new XMLHttpRequest();
- xhr.onreadystatechange = function() {
- if (this.readyState == 4) {
- if (this.status >= 200 && this.status < 300) {
- if (reload) {
- location.reload();
- } else {
- form.result.value = "Success."
- }
- } else {
- form.result.value = "ERROR: " + this.responseText;
- }
- }
- }
- xhr.open(form.method, form.action);
- xhr.send(new FormData(form));
+ let form = document[formName];
+ form.result.value = "request pending...";
+ let xhr = new XMLHttpRequest();
+ xhr.onreadystatechange = function() {
+ if (this.readyState == 4) {
+ if (this.status >= 200 && this.status < 300) {
+ if (reload) {
+ location.reload();
+ } else {
+ form.result.value = "Success."
+ }
+ } else {
+ form.result.value = "ERROR: " + this.responseText;
+ }
+ }
+ }
+ xhr.open(form.method, form.action);
+ xhr.send(new FormData(form));
}
</script>
<style>
body {
- font-family: Verdana, Arial, sans-serif;
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI",
+ "Roboto", "Oxygen", "Ubuntu", "Helvetica Neue", Arial, sans-serif;
}
label {
- display: inline-block;
- text-align: right;
- width: 200px;
+ display: inline-block;
+ text-align: right;
+ width: 12.5rem;
}
input {
- width: 200px;
+ width: 12.5rem;
}
button {
- width: 200px;
- background-color: #eee;
- border-radius: 2px;
- border: 1px solid #ccc;
+ width: 12.5rem;
+ background-color: #eee;
+ border-radius: 0.125rem;
+ border: 0.0625rem solid #ccc;
}
table {
- border-spacing: 10px 2px;
- padding: 3px 0 2px 0;
- font-size: 12px;
+ border-spacing: 0.625rem 0.125rem;
+ padding: 0.1875rem 0 0.125rem 0;
+ font-size: 0.75rem;
}
td {
- border-top: 1px solid #ddd;
+ border-top: 0.0625rem solid #ddd;
}
</style>
<title>{{.Env}} Worker</title>
@@ -94,67 +95,93 @@
<p>
<a href="https://cloud.google.com/console/cloudtasks/queue/{{.ResourcePrefix}}fetch-tasks?project={{.Config.ProjectID}}"
- target="_blank" rel="noreferrer">
- Task Queue
+ target="_blank" rel="noreferrer">
+ Task Queue
</a> |
<a href="https://cloud.google.com/console/cloudscheduler?project={{.Config.ProjectID}}"
- target="_blank" rel="noreferrer">
- Scheduler
+ target="_blank" rel="noreferrer">
+ Scheduler
</a> |
<a href="https://cloud.google.com/console/logs/viewer?project={{.Config.ProjectID}}&resource=gae_app%2Fmodule_id%2F{{.Config.ServiceID}}"
- target="_blank" rel="noreferrer">
- Logs (switch to "All Logs" when you get there)
+ target="_blank" rel="noreferrer">
+ Logs (switch to "All Logs" when you get there)
</a>
</p>
<div class="actions">
- <form action="/poll-and-queue" method="post" name="queueForm">
- <button title="Poll the module index for up to 2000 new versions, and enqueue them for processing."
- onclick="submitForm('queueForm', false); return false">Enqueue From Module Index</button>
- <input type="number" name="limit" value="10"></input>
- <output name="result"></output>
- </form>
- <form action="/requeue" method="post" name="requeueForm">
- <button title="Query the discovery database for failed versions, and re-queue them for processing."
- onclick="submitForm('requeueForm', true); return false">Requeue Failed Versions</button>
- <input type="number" name="limit" value="10">
- <output name="result"></output>
- </form>
- <form action="/reprocess" method="post" name="reprocessForm">
- <button title="Mark all versions created before the specified app_version to be reprocessed."
- onclick="submitForm('reprocessForm', true); return false">Reprocess Versions</button>
- <input type="text" name="app_version">
- <output name="result"></output>
- </form>
- <form action="/populate-stdlib" method="post" name="populateStdlibForm">
- <button title="Populates the database with all supported versions of the Go standard library."
- onclick="submitForm('populateStdlibForm', false); return false">Populate Standard Library</button>
- <output name="result"></output>
- </form>
+ <form action="/poll-and-queue" method="post" name="queueForm">
+ <button title="Poll the module index for up to 2000 new versions, and enqueue them for processing."
+ onclick="submitForm('queueForm', false); return false">Enqueue From Module Index</button>
+ <input type="number" name="limit" value="10"></input>
+ <output name="result"></output>
+ </form>
+ <form action="/requeue" method="post" name="requeueForm">
+ <button title="Query the discovery database for failed versions, and re-queue them for processing."
+ onclick="submitForm('requeueForm', true); return false">Requeue Failed Versions</button>
+ <input type="number" name="limit" value="10">
+ <output name="result"></output>
+ </form>
+ <form action="/reprocess" method="post" name="reprocessForm">
+ <button title="Mark all versions created before the specified app_version to be reprocessed."
+ onclick="submitForm('reprocessForm', true); return false">Reprocess Versions</button>
+ <input type="text" name="app_version">
+ <output name="result"></output>
+ </form>
+ <form action="/populate-stdlib" method="post" name="populateStdlibForm">
+ <button title="Populates the database with all supported versions of the Go standard library."
+ onclick="submitForm('populateStdlibForm', false); return false">Populate Standard Library</button>
+ <output name="result"></output>
+ </form>
</div>
<div class="config">
<h3>Config</h3>
<table>
- <tr><td>App Version</td><td>{{.Config.VersionID}}</td></tr>
- <tr><td>Zone</td><td>{{.Config.ZoneID}}</td></tr>
- <tr><td>DB Host</td><td>{{.Config.DBHost}}</td></tr>
- <tr><td>Redis Cache Host</td><td>{{.Config.RedisCacheHost}}</td></tr>
- <tr><td>Redis HA Host</td><td>{{.Config.RedisHAHost}}</td></tr>
+ <tr><td>App Version</td><td>{{.Config.VersionID}}</td></tr>
+ <tr><td>Zone</td><td>{{.Config.ZoneID}}</td></tr>
+ <tr><td>DB Host</td><td>{{.Config.DBHost}}</td></tr>
+ <tr><td>Redis Cache Host</td><td>{{.Config.RedisCacheHost}}</td></tr>
+ <tr><td>Redis HA Host</td><td>{{.Config.RedisHAHost}}</td></tr>
</table>
</div>
+<div>
+ <h3>Experiments</h3>
+ {{if .Experiments}}
+ <table>
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Rollout</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ {{range .Experiments}}
+ <tr>
+ <td>{{.Name}}</td>
+ <td>{{.Rollout}}</td>
+ <td>{{.Description}}</td>
+ </tr>
+ {{end}}
+ </tbody>
+ </table>
+ {{else}}
+ <p>No experiments.</p>
+ {{end}}
+</div>
+
<div class="stats">
<h3>Statistics</h3>
<p>Latest timestamp from the module index: {{.LatestTimestamp | timefmt}}</p>
<table>
- <caption>Results by status:</caption>
- <thead><tr><th>Code</th><th>Status</th><th>Count</th></tr></thead>
- <tbody>
- {{range .Counts}}
- <tr><td>{{.Code}}</td><td>{{.Desc}}</td><td>{{.Count}}</td></tr>
- {{end}}
- </tbody>
+ <caption>Results by status:</caption>
+ <thead><tr><th>Code</th><th>Status</th><th>Count</th></tr></thead>
+ <tbody>
+ {{range .Counts}}
+ <tr><td>{{.Code}}</td><td>{{.Desc}}</td><td>{{.Count}}</td></tr>
+ {{end}}
+ </tbody>
</table>
</div>
@@ -166,3 +193,21 @@
<h3>Recent failed attempts:</h3>
{{template "versionTable" .RecentFailures}}
+
+<div>
+ <h3>Excluded Prefixes</h3>
+ {{if .Excluded}}
+ <table>
+ <thead>
+ <tr><th>Prefix</th></tr>
+ </thead>
+ <tbody>
+ {{range .Excluded}}
+ <tr><td>{{.}}</td></tr>
+ {{end}}
+ </tbody>
+ </table>
+ {{else}}
+ <p>No excluded prefixes.</p>
+ {{end}}
+</div>
diff --git a/internal/postgres/excluded.go b/internal/postgres/excluded.go
index 9010f2c..4f5e5b6 100644
--- a/internal/postgres/excluded.go
+++ b/internal/postgres/excluded.go
@@ -76,9 +76,10 @@
if time.Since(lastFetched) < excludedPrefixesExpiration {
return
}
- prefixes, err := db.readExcludedPrefixes(ctx)
+ prefixes, err := db.GetExcludedPrefixes(ctx)
excludedPrefixes.mu.Lock()
defer excludedPrefixes.mu.Unlock()
+ excludedPrefixes.lastFetched = time.Now()
excludedPrefixes.prefixes = prefixes
excludedPrefixes.err = err
if err != nil {
@@ -86,8 +87,8 @@
}
}
-// readExcludedPrefixes reads all the excluded prefixes from the database.
-func (db *DB) readExcludedPrefixes(ctx context.Context) ([]string, error) {
+// GetExcludedPrefixes reads all the excluded prefixes from the database.
+func (db *DB) GetExcludedPrefixes(ctx context.Context) ([]string, error) {
var eps []string
err := db.db.RunQuery(ctx, `SELECT prefix FROM excluded_prefixes`, func(rows *sql.Rows) error {
var ep string
@@ -100,6 +101,5 @@
if err != nil {
return nil, err
}
- setExcludedPrefixesLastFetched(time.Now())
return eps, nil
}
diff --git a/internal/worker/server.go b/internal/worker/server.go
index a7effce..7097291 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -69,7 +69,7 @@
func NewServer(cfg *config.Config, scfg ServerConfig) (_ *Server, err error) {
defer derrors.Wrap(&err, "NewServer(db, %+v)", scfg)
- indexTemplate, err := parseTemplate(scfg.StaticPath)
+ indexTemplate, err := parseTemplate(scfg.StaticPath, "index.tmpl")
if err != nil {
return nil, err
}
@@ -84,8 +84,8 @@
redisCacheClient: scfg.RedisCacheClient,
queue: scfg.Queue,
reportingClient: scfg.ReportingClient,
- indexTemplate: indexTemplate,
taskIDChangeInterval: scfg.TaskIDChangeInterval,
+ indexTemplate: indexTemplate,
}, nil
}
@@ -347,6 +347,8 @@
var (
next, failures, recents []*internal.ModuleVersionState
stats *postgres.VersionStats
+ experiments []*internal.Experiment
+ excluded []string
)
type annotation struct {
error
@@ -385,6 +387,22 @@
}
return nil
})
+ g.Go(func() error {
+ var err error
+ experiments, err = s.db.GetExperiments(ctx)
+ if err != nil {
+ return annotation{err, "error fetching experiments"}
+ }
+ return nil
+ })
+ g.Go(func() error {
+ var err error
+ excluded, err = s.db.GetExcludedPrefixes(ctx)
+ if err != nil {
+ return annotation{err, "error fetching excluded"}
+ }
+ return nil
+ })
if err := g.Wait(); err != nil {
var e annotation
if errors.As(err, &e) {
@@ -426,6 +444,8 @@
LatestTimestamp *time.Time
Counts []*count
Next, Recent, RecentFailures []*internal.ModuleVersionState
+ Experiments []*internal.Experiment
+ Excluded []string
}{
Config: s.cfg,
Env: env,
@@ -435,14 +455,16 @@
Next: next,
Recent: recents,
RecentFailures: failures,
+ Experiments: experiments,
+ Excluded: excluded,
}
var buf bytes.Buffer
if err := s.indexTemplate.Execute(&buf, page); err != nil {
return "error rendering template", err
}
if _, err := io.Copy(w, &buf); err != nil {
- http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
log.Errorf(ctx, "Error copying buffer to ResponseWriter: %v", err)
+ http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}
return "", nil
}
@@ -511,12 +533,12 @@
}
// Parse the template for the status page.
-func parseTemplate(staticPath string) (*template.Template, error) {
+func parseTemplate(staticPath, filename string) (*template.Template, error) {
if staticPath == "" {
return nil, nil
}
- templatePath := filepath.Join(staticPath, "html/worker/index.tmpl")
- return template.New("index.tmpl").Funcs(template.FuncMap{
+ templatePath := filepath.Join(staticPath, "html/worker/"+filename)
+ return template.New(filename).Funcs(template.FuncMap{
"truncate": truncate,
"timefmt": formatTime,
}).ParseFiles(templatePath)
