internal/worker: initial code for server
Set up a basic HTML server for the worker.
Change-Id: I4861b887eb5e9dca9e74eacfb3807e2017576342
Reviewed-on: https://go-review.googlesource.com/c/vuln/+/368294
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
new file mode 100644
index 0000000..2e88dbf
--- /dev/null
+++ b/cmd/worker/main.go
@@ -0,0 +1,75 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command worker runs the vuln worker server.
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "net/http"
+ "os"
+
+ "cloud.google.com/go/errorreporting"
+ "golang.org/x/vuln/internal/derrors"
+ "golang.org/x/vuln/internal/worker"
+ "golang.org/x/vuln/internal/worker/log"
+ "golang.org/x/vuln/internal/worker/store"
+)
+
+var (
+ project = flag.String("project", os.Getenv("GOOGLE_CLOUD_PROJECT"), "project ID")
+ namespace = flag.String("namespace", os.Getenv("VULN_WORKER_NAMESPACE"), "Firestore namespace")
+ errorReporting = flag.Bool("reporterrors", os.Getenv("VULN_WORKER_REPORT_ERRORS") == "true", "use the error reporting API")
+)
+
+const serviceID = "vuln-worker"
+
+func main() {
+ flag.Parse()
+ if *project == "" {
+ die("need -project or GOOGLE_CLOUD_PROJECT")
+ }
+ if *namespace == "" {
+ die("need -namespace or VULN_WORKER_NAMESPACE")
+ }
+ if os.Getenv("PORT") == "" && flag.NArg() == 0 {
+ die("need PORT or command-line args")
+ }
+
+ ctx := log.WithLineLogger(context.Background())
+
+ fstore, err := store.NewFireStore(ctx, *project, *namespace)
+ if err != nil {
+ die("firestore: %v", err)
+ }
+
+ if *errorReporting {
+ reportingClient, err := errorreporting.NewClient(ctx, *project, errorreporting.Config{
+ ServiceName: serviceID,
+ OnError: func(err error) {
+ log.Errorf(ctx, "Error reporting failed: %v", err)
+ },
+ })
+ if err != nil {
+ die("errorreporting: %v", err)
+ }
+ derrors.SetReportingClient(reportingClient)
+ }
+
+ _, err = worker.NewServer(ctx, *namespace, fstore)
+ if err != nil {
+ die("NewServer: %v", err)
+ }
+ addr := ":" + os.Getenv("PORT")
+ log.Infof(ctx, "Listening on addr %s", addr)
+ die("listening: %v", http.ListenAndServe(addr, nil))
+}
+
+func die(format string, args ...interface{}) {
+ fmt.Fprintf(os.Stderr, format, args...)
+ fmt.Fprintln(os.Stderr)
+ os.Exit(1)
+}
diff --git a/go.mod b/go.mod
index f8183a0..7ec2e2c 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@
require (
cloud.google.com/go v0.97.0 // indirect
+ cloud.google.com/go/errorreporting v0.1.0
cloud.google.com/go/firestore v1.6.1
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
@@ -21,6 +22,7 @@
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.6
+ github.com/google/safehtml v0.0.2
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -46,3 +48,5 @@
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0
)
+
+require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
diff --git a/go.sum b/go.sum
index c0f21e2..e93dcf8 100644
--- a/go.sum
+++ b/go.sum
@@ -22,6 +22,7 @@
cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
+cloud.google.com/go v0.92.2/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8=
@@ -34,6 +35,8 @@
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/errorreporting v0.1.0 h1:z40EhrjRspplwbpO+9DSnC4kgDokBi94T/gYwtdKL5Q=
+cloud.google.com/go/errorreporting v0.1.0/go.mod h1:cZSiBMvrnl0X13pD9DwKf9sQ8Eqy3EzHqkyKBZxiIrM=
cloud.google.com/go/firestore v1.6.1 h1:8rBq3zRjnHx8UtBvaOWqBB1xq9jH6/wltfQLlTMh2Fw=
cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@@ -237,6 +240,8 @@
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/safehtml v0.0.2 h1:ZOt2VXg4x24bW0m2jtzAOkhoXV0iM8vNKc0paByCZqM=
+github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -593,6 +598,7 @@
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/internal/derrors/derrors.go b/internal/derrors/derrors.go
index c695747..cc1faf0 100644
--- a/internal/derrors/derrors.go
+++ b/internal/derrors/derrors.go
@@ -6,7 +6,11 @@
// types error semantics supported by x/vuln.
package derrors
-import "fmt"
+import (
+ "fmt"
+
+ "cloud.google.com/go/errorreporting"
+)
// Wrap adds context to the error and allows
// unwrapping the result to recover the original error.
@@ -19,3 +23,25 @@
*errp = fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), *errp)
}
}
+
+// WrapAndReport calls Wrap followed by Report.
+func WrapAndReport(errp *error, format string, args ...interface{}) {
+ Wrap(errp, format, args...)
+ if *errp != nil {
+ Report(*errp)
+ }
+}
+
+var repClient *errorreporting.Client
+
+// SetReportingClient sets an errorreporting client, for use by Report.
+func SetReportingClient(c *errorreporting.Client) {
+ repClient = c
+}
+
+// Report uses the errorreporting API to report an error.
+func Report(err error) {
+ if repClient != nil {
+ repClient.Report(errorreporting.Entry{Error: err})
+ }
+}
diff --git a/internal/worker/server.go b/internal/worker/server.go
new file mode 100644
index 0000000..d4d3311
--- /dev/null
+++ b/internal/worker/server.go
@@ -0,0 +1,124 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "net/http"
+ "path/filepath"
+ "time"
+
+ "github.com/google/safehtml/template"
+ "golang.org/x/exp/event"
+ "golang.org/x/vuln/internal/derrors"
+ "golang.org/x/vuln/internal/worker/log"
+ "golang.org/x/vuln/internal/worker/store"
+)
+
+var staticPath = template.TrustedSourceFromConstant("internal/worker/static")
+
+type Server struct {
+ namespace string
+ st store.Store
+
+ indexTemplate *template.Template
+}
+
+func NewServer(ctx context.Context, namespace string, st store.Store) (_ *Server, err error) {
+ defer derrors.Wrap(&err, "NewServer(%q)", namespace)
+
+ s := &Server{namespace: namespace, st: st}
+ s.indexTemplate, err = parseTemplate(staticPath, template.TrustedSourceFromConstant("index.tmpl"))
+ if err != nil {
+ return nil, err
+ }
+ s.handle(ctx, "/", s.indexPage)
+ http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath.String()))))
+ s.handle(ctx, "/favicon.ico", func(w http.ResponseWriter, r *http.Request) error {
+ http.ServeFile(w, r, filepath.Join(staticPath.String(), "favicon.ico"))
+ return nil
+ })
+ return s, nil
+}
+
+func (s *Server) indexPage(w http.ResponseWriter, r *http.Request) error {
+ type data struct {
+ Namespace string
+ }
+ page := data{
+ Namespace: s.namespace,
+ }
+ return renderPage(r.Context(), w, page, s.indexTemplate)
+}
+
+func (s *Server) handle(ctx context.Context, pattern string, handler func(w http.ResponseWriter, r *http.Request) error) {
+ http.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
+ start := time.Now()
+ traceID := r.Header.Get("X-Cloud-Trace-Context")
+
+ log.Info(ctx, "request start",
+ event.Value("httpRequest", r),
+ event.String("traceID", traceID))
+
+ r = r.WithContext(log.WithLineLogger(r.Context()))
+ w2 := &responseWriter{ResponseWriter: w}
+ if err := handler(w2, r); err != nil {
+ s.serveError(ctx, w2, r, err)
+ }
+
+ log.Info(ctx, "request end",
+ event.Value("traceID", traceID),
+ event.Duration("latency", time.Since(start)),
+ event.Int64("status", translateStatus(w2.status)))
+ })
+}
+
+func (s *Server) serveError(ctx context.Context, w http.ResponseWriter, r *http.Request, err error) {
+ errString := err.Error()
+ log.Error(ctx, errString)
+ http.Error(w, errString, http.StatusInternalServerError)
+}
+
+type responseWriter struct {
+ http.ResponseWriter
+ status int
+}
+
+func (rw *responseWriter) WriteHeader(code int) {
+ rw.status = code
+ rw.ResponseWriter.WriteHeader(code)
+}
+
+func translateStatus(code int) int64 {
+ if code == 0 {
+ return http.StatusOK
+ }
+ return int64(code)
+}
+
+// Parse a template.
+func parseTemplate(staticPath, filename template.TrustedSource) (*template.Template, error) {
+ if staticPath.String() == "" {
+ return nil, nil
+ }
+ templatePath := template.TrustedSourceJoin(staticPath, filename)
+ return template.New(filename.String()).ParseFilesFromTrustedSources(templatePath)
+}
+
+func renderPage(ctx context.Context, w http.ResponseWriter, page interface{}, tmpl *template.Template) (err error) {
+ defer derrors.Wrap(&err, "renderPage")
+
+ var buf bytes.Buffer
+ if err := tmpl.Execute(&buf, page); err != nil {
+ return err
+ }
+ if _, err := io.Copy(w, &buf); err != nil {
+ log.Error(ctx, "copying buffer to ResponseWriter", event.Value("error", err))
+ return err
+ }
+ return nil
+}
diff --git a/internal/worker/static/favicon.ico b/internal/worker/static/favicon.ico
new file mode 100644
index 0000000..43dc397
--- /dev/null
+++ b/internal/worker/static/favicon.ico
Binary files differ
diff --git a/internal/worker/static/index.tmpl b/internal/worker/static/index.tmpl
new file mode 100644
index 0000000..85094b1
--- /dev/null
+++ b/internal/worker/static/index.tmpl
@@ -0,0 +1,20 @@
+<!--
+ Copyright 2021 The Go Authors. All rights reserved.
+ Use of this source code is governed by a BSD-style
+ license that can be found in the LICENSE file.
+-->
+
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="utf-8">
+<link href="/static/worker.css" rel="stylesheet">
+<title>{{.Namespace}} Vuln Worker</title>
+
+<body>
+ <h1>{{.Namespace}} Vuln Worker</h1>
+ <p>All times in America/New_York.</p>
+</body>
+</html>
+
+
+
diff --git a/internal/worker/static/worker.css b/internal/worker/static/worker.css
new file mode 100644
index 0000000..81a9f72
--- /dev/null
+++ b/internal/worker/static/worker.css
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2021 The Go Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+
+:root {
+ --white: #eee;
+ --gray: #ccc;
+ --red: red;
+}
+
+body {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu',
+ 'Helvetica Neue', Arial, sans-serif;
+}
+label {
+ display: inline-block;
+ text-align: right;
+ width: 12.5rem;
+}
+input {
+ width: 12.5rem;
+}
+button {
+ background-color: var(--white);
+ border: 0.0625rem solid var(--gray);
+ border-radius: 0.125rem;
+ width: 16rem;
+}
+table {
+ border-spacing: 0.625rem 0.125rem;
+ font-size: 0.75rem;
+ padding: 0.1875rem 0 0.125rem 0;
+}
+td {
+ border-top: 0.0625rem solid var(--gray);
+}
