internal/worker: home page
Add information about noteworthy CVEs to the home page.
Change-Id: I86c30e333cc3ac45d5689928ba5a7d5dea457985
Reviewed-on: https://go-review.googlesource.com/c/vuln/+/368855
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/go.mod b/go.mod
index fddd1f5..df353c7 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@
github.com/google/safehtml v0.0.2
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
+ github.com/jba/templatecheck v0.6.0
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -36,6 +37,7 @@
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 // indirect
+ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
@@ -49,5 +51,3 @@
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0
)
-
-require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
diff --git a/go.sum b/go.sum
index 26a3970..a1e2dc5 100644
--- a/go.sum
+++ b/go.sum
@@ -287,6 +287,8 @@
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jba/templatecheck v0.6.0 h1:SwM8C4hlK/YNLsdcXStfnHWE2HKkuTVwy5FKQHt5ro8=
+github.com/jba/templatecheck v0.6.0/go.mod h1:/1k7EajoSErFI9GLHAsiIJEaNLt3ALKNw2TV7z2SYv4=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
diff --git a/internal/worker/server.go b/internal/worker/server.go
index d4d3311..ae9fb81 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -9,12 +9,15 @@
"context"
"io"
"net/http"
+ "os"
"path/filepath"
"time"
"github.com/google/safehtml/template"
"golang.org/x/exp/event"
+ "golang.org/x/sync/errgroup"
"golang.org/x/vuln/internal/derrors"
+ "golang.org/x/vuln/internal/gitrepo"
"golang.org/x/vuln/internal/worker/log"
"golang.org/x/vuln/internal/worker/store"
)
@@ -45,16 +48,6 @@
return s, nil
}
-func (s *Server) indexPage(w http.ResponseWriter, r *http.Request) error {
- type data struct {
- Namespace string
- }
- page := data{
- Namespace: s.namespace,
- }
- return renderPage(r.Context(), w, page, s.indexTemplate)
-}
-
func (s *Server) handle(ctx context.Context, pattern string, handler func(w http.ResponseWriter, r *http.Request) error) {
http.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
@@ -106,7 +99,27 @@
return nil, nil
}
templatePath := template.TrustedSourceJoin(staticPath, filename)
- return template.New(filename.String()).ParseFilesFromTrustedSources(templatePath)
+ return template.New(filename.String()).Funcs(template.FuncMap{
+ "timefmt": formatTime,
+ }).ParseFilesFromTrustedSources(templatePath)
+}
+
+var locNewYork *time.Location
+
+func init() {
+ var err error
+ locNewYork, err = time.LoadLocation("America/New_York")
+ if err != nil {
+ log.Errorf(context.Background(), "time.LoadLocation: %v", err)
+ os.Exit(1)
+ }
+}
+
+func formatTime(t *time.Time) string {
+ if t == nil || t.IsZero() {
+ return "-"
+ }
+ return t.In(locNewYork).Format("2006-01-02 15:04:05")
}
func renderPage(ctx context.Context, w http.ResponseWriter, page interface{}, tmpl *template.Template) (err error) {
@@ -122,3 +135,48 @@
}
return nil
}
+
+type indexPage struct {
+ CVEListRepoURL string
+ Namespace string
+ Updates []*store.CommitUpdateRecord
+ CVEsNeedingIssue []*store.CVERecord
+ CVEsUpdatedSince []*store.CVERecord
+}
+
+func (s *Server) indexPage(w http.ResponseWriter, r *http.Request) error {
+
+ var (
+ updates []*store.CommitUpdateRecord
+ needingIssue, updatedSince []*store.CVERecord
+ )
+
+ g, ctx := errgroup.WithContext(r.Context())
+ g.Go(func() error {
+ var err error
+ updates, err = s.st.ListCommitUpdateRecords(ctx, 10)
+ return err
+ })
+ g.Go(func() error {
+ var err error
+ needingIssue, err = s.st.ListCVERecordsWithTriageState(ctx, store.TriageStateNeedsIssue)
+ return err
+ })
+ g.Go(func() error {
+ var err error
+ updatedSince, err = s.st.ListCVERecordsWithTriageState(ctx, store.TriageStateUpdatedSinceIssueCreation)
+ return err
+ })
+ if err := g.Wait(); err != nil {
+ return err
+ }
+
+ page := indexPage{
+ CVEListRepoURL: gitrepo.CVEListRepoURL,
+ Namespace: s.namespace,
+ Updates: updates,
+ CVEsNeedingIssue: needingIssue,
+ CVEsUpdatedSince: updatedSince,
+ }
+ return renderPage(r.Context(), w, page, s.indexTemplate)
+}
diff --git a/internal/worker/server_test.go b/internal/worker/server_test.go
new file mode 100644
index 0000000..c4e63b1
--- /dev/null
+++ b/internal/worker/server_test.go
@@ -0,0 +1,24 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "testing"
+
+ "github.com/google/safehtml/template"
+ "github.com/jba/templatecheck"
+)
+
+func TestTemplates(t *testing.T) {
+ // Check parsed templates.
+ staticPath := template.TrustedSourceFromConstant("static")
+ index, err := parseTemplate(staticPath, template.TrustedSourceFromConstant("index.tmpl"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := templatecheck.CheckSafe(index, indexPage{}); err != nil {
+ t.Error(err)
+ }
+}
diff --git a/internal/worker/static/index.tmpl b/internal/worker/static/index.tmpl
index 85094b1..51d4a3a 100644
--- a/internal/worker/static/index.tmpl
+++ b/internal/worker/static/index.tmpl
@@ -13,8 +13,62 @@
<body>
<h1>{{.Namespace}} Vuln Worker</h1>
<p>All times in America/New_York.</p>
+
+
+ <h2>Recent Updates</h2>
+ {{with .Updates}}
+ <table>
+ <tr>
+ <th>Started</th><th>Ended</th><th>Commit</th><th>Processed</th><th>Added</th><th>Modified</th><th>Error</th>
+ </tr>
+ {{range .}}
+ <tr>
+ <td>{{.StartedAt | timefmt}}</td>
+ <td>{{.EndedAt | timefmt}}</td>
+ <td><a href="{{$.CVEListRepoURL}}/tree/{{.CommitHash}}">{{.CommitHash}}</a></td>
+ <td>{{.NumProcessed}}/{{.NumTotal}}</td>
+ <td>{{.NumAdded}}</td>
+ <td>{{.NumModified}}</td>
+ <td>{{.Error}}</td>
+ </tr>
+ {{end}}
+ </table>
+ {{else}}
+ No updates.
+ {{end}}
+
+ <h2>CVEs Needing Issue</h2>
+ <p>{{len .CVEsNeedingIssue}} records.</p>
+ <table>
+ <tr>
+ <th>ID</th><th>Reason</th>
+ </tr>
+ {{range .CVEsNeedingIssue}}
+ <tr>
+ <td><a href="{{$.CVEListRepoURL}}/tree/{{.CommitHash}}/{{.Path}}">{{.ID}}</a></td>
+ <td>{{.TriageState}}</td>
+ <td>{{.TriageStateReason}}</td>
+ </tr>
+ {{end}}
+ </table>
+
+ <h2>CVEs Updated Since Issue Created</h2>
+ <p>{{len .CVEsUpdatedSince}} records.</p>
+ <table>
+ <tr>
+ <th>ID</th><th>Reason</th><th>Issue</th><th>Issue Created</th>
+ </tr>
+ {{range .CVEsUpdatedSince}}
+ <tr>
+ <td><a href="{{$.CVEListRepoURL}}/tree/{{.CommitHash}}/{{.Path}}">{{.ID}}</a></td>
+ <td>{{.TriageState}}</td>
+ <td>{{.TriageStateReason}}</td>
+ <td>{{.IssueReference}}</td>
+ <td>{{.IssueCreatedAt | timefmt}}</td>
+ </tr>
+ {{end}}
+ </table>
+
</body>
</html>
-
-
diff --git a/internal/worker/store/fire_store.go b/internal/worker/store/fire_store.go
index 3c7b037..9e4dfbe 100644
--- a/internal/worker/store/fire_store.go
+++ b/internal/worker/store/fire_store.go
@@ -9,6 +9,7 @@
"errors"
"fmt"
"strings"
+ "time"
"cloud.google.com/go/firestore"
"golang.org/x/vuln/internal/derrors"
@@ -118,6 +119,18 @@
Hash string
}
+// ListCVERecordsWithTriageState implements Store.ListCVERecordsWithTriageState.
+func (fs *FireStore) ListCVERecordsWithTriageState(ctx context.Context, ts TriageState) (_ []*CVERecord, err error) {
+ defer derrors.Wrap(&err, "ListCVERecordsWithTriageState(%s)", ts)
+
+ q := fs.nsDoc.Collection(cveCollection).Where("TriageState", "==", ts).OrderBy("ID", firestore.Asc)
+ docsnaps, err := q.Documents(ctx).GetAll()
+ if err != nil {
+ return nil, err
+ }
+ return docsnapsToCVERecords(docsnaps)
+}
+
// dirHashRef returns a DocumentRef for the directory dir.
func (s *FireStore) dirHashRef(dir string) *firestore.DocumentRef {
// Firestore IDs cannot contain slashes.
@@ -154,6 +167,25 @@
return err
}
+func (fs *FireStore) GetAllCVERecords(ctx context.Context) ([]*CVERecord, error) {
+ start := time.Now()
+ ds, err := fs.nsDoc.Collection(cveCollection).Select().Where("TriageState", "==", "NoActionNeeded").Documents(ctx).GetAll()
+ if err != nil {
+ return nil, err
+ }
+ fmt.Printf("#### where needsissue: %d recs in %s\n", len(ds), time.Since(start))
+ _ = ds
+
+ docsnaps, err := fs.nsDoc.Collection(cveCollection).
+ Select("ID", "TriageState").
+ Limit(100).
+ Documents(ctx).GetAll()
+ if err != nil {
+ return nil, err
+ }
+ return docsnapsToCVERecords(docsnaps)
+}
+
// RunTransaction implements Store.RunTransaction.
func (fs *FireStore) RunTransaction(ctx context.Context, f func(context.Context, Transaction) error) error {
return fs.client.RunTransaction(ctx,
@@ -163,8 +195,8 @@
}
// cveRecordRef returns a DocumentRef to the CVERecord with id.
-func (s *FireStore) cveRecordRef(id string) *firestore.DocumentRef {
- return s.nsDoc.Collection(cveCollection).Doc(id)
+func (fs *FireStore) cveRecordRef(id string) *firestore.DocumentRef {
+ return fs.nsDoc.Collection(cveCollection).Doc(id)
}
// fsTransaction implements Transaction
@@ -206,6 +238,10 @@
if err != nil {
return nil, err
}
+ return docsnapsToCVERecords(docsnaps)
+}
+
+func docsnapsToCVERecords(docsnaps []*firestore.DocumentSnapshot) ([]*CVERecord, error) {
var crs []*CVERecord
for _, ds := range docsnaps {
var cr CVERecord
diff --git a/internal/worker/store/mem_store.go b/internal/worker/store/mem_store.go
index 06b72e7..1839348 100644
--- a/internal/worker/store/mem_store.go
+++ b/internal/worker/store/mem_store.go
@@ -42,6 +42,15 @@
return ms.cveRecords
}
+func (ms *MemStore) GetAllCVERecords(ctx context.Context) ([]*CVERecord, error) {
+ var rs []*CVERecord
+ for _, r := range ms.cveRecords {
+ rs = append(rs, r)
+ }
+ sort.Slice(rs, func(i, j int) bool { return rs[i].ID < rs[j].ID })
+ return rs, nil
+}
+
// CreateCommitUpdateRecord implements Store.CreateCommitUpdateRecord.
func (ms *MemStore) CreateCommitUpdateRecord(ctx context.Context, r *CommitUpdateRecord) error {
r.ID = fmt.Sprint(rand.Uint32())
@@ -78,6 +87,20 @@
return urs, nil
}
+// ListCVERecordsWithTriageState implements Store.ListCVERecordsWithTriageState.
+func (ms *MemStore) ListCVERecordsWithTriageState(_ context.Context, ts TriageState) ([]*CVERecord, error) {
+ var crs []*CVERecord
+ for _, r := range ms.cveRecords {
+ if r.TriageState == ts {
+ crs = append(crs, r)
+ }
+ }
+ sort.Slice(crs, func(i, j int) bool {
+ return crs[i].ID < crs[j].ID
+ })
+ return crs, nil
+}
+
// GetDirectoryHash implements Transaction.GetDirectoryHash.
func (ms *MemStore) GetDirectoryHash(_ context.Context, dir string) (string, error) {
return ms.dirHashes[dir], nil
diff --git a/internal/worker/store/store.go b/internal/worker/store/store.go
index c049474..2639294 100644
--- a/internal/worker/store/store.go
+++ b/internal/worker/store/store.go
@@ -135,6 +135,10 @@
// least recent.
ListCommitUpdateRecords(ctx context.Context, limit int) ([]*CommitUpdateRecord, error)
+ // ListCVERecordsWithTriageState returns all CVERecords with the given triage state,
+ // ordered by ID.
+ ListCVERecordsWithTriageState(ctx context.Context, ts TriageState) ([]*CVERecord, error)
+
// GetDirectoryHash returns the hash for the tree object corresponding to dir.
// If dir isn't found, it succeeds with the empty string.
GetDirectoryHash(ctx context.Context, dir string) (string, error)
diff --git a/internal/worker/store/store_test.go b/internal/worker/store/store_test.go
index e68b805..f667956 100644
--- a/internal/worker/store/store_test.go
+++ b/internal/worker/store/store_test.go
@@ -149,6 +149,12 @@
want.CVEState = cveschema.StateRejected
want.CommitHash = "999"
diff(t, &want, got)
+
+ gotNoAction, err := s.ListCVERecordsWithTriageState(ctx, TriageStateNoActionNeeded)
+ if err != nil {
+ t.Fatal(err)
+ }
+ diff(t, crs[1:], gotNoAction)
}
func testDirHashes(t *testing.T, s Store) {
