internal/worker: home page

Add information about noteworthy CVEs to the home page.

Change-Id: I86c30e333cc3ac45d5689928ba5a7d5dea457985
Reviewed-on: https://go-review.googlesource.com/c/vuln/+/368855
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/go.mod b/go.mod
index fddd1f5..df353c7 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@
 	github.com/google/safehtml v0.0.2
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/jba/templatecheck v0.6.0
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -36,6 +37,7 @@
 	golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57
 	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
@@ -49,5 +51,3 @@
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )
-
-require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
diff --git a/go.sum b/go.sum
index 26a3970..a1e2dc5 100644
--- a/go.sum
+++ b/go.sum
@@ -287,6 +287,8 @@
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jba/templatecheck v0.6.0 h1:SwM8C4hlK/YNLsdcXStfnHWE2HKkuTVwy5FKQHt5ro8=
+github.com/jba/templatecheck v0.6.0/go.mod h1:/1k7EajoSErFI9GLHAsiIJEaNLt3ALKNw2TV7z2SYv4=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
diff --git a/internal/worker/server.go b/internal/worker/server.go
index d4d3311..ae9fb81 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -9,12 +9,15 @@
 	"context"
 	"io"
 	"net/http"
+	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/google/safehtml/template"
 	"golang.org/x/exp/event"
+	"golang.org/x/sync/errgroup"
 	"golang.org/x/vuln/internal/derrors"
+	"golang.org/x/vuln/internal/gitrepo"
 	"golang.org/x/vuln/internal/worker/log"
 	"golang.org/x/vuln/internal/worker/store"
 )
@@ -45,16 +48,6 @@
 	return s, nil
 }
 
-func (s *Server) indexPage(w http.ResponseWriter, r *http.Request) error {
-	type data struct {
-		Namespace string
-	}
-	page := data{
-		Namespace: s.namespace,
-	}
-	return renderPage(r.Context(), w, page, s.indexTemplate)
-}
-
 func (s *Server) handle(ctx context.Context, pattern string, handler func(w http.ResponseWriter, r *http.Request) error) {
 	http.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
@@ -106,7 +99,27 @@
 		return nil, nil
 	}
 	templatePath := template.TrustedSourceJoin(staticPath, filename)
-	return template.New(filename.String()).ParseFilesFromTrustedSources(templatePath)
+	return template.New(filename.String()).Funcs(template.FuncMap{
+		"timefmt": formatTime,
+	}).ParseFilesFromTrustedSources(templatePath)
+}
+
+var locNewYork *time.Location
+
+func init() {
+	var err error
+	locNewYork, err = time.LoadLocation("America/New_York")
+	if err != nil {
+		log.Errorf(context.Background(), "time.LoadLocation: %v", err)
+		os.Exit(1)
+	}
+}
+
+func formatTime(t *time.Time) string {
+	if t == nil || t.IsZero() {
+		return "-"
+	}
+	return t.In(locNewYork).Format("2006-01-02 15:04:05")
 }
 
 func renderPage(ctx context.Context, w http.ResponseWriter, page interface{}, tmpl *template.Template) (err error) {
@@ -122,3 +135,48 @@
 	}
 	return nil
 }
+
+type indexPage struct {
+	CVEListRepoURL   string
+	Namespace        string
+	Updates          []*store.CommitUpdateRecord
+	CVEsNeedingIssue []*store.CVERecord
+	CVEsUpdatedSince []*store.CVERecord
+}
+
+func (s *Server) indexPage(w http.ResponseWriter, r *http.Request) error {
+
+	var (
+		updates                    []*store.CommitUpdateRecord
+		needingIssue, updatedSince []*store.CVERecord
+	)
+
+	g, ctx := errgroup.WithContext(r.Context())
+	g.Go(func() error {
+		var err error
+		updates, err = s.st.ListCommitUpdateRecords(ctx, 10)
+		return err
+	})
+	g.Go(func() error {
+		var err error
+		needingIssue, err = s.st.ListCVERecordsWithTriageState(ctx, store.TriageStateNeedsIssue)
+		return err
+	})
+	g.Go(func() error {
+		var err error
+		updatedSince, err = s.st.ListCVERecordsWithTriageState(ctx, store.TriageStateUpdatedSinceIssueCreation)
+		return err
+	})
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	page := indexPage{
+		CVEListRepoURL:   gitrepo.CVEListRepoURL,
+		Namespace:        s.namespace,
+		Updates:          updates,
+		CVEsNeedingIssue: needingIssue,
+		CVEsUpdatedSince: updatedSince,
+	}
+	return renderPage(r.Context(), w, page, s.indexTemplate)
+}
diff --git a/internal/worker/server_test.go b/internal/worker/server_test.go
new file mode 100644
index 0000000..c4e63b1
--- /dev/null
+++ b/internal/worker/server_test.go
@@ -0,0 +1,24 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+	"testing"
+
+	"github.com/google/safehtml/template"
+	"github.com/jba/templatecheck"
+)
+
+func TestTemplates(t *testing.T) {
+	// Check parsed templates.
+	staticPath := template.TrustedSourceFromConstant("static")
+	index, err := parseTemplate(staticPath, template.TrustedSourceFromConstant("index.tmpl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := templatecheck.CheckSafe(index, indexPage{}); err != nil {
+		t.Error(err)
+	}
+}
diff --git a/internal/worker/static/index.tmpl b/internal/worker/static/index.tmpl
index 85094b1..51d4a3a 100644
--- a/internal/worker/static/index.tmpl
+++ b/internal/worker/static/index.tmpl
@@ -13,8 +13,62 @@
 <body>
   <h1>{{.Namespace}} Vuln Worker</h1>
   <p>All times in America/New_York.</p>
+
+
+  <h2>Recent Updates</h2>
+  {{with .Updates}}
+    <table>
+      <tr>
+        <th>Started</th><th>Ended</th><th>Commit</th><th>Processed</th><th>Added</th><th>Modified</th><th>Error</th>
+      </tr>
+      {{range .}}
+        <tr>
+          <td>{{.StartedAt | timefmt}}</td>
+          <td>{{.EndedAt | timefmt}}</td>
+          <td><a href="{{$.CVEListRepoURL}}/tree/{{.CommitHash}}">{{.CommitHash}}</a></td>
+          <td>{{.NumProcessed}}/{{.NumTotal}}</td>
+          <td>{{.NumAdded}}</td>
+          <td>{{.NumModified}}</td>
+          <td>{{.Error}}</td>
+        </tr>
+      {{end}}
+    </table>
+  {{else}}
+    No updates.
+  {{end}}
+
+  <h2>CVEs Needing Issue</h2>
+  <p>{{len .CVEsNeedingIssue}} records.</p>
+  <table>
+    <tr>
+      <th>ID</th><th>Reason</th>
+    </tr>
+    {{range .CVEsNeedingIssue}}
+      <tr>
+        <td><a href="{{$.CVEListRepoURL}}/tree/{{.CommitHash}}/{{.Path}}">{{.ID}}</a></td>
+        <td>{{.TriageState}}</td>
+        <td>{{.TriageStateReason}}</td>
+      </tr>
+    {{end}}
+  </table>
+
+  <h2>CVEs Updated Since Issue Created</h2>
+  <p>{{len .CVEsUpdatedSince}} records.</p>
+  <table>
+    <tr>
+      <th>ID</th><th>Reason</th><th>Issue</th><th>Issue Created</th>
+    </tr>
+    {{range .CVEsUpdatedSince}}
+      <tr>
+        <td><a href="{{$.CVEListRepoURL}}/tree/{{.CommitHash}}/{{.Path}}">{{.ID}}</a></td>
+        <td>{{.TriageState}}</td>
+        <td>{{.TriageStateReason}}</td>
+        <td>{{.IssueReference}}</td>
+        <td>{{.IssueCreatedAt | timefmt}}</td>
+      </tr>
+    {{end}}
+  </table>
+    
 </body>
 </html>
 
-
-
diff --git a/internal/worker/store/fire_store.go b/internal/worker/store/fire_store.go
index 3c7b037..9e4dfbe 100644
--- a/internal/worker/store/fire_store.go
+++ b/internal/worker/store/fire_store.go
@@ -9,6 +9,7 @@
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"cloud.google.com/go/firestore"
 	"golang.org/x/vuln/internal/derrors"
@@ -118,6 +119,18 @@
 	Hash string
 }
 
+// ListCVERecordsWithTriageState implements Store.ListCVERecordsWithTriageState.
+func (fs *FireStore) ListCVERecordsWithTriageState(ctx context.Context, ts TriageState) (_ []*CVERecord, err error) {
+	defer derrors.Wrap(&err, "ListCVERecordsWithTriageState(%s)", ts)
+
+	q := fs.nsDoc.Collection(cveCollection).Where("TriageState", "==", ts).OrderBy("ID", firestore.Asc)
+	docsnaps, err := q.Documents(ctx).GetAll()
+	if err != nil {
+		return nil, err
+	}
+	return docsnapsToCVERecords(docsnaps)
+}
+
 // dirHashRef returns a DocumentRef for the directory dir.
 func (s *FireStore) dirHashRef(dir string) *firestore.DocumentRef {
 	// Firestore IDs cannot contain slashes.
@@ -154,6 +167,25 @@
 	return err
 }
 
+func (fs *FireStore) GetAllCVERecords(ctx context.Context) ([]*CVERecord, error) {
+	start := time.Now()
+	ds, err := fs.nsDoc.Collection(cveCollection).Select().Where("TriageState", "==", "NoActionNeeded").Documents(ctx).GetAll()
+	if err != nil {
+		return nil, err
+	}
+	fmt.Printf("#### where needsissue: %d recs in %s\n", len(ds), time.Since(start))
+	_ = ds
+
+	docsnaps, err := fs.nsDoc.Collection(cveCollection).
+		Select("ID", "TriageState").
+		Limit(100).
+		Documents(ctx).GetAll()
+	if err != nil {
+		return nil, err
+	}
+	return docsnapsToCVERecords(docsnaps)
+}
+
 // RunTransaction implements Store.RunTransaction.
 func (fs *FireStore) RunTransaction(ctx context.Context, f func(context.Context, Transaction) error) error {
 	return fs.client.RunTransaction(ctx,
@@ -163,8 +195,8 @@
 }
 
 // cveRecordRef returns a DocumentRef to the CVERecord with id.
-func (s *FireStore) cveRecordRef(id string) *firestore.DocumentRef {
-	return s.nsDoc.Collection(cveCollection).Doc(id)
+func (fs *FireStore) cveRecordRef(id string) *firestore.DocumentRef {
+	return fs.nsDoc.Collection(cveCollection).Doc(id)
 }
 
 // fsTransaction implements Transaction
@@ -206,6 +238,10 @@
 	if err != nil {
 		return nil, err
 	}
+	return docsnapsToCVERecords(docsnaps)
+}
+
+func docsnapsToCVERecords(docsnaps []*firestore.DocumentSnapshot) ([]*CVERecord, error) {
 	var crs []*CVERecord
 	for _, ds := range docsnaps {
 		var cr CVERecord
diff --git a/internal/worker/store/mem_store.go b/internal/worker/store/mem_store.go
index 06b72e7..1839348 100644
--- a/internal/worker/store/mem_store.go
+++ b/internal/worker/store/mem_store.go
@@ -42,6 +42,15 @@
 	return ms.cveRecords
 }
 
+func (ms *MemStore) GetAllCVERecords(ctx context.Context) ([]*CVERecord, error) {
+	var rs []*CVERecord
+	for _, r := range ms.cveRecords {
+		rs = append(rs, r)
+	}
+	sort.Slice(rs, func(i, j int) bool { return rs[i].ID < rs[j].ID })
+	return rs, nil
+}
+
 // CreateCommitUpdateRecord implements Store.CreateCommitUpdateRecord.
 func (ms *MemStore) CreateCommitUpdateRecord(ctx context.Context, r *CommitUpdateRecord) error {
 	r.ID = fmt.Sprint(rand.Uint32())
@@ -78,6 +87,20 @@
 	return urs, nil
 }
 
+// ListCVERecordsWithTriageState implements Store.ListCVERecordsWithTriageState.
+func (ms *MemStore) ListCVERecordsWithTriageState(_ context.Context, ts TriageState) ([]*CVERecord, error) {
+	var crs []*CVERecord
+	for _, r := range ms.cveRecords {
+		if r.TriageState == ts {
+			crs = append(crs, r)
+		}
+	}
+	sort.Slice(crs, func(i, j int) bool {
+		return crs[i].ID < crs[j].ID
+	})
+	return crs, nil
+}
+
 // GetDirectoryHash implements Transaction.GetDirectoryHash.
 func (ms *MemStore) GetDirectoryHash(_ context.Context, dir string) (string, error) {
 	return ms.dirHashes[dir], nil
diff --git a/internal/worker/store/store.go b/internal/worker/store/store.go
index c049474..2639294 100644
--- a/internal/worker/store/store.go
+++ b/internal/worker/store/store.go
@@ -135,6 +135,10 @@
 	// least recent.
 	ListCommitUpdateRecords(ctx context.Context, limit int) ([]*CommitUpdateRecord, error)
 
+	// ListCVERecordsWithTriageState returns all CVERecords with the given triage state,
+	// ordered by ID.
+	ListCVERecordsWithTriageState(ctx context.Context, ts TriageState) ([]*CVERecord, error)
+
 	// GetDirectoryHash returns the hash for the tree object corresponding to dir.
 	// If dir isn't found, it succeeds with the empty string.
 	GetDirectoryHash(ctx context.Context, dir string) (string, error)
diff --git a/internal/worker/store/store_test.go b/internal/worker/store/store_test.go
index e68b805..f667956 100644
--- a/internal/worker/store/store_test.go
+++ b/internal/worker/store/store_test.go
@@ -149,6 +149,12 @@
 	want.CVEState = cveschema.StateRejected
 	want.CommitHash = "999"
 	diff(t, &want, got)
+
+	gotNoAction, err := s.ListCVERecordsWithTriageState(ctx, TriageStateNoActionNeeded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	diff(t, crs[1:], gotNoAction)
 }
 
 func testDirHashes(t *testing.T, s Store) {