blob: 6c5d2b3942ac8901a51c8f150e8f294698028524 [file] [log] [blame]
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Telemetrygodev serves the telemetry.go.dev website.
package main
import (
"context"
_ "embed"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"log"
"net/http"
"os"
"strings"
"time"
"golang.org/x/exp/slog"
"golang.org/x/mod/semver"
"golang.org/x/telemetry/godev/internal/config"
"golang.org/x/telemetry/godev/internal/content"
ilog "golang.org/x/telemetry/godev/internal/log"
"golang.org/x/telemetry/godev/internal/middleware"
"golang.org/x/telemetry/godev/internal/storage"
"golang.org/x/telemetry/internal/chartconfig"
tconfig "golang.org/x/telemetry/internal/config"
contentfs "golang.org/x/telemetry/internal/content"
"golang.org/x/telemetry/internal/telemetry"
"golang.org/x/telemetry/internal/unionfs"
)
func main() {
flag.Parse()
ctx := context.Background()
cfg := config.NewConfig()
if cfg.UseGCS {
// We are likely running on GCP. Use GCP logging JSON format.
slog.SetDefault(slog.New(ilog.NewGCPLogHandler()))
}
handler := newHandler(ctx, cfg)
fmt.Printf("server listening at http://:%s\n", cfg.ServerPort)
log.Fatal(http.ListenAndServe(":"+cfg.ServerPort, handler))
}
// renderer implements shared template rendering for handlers below.
type renderer func(w http.ResponseWriter, tmpl string, page any) error
func newHandler(ctx context.Context, cfg *config.Config) http.Handler {
buckets, err := storage.NewAPI(ctx, cfg)
if err != nil {
log.Fatal(err)
}
ucfg, err := tconfig.ReadConfig(cfg.UploadConfig)
if err != nil {
log.Fatal(err)
}
fsys := fsys(cfg.DevMode)
mux := http.NewServeMux()
render := func(w http.ResponseWriter, tmpl string, page any) error {
return content.Template(w, fsys, tmpl, page, http.StatusOK)
}
logger := slog.Default()
// TODO(rfindley): use Go 1.22 routing once 1.23 is released and we can bump
// the go directive to 1.22.
mux.Handle("/", handleRoot(render, fsys, buckets.Chart, logger))
mux.Handle("/config", handleConfig(fsys, ucfg))
// TODO(rfindley): restrict this routing to POST
mux.Handle("/upload/", handleUpload(ucfg, buckets.Upload))
mux.Handle("/charts/", handleCharts(render, buckets.Chart))
mux.Handle("/data/", handleData(render, buckets.Merge))
mw := middleware.Chain(
middleware.Log(logger),
middleware.Timeout(cfg.RequestTimeout),
middleware.RequestSize(cfg.MaxRequestBytes),
middleware.Recover(),
)
return mw(mux)
}
// breadcrumb holds a breadcrumb nav element.
//
// If Link is empty, breadcrumbs are rendered as plain text.
type breadcrumb struct {
Link, Label string
}
type indexPage struct {
ChartTitle string
Charts map[string]any
ChartError string // if set, the error
}
func (indexPage) Breadcrumbs() []breadcrumb {
return []breadcrumb{{Link: "/", Label: "Go Telemetry"}, {Label: "Home"}}
}
func handleRoot(render renderer, fsys fs.FS, chartBucket storage.BucketHandle, log *slog.Logger) content.HandlerFunc {
// TODO(rfindley): handle static serving with a different route.
cserv := content.Server(fsys)
return func(w http.ResponseWriter, r *http.Request) error {
if r.URL.Path != "/" {
cserv.ServeHTTP(w, r)
return nil
}
page := indexPage{}
ctx := r.Context()
var (
chartDate string // end date of chart data
chartObj string // object name of chart file
)
it := chartBucket.Objects(ctx, "")
for {
obj, err := it.Next()
if errors.Is(err, storage.ErrObjectIteratorDone) {
break
} else if err != nil {
return err
}
date := strings.TrimSuffix(obj, ".json")
if date == obj {
// We have discussed eventually have nested subdirectories in the
// charts bucket. Defensively check for json files.
continue // not a chart object
}
// Chart objects may be for a single date (<date>.json), or for a date
// span (<start>_<end>.json).
_, end, aggregate := strings.Cut(date, "_")
if aggregate {
date = end
}
if date >= chartDate {
chartDate = date
// Prefer aggregate charts to daily charts, but consider the latest
// available date.
if aggregate || date > chartDate {
chartObj = obj
}
}
}
if chartObj == "" {
page.ChartError = "No data."
} else {
page.ChartTitle = chartTitle(chartObj)
charts, err := loadCharts(ctx, chartObj, chartBucket)
if err != nil {
log.ErrorContext(ctx, fmt.Sprintf("error loading index charts: %v", err))
page.ChartError = "Error loading charts."
} else {
page.Charts = charts
}
}
return render(w, "index.html", page)
}
}
func chartTitle(objName string) string {
start, end, aggregate := strings.Cut(strings.TrimSuffix(objName, ".json"), "_")
if aggregate {
return fmt.Sprintf("Aggregate charts for %s to %s", start, end)
}
return fmt.Sprintf("Charts for %s", start)
}
type chartsPage []string
func (chartsPage) Breadcrumbs() []breadcrumb {
return []breadcrumb{{Link: "/", Label: "Go Telemetry"}, {Label: "Charts"}}
}
func handleCharts(render renderer, chartBucket storage.BucketHandle) content.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) error {
ctx := r.Context()
if p := strings.TrimPrefix(r.URL.Path, "/charts/"); p != "" {
return handleChart(ctx, w, p, render, chartBucket)
}
it := chartBucket.Objects(ctx, "")
var page chartsPage
for {
obj, err := it.Next()
if errors.Is(err, storage.ErrObjectIteratorDone) {
break
} else if err != nil {
return err
}
date := strings.TrimSuffix(obj, ".json")
if date == obj {
continue // not a chart object
}
page = append(page, date)
}
return render(w, "allcharts.html", page)
}
}
type chartPage struct {
Date string
ChartTitle string
Charts map[string]any
}
func (p chartPage) Breadcrumbs() []breadcrumb {
return []breadcrumb{
{Link: "/", Label: "Go Telemetry"},
{Link: "/charts/", Label: "Charts"},
{Label: p.Date},
}
}
func handleChart(ctx context.Context, w http.ResponseWriter, date string, render renderer, chartBucket storage.BucketHandle) error {
// TODO(rfindley): refactor to return a content.HandlerFunc once we can use Go 1.22 routing.
page := chartPage{Date: date}
var err error
objName := date + ".json"
page.ChartTitle = chartTitle(objName)
page.Charts, err = loadCharts(ctx, objName, chartBucket)
if errors.Is(err, storage.ErrObjectNotExist) {
return content.Status(w, http.StatusNotFound)
} else if err != nil {
return err
}
return render(w, "charts.html", page)
}
type dataPage struct {
BucketURL string
Dates []string
}
func (dataPage) Breadcrumbs() []breadcrumb {
return []breadcrumb{{Link: "/", Label: "Go Telemetry"}, {Label: "Data"}}
}
func handleData(render renderer, mergeBucket storage.BucketHandle) content.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) error {
it := mergeBucket.Objects(r.Context(), "")
var page dataPage
page.BucketURL = mergeBucket.URI()
for {
obj, err := it.Next()
if errors.Is(err, storage.ErrObjectIteratorDone) {
break
} else if err != nil {
return err
}
date := strings.TrimSuffix(obj, ".json")
if date == obj {
continue // not a data object
}
page.Dates = append(page.Dates, date)
}
return render(w, "data.html", page)
}
}
func loadCharts(ctx context.Context, chartObj string, bucket storage.BucketHandle) (map[string]any, error) {
reader, err := bucket.Object(chartObj).NewReader(ctx)
if err != nil {
return nil, err
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
var charts map[string]any
if err := json.Unmarshal(data, &charts); err != nil {
return nil, err
}
return charts, nil
}
func handleUpload(ucfg *tconfig.Config, uploadBucket storage.BucketHandle) content.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) error {
if r.Method == "POST" {
ctx := r.Context()
var report telemetry.Report
if err := json.NewDecoder(r.Body).Decode(&report); err != nil {
return content.Error(fmt.Errorf("invalid JSON payload: %v", err), http.StatusBadRequest)
}
if err := validate(&report, ucfg); err != nil {
return content.Error(fmt.Errorf("invalid report: %v", err), http.StatusBadRequest)
}
// TODO: capture metrics for collisions.
name := fmt.Sprintf("%s/%g.json", report.Week, report.X)
f, err := uploadBucket.Object(name).NewWriter(ctx)
if err != nil {
return err
}
defer f.Close()
if err := json.NewEncoder(f).Encode(report); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return content.Status(w, http.StatusOK)
}
return content.Status(w, http.StatusMethodNotAllowed)
}
}
// validate validates the telemetry report data against the latest config.
func validate(r *telemetry.Report, cfg *tconfig.Config) error {
// TODO: reject/drop data arrived too early or too late.
if _, err := time.Parse(telemetry.DateOnly, r.Week); err != nil {
return fmt.Errorf("invalid week %s", r.Week)
}
if !semver.IsValid(r.Config) {
return fmt.Errorf("invalid config %s", r.Config)
}
if r.X == 0 {
return fmt.Errorf("invalid X %g", r.X)
}
// TODO: We can probably keep known programs and counters even when a report
// includes something that has been removed from the latest config.
for _, p := range r.Programs {
if !cfg.HasGOARCH(p.GOARCH) ||
!cfg.HasGOOS(p.GOOS) ||
!cfg.HasGoVersion(p.GoVersion) ||
!cfg.HasProgram(p.Program) ||
!cfg.HasVersion(p.Program, p.Version) {
return fmt.Errorf("unknown program build %s@%q %q %s/%s", p.Program, p.Version, p.GoVersion, p.GOOS, p.GOARCH)
}
for c := range p.Counters {
if !cfg.HasCounter(p.Program, c) {
return fmt.Errorf("unknown counter %s", c)
}
}
for s := range p.Stacks {
prefix, _, _ := strings.Cut(s, "\n")
if !cfg.HasStack(p.Program, prefix) {
return fmt.Errorf("unknown stack %s", s)
}
}
}
return nil
}
func fsys(fromOS bool) fs.FS {
var f fs.FS = contentfs.FS
if fromOS {
f = os.DirFS("internal/content")
contentfs.RunESBuild(true)
}
f, err := unionfs.Sub(f, "telemetrygodev", "shared")
if err != nil {
log.Fatal(err)
}
return f
}
type configPage struct {
Version string
ChartConfig string
UploadConfig string
}
func (configPage) Breadcrumbs() []breadcrumb {
return []breadcrumb{{Link: "/", Label: "Go Telemetry"}, {Label: "Upload Configuration"}}
}
func handleConfig(fsys fs.FS, ucfg *tconfig.Config) content.HandlerFunc {
ccfg := chartconfig.Raw()
cfg := ucfg.UploadConfig
version := "default"
return func(w http.ResponseWriter, r *http.Request) error {
cfgJSON, err := json.MarshalIndent(cfg, "", "\t")
if err != nil {
cfgJSON = []byte("unknown")
}
page := configPage{
Version: version,
ChartConfig: string(ccfg),
UploadConfig: string(cfgJSON),
}
return content.Template(w, fsys, "config.html", page, http.StatusOK)
}
}