internal/worker/pages.go

// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"bytes"
	"context"
	"errors"
	"io"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/google/safehtml/template"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/config"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/fetch"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/sync/errgroup"
)

type annotation struct {
	error
	msg string
}

var startTime = time.Now()

// doIndexPage writes the status page. On error it returns the error and a short
// string to be written back to the client.
func (s *Server) doIndexPage(w http.ResponseWriter, r *http.Request) (err error) {
	defer derrors.Wrap(&err, "doIndexPage")
	var (
		experiments []*internal.Experiment
		excluded    []string
	)
	if s.getExperiments != nil {
		experiments = s.getExperiments()
	}
	g, ctx := errgroup.WithContext(r.Context())
	g.Go(func() error {
		var err error
		excluded, err = s.db.GetExcludedPrefixes(ctx)
		if err != nil {
			return annotation{err, "error fetching excluded"}
		}
		return nil
	})
	if err := g.Wait(); err != nil {
		var e annotation
		if errors.As(err, &e) {
			log.Errorf(ctx, e.msg, err)
		}
		return err
	}

	var gms runtime.MemStats
	runtime.ReadMemStats(&gms)
	sms, err := getSystemMemStats()
	if err != nil {
		log.Errorf(ctx, "could not get system stats: %v", err)
	}
	pms, err := getProcessMemStats()
	if err != nil {
		log.Errorf(ctx, "could not get process stats: %v", err)
	}

	var logsURL string
	if s.cfg.OnGKE() {
		env := s.cfg.DeploymentEnvironment()
		cluster := env + "-" + "pkgsite"
		logsURL = `https://pantheon.corp.google.com/logs/query;query=resource.type%3D%22k8s_container%22%20resource.labels.cluster_name%3D%22` +
			cluster +
			`%22%20resource.labels.container_name%3D%22worker%22?project=` +
			s.cfg.ProjectID
	} else {
		logsURL = `https://cloud.google.com/console/logs/viewer?resource=gae_app%2Fmodule_id%2F` + s.cfg.ServiceID + `&project=` +
			s.cfg.ProjectID
	}
	page := struct {
		Config          *config.Config
		Env             string
		ResourcePrefix  string
		LatestTimestamp *time.Time
		LocationID      string
		Hostname        string
		StartTime       time.Time
		Experiments     []*internal.Experiment
		Excluded        []string
		LoadShedStats   fetch.LoadShedStats
		GoMemStats      runtime.MemStats
		ProcessStats    processMemStats
		SystemStats     systemMemStats
		CgroupStats     map[string]uint64
		Fetches         []*fetch.FetchInfo
		LogsURL         string
	}{
		Config:         s.cfg,
		Env:            env(s.cfg),
		ResourcePrefix: strings.ToLower(env(s.cfg)) + "-",
		LocationID:     s.cfg.LocationID,
		Hostname:       os.Getenv("HOSTNAME"),
		StartTime:      startTime,
		Experiments:    experiments,
		Excluded:       excluded,
		LoadShedStats:  fetch.ZipLoadShedStats(),
		GoMemStats:     gms,
		ProcessStats:   pms,
		SystemStats:    sms,
		CgroupStats:    getCgroupMemStats(),
		Fetches:        fetch.FetchInfos(),
		LogsURL:        logsURL,
	}
	return renderPage(ctx, w, page, s.templates[indexTemplate])
}

func (s *Server) doVersionsPage(w http.ResponseWriter, r *http.Request) (err error) {
	defer derrors.Wrap(&err, "doVersionsPage")
	const pageSize = 20
	g, ctx := errgroup.WithContext(r.Context())
	var (
		next, failures, recents []*internal.ModuleVersionState
		stats                   *postgres.VersionStats
	)
	g.Go(func() error {
		var err error
		next, err = s.db.GetNextModulesToFetch(ctx, pageSize)
		if err != nil {
			return annotation{err, "error fetching next versions"}
		}
		return nil
	})
	g.Go(func() error {
		var err error
		failures, err = s.db.GetRecentFailedVersions(ctx, pageSize)
		if err != nil {
			return annotation{err, "error fetching recent failures"}
		}
		return nil
	})
	g.Go(func() error {
		var err error
		recents, err = s.db.GetRecentVersions(ctx, pageSize)
		if err != nil {
			return annotation{err, "error fetching recent versions"}
		}
		return nil
	})
	g.Go(func() error {
		var err error
		stats, err = s.db.GetVersionStats(ctx)
		if err != nil {
			return annotation{err, "error fetching stats"}
		}
		return nil
	})
	if err := g.Wait(); err != nil {
		var e annotation
		if errors.As(err, &e) {
			log.Errorf(ctx, e.msg, err)
		}
		return err
	}

	type count struct {
		Code  int
		Desc  string
		Count int
	}
	var counts []*count
	for code, n := range stats.VersionCounts {
		c := &count{Code: code, Count: n}
		if e := derrors.FromStatus(code, ""); e != nil && e != derrors.Unknown {
			c.Desc = e.Error()
		}
		counts = append(counts, c)
	}
	sort.Slice(counts, func(i, j int) bool { return counts[i].Code < counts[j].Code })
	page := struct {
		Next, Recent, RecentFailures []*internal.ModuleVersionState
		Config                       *config.Config
		Env                          string
		ResourcePrefix               string
		LatestTimestamp              *time.Time
		Counts                       []*count
	}{
		Next:            next,
		Recent:          recents,
		RecentFailures:  failures,
		Config:          s.cfg,
		Env:             env(s.cfg),
		ResourcePrefix:  strings.ToLower(env(s.cfg)) + "-",
		LatestTimestamp: &stats.LatestTimestamp,
		Counts:          counts,
	}
	return renderPage(ctx, w, page, s.templates[versionsTemplate])
}

func env(cfg *config.Config) string {
	e := cfg.DeploymentEnvironment()
	return strings.ToUpper(e[:1]) + e[1:]
}

func renderPage(ctx context.Context, w http.ResponseWriter, page interface{}, tmpl *template.Template) (err error) {
	defer derrors.Wrap(&err, "renderPage")
	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, page); err != nil {
		return err
	}
	if _, err := io.Copy(w, &buf); err != nil {
		log.Errorf(ctx, "Error copying buffer to ResponseWriter: %v", err)
		return err
	}
	return nil
}