internal/upload/reports.go

// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package upload

import (
	"crypto/rand"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/telemetry/internal/config"
	"golang.org/x/telemetry/internal/counter"
	"golang.org/x/telemetry/internal/telemetry"
)

// reports generates reports from inactive count files
func (u *uploader) reports(todo *work) ([]string, error) {
	if mode, _ := u.dir.Mode(); mode == "off" {
		return nil, nil // no reports
	}
	thisInstant := u.startTime
	today := thisInstant.Format(time.DateOnly)
	lastWeek := latestReport(todo.uploaded)
	if lastWeek >= today { //should never happen
		lastWeek = ""
	}
	u.logger.Printf("Last week: %s, today: %s", lastWeek, today)
	countFiles := make(map[string][]string) // expiry date string->filenames
	earliest := make(map[string]time.Time)  // earliest begin time for any counter
	for _, f := range todo.countfiles {
		begin, end, err := u.counterDateSpan(f)
		if err != nil {
			// This shouldn't happen: we should have already skipped count files that
			// don't contain valid start or end times.
			u.logger.Printf("BUG: failed to parse expiry for collected count file: %v", err)
			continue
		}

		if end.Before(thisInstant) {
			expiry := end.Format(dateFormat)
			countFiles[expiry] = append(countFiles[expiry], f)
			if earliest[expiry].IsZero() || earliest[expiry].After(begin) {
				earliest[expiry] = begin
			}
		}
	}
	for expiry, files := range countFiles {
		if notNeeded(expiry, *todo) {
			u.logger.Printf("Files for %s not needed, deleting %v", expiry, files)
			// The report already exists.
			// There's another check in createReport.
			u.deleteFiles(files)
			continue
		}
		fname, err := u.createReport(earliest[expiry], expiry, files, lastWeek)
		if err != nil {
			u.logger.Printf("Failed to create report for %s: %v", expiry, err)
			continue
		}
		if fname != "" {
			u.logger.Printf("Ready to upload: %s", filepath.Base(fname))
			todo.readyfiles = append(todo.readyfiles, fname)
		}
	}
	return todo.readyfiles, nil
}

// latestReport returns the YYYY-MM-DD of the last report uploaded
// or the empty string if there are no reports.
func latestReport(uploaded map[string]bool) string {
	var latest string
	for name := range uploaded {
		if strings.HasSuffix(name, ".json") {
			if name > latest {
				latest = name
			}
		}
	}
	if latest == "" {
		return ""
	}
	// strip off the .json
	return latest[:len(latest)-len(".json")]
}

// notNeeded returns true if the report for date has already been created
func notNeeded(date string, todo work) bool {
	if todo.uploaded != nil && todo.uploaded[date+".json"] {
		return true
	}
	// maybe the report is already in todo.readyfiles
	for _, f := range todo.readyfiles {
		if strings.Contains(f, date) {
			return true
		}
	}
	return false
}

func (u *uploader) deleteFiles(files []string) {
	for _, f := range files {
		if err := os.Remove(f); err != nil {
			// this could be a race condition.
			// conversely, on Windows, err may be nil and
			// the file not deleted if anyone has it open.
			u.logger.Printf("%v failed to remove %s", err, f)
		}
	}
}

// createReport creates local and upload report files by
// combining all the count files for the expiryDate, and
// returns the upload report file's path.
// It may delete the count files once local and upload report
// files are successfully created.
func (u *uploader) createReport(start time.Time, expiryDate string, countFiles []string, lastWeek string) (string, error) {
	uploadOK := true
	mode, asof := u.dir.Mode()
	if mode != "on" {
		u.logger.Printf("No upload config or mode %q is not 'on'", mode)
		uploadOK = false // no config, nothing to upload
	}
	if u.tooOld(expiryDate, u.startTime) {
		u.logger.Printf("Expiry date %s is too old", expiryDate)
		uploadOK = false
	}
	// If the mode is recorded with an asof date, don't upload if the report
	// includes any data on or before the asof date.
	if !asof.IsZero() && !asof.Before(start) {
		u.logger.Printf("As-of date %s is not before start %s", asof, start)
		uploadOK = false
	}
	// TODO(rfindley): check that all the x.Meta are consistent for GOOS, GOARCH, etc.
	report := &telemetry.Report{
		Config:   u.configVersion,
		X:        computeRandom(), // json encodes all the bits
		Week:     expiryDate,
		LastWeek: lastWeek,
	}
	if report.X > u.config.SampleRate && u.config.SampleRate > 0 {
		u.logger.Printf("X: %f > SampleRate:%f, not uploadable", report.X, u.config.SampleRate)
		uploadOK = false
	}
	var succeeded bool
	for _, f := range countFiles {
		fok := false
		x, err := u.parseCountFile(f)
		if err != nil {
			u.logger.Printf("Unparseable count file %s: %v", filepath.Base(f), err)
			continue
		}
		prog := findProgReport(x.Meta, report)
		for k, v := range x.Count {
			if counter.IsStackCounter(k) {
				// stack
				prog.Stacks[k] += int64(v)
			} else {
				// counter
				prog.Counters[k] += int64(v)
			}
			succeeded = true
			fok = true
		}
		if !fok {
			u.logger.Printf("no counters found in %s", f)
		}
	}
	if !succeeded {
		return "", fmt.Errorf("none of the %d count files for %s contained counters", len(countFiles), expiryDate)
	}
	// 1. generate the local report
	localContents, err := json.MarshalIndent(report, "", " ")
	if err != nil {
		return "", fmt.Errorf("failed to marshal report for %s: %v", expiryDate, err)
	}
	// check that the report can be read back
	// TODO(pjw): remove for production?
	var report2 telemetry.Report
	if err := json.Unmarshal(localContents, &report2); err != nil {
		return "", fmt.Errorf("failed to unmarshal local report for %s: %v", expiryDate, err)
	}

	var uploadContents []byte
	if uploadOK {
		// 2. create the uploadable version
		cfg := config.NewConfig(u.config)
		upload := &telemetry.Report{
			Week:     report.Week,
			LastWeek: report.LastWeek,
			X:        report.X,
			Config:   report.Config,
		}
		for _, p := range report.Programs {
			// does the uploadConfig want this program?
			// if so, copy over the Stacks and Counters
			// that the uploadConfig mentions.
			if !cfg.HasGoVersion(p.GoVersion) || !cfg.HasProgram(p.Program) || !cfg.HasVersion(p.Program, p.Version) {
				continue
			}
			x := &telemetry.ProgramReport{
				Program:   p.Program,
				Version:   p.Version,
				GOOS:      p.GOOS,
				GOARCH:    p.GOARCH,
				GoVersion: p.GoVersion,
				Counters:  make(map[string]int64),
				Stacks:    make(map[string]int64),
			}
			upload.Programs = append(upload.Programs, x)
			for k, v := range p.Counters {
				if cfg.HasCounter(p.Program, k) && report.X <= cfg.Rate(p.Program, k) {
					x.Counters[k] = v
				}
			}
			// and the same for Stacks
			// this can be made more efficient, when it matters
			for k, v := range p.Stacks {
				before, _, _ := strings.Cut(k, "\n")
				if cfg.HasStack(p.Program, before) && report.X <= cfg.Rate(p.Program, before) {
					x.Stacks[k] = v
				}
			}
		}

		uploadContents, err = json.MarshalIndent(upload, "", " ")
		if err != nil {
			return "", fmt.Errorf("failed to marshal upload report for %s: %v", expiryDate, err)
		}
	}
	localFileName := filepath.Join(u.dir.LocalDir(), "local."+expiryDate+".json")
	uploadFileName := filepath.Join(u.dir.LocalDir(), expiryDate+".json")

	/* Prepare to write files */
	// if either file exists, someone has been here ahead of us
	// (there is still a race, but this check shortens the open window)
	if _, err := os.Stat(localFileName); err == nil {
		u.deleteFiles(countFiles)
		return "", fmt.Errorf("local report %s already exists", localFileName)
	}
	if _, err := os.Stat(uploadFileName); err == nil {
		u.deleteFiles(countFiles)
		return "", fmt.Errorf("report %s already exists", uploadFileName)
	}
	// write the uploadable file
	var errUpload, errLocal error
	if uploadOK {
		_, errUpload = exclusiveWrite(uploadFileName, uploadContents)
	}
	// write the local file
	_, errLocal = exclusiveWrite(localFileName, localContents)
	/*  Wrote the files */

	// even though these errors won't occur, what should happen
	// if errUpload == nil and it is ok to upload, and errLocal != nil?
	if errLocal != nil {
		return "", fmt.Errorf("failed to write local file %s (%v)", localFileName, errLocal)
	}
	if errUpload != nil {
		return "", fmt.Errorf("failed to write upload file %s (%v)", uploadFileName, errUpload)
	}
	u.logger.Printf("Created %s, deleting %d count files", filepath.Base(uploadFileName), len(countFiles))
	u.deleteFiles(countFiles)
	if uploadOK {
		return uploadFileName, nil
	}
	return "", nil
}

// exclusiveWrite attempts to create filename exclusively, and if successful,
// writes content to the resulting file handle.
//
// It returns a boolean indicating whether the exclusive handle was acquired,
// and an error indicating whether the operation succeeded.
// If the file already exists, exclusiveWrite returns (false, nil).
func exclusiveWrite(filename string, content []byte) (_ bool, rerr error) {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
	if err != nil {
		if os.IsExist(err) {
			return false, nil
		}
		return false, err
	}
	defer func() {
		if err := f.Close(); err != nil && rerr == nil {
			rerr = err
		}
	}()
	if _, err := f.Write(content); err != nil {
		return false, err
	}
	return true, nil
}

// return an existing ProgremReport, or create anew
func findProgReport(meta map[string]string, report *telemetry.Report) *telemetry.ProgramReport {
	for _, prog := range report.Programs {
		if prog.Program == meta["Program"] && prog.Version == meta["Version"] &&
			prog.GoVersion == meta["GoVersion"] && prog.GOOS == meta["GOOS"] &&
			prog.GOARCH == meta["GOARCH"] {
			return prog
		}
	}
	prog := telemetry.ProgramReport{
		Program:   meta["Program"],
		Version:   meta["Version"],
		GoVersion: meta["GoVersion"],
		GOOS:      meta["GOOS"],
		GOARCH:    meta["GOARCH"],
		Counters:  make(map[string]int64),
		Stacks:    make(map[string]int64),
	}
	report.Programs = append(report.Programs, &prog)
	return &prog
}

// computeRandom returns a cryptographic random float64 in the range [0, 1],
// with 52 bits of precision.
func computeRandom() float64 {
	for {
		b := make([]byte, 8)
		_, err := rand.Read(b)
		if err != nil {
			panic(fmt.Sprintf("rand.Read failed: %v", err))
		}
		// and turn it into a float64
		x := math.Float64frombits(binary.LittleEndian.Uint64(b))
		if math.IsNaN(x) || math.IsInf(x, 0) {
			continue
		}
		x = math.Abs(x)
		if x < 0x1p-1000 { // avoid underflow patterns
			continue
		}
		frac, _ := math.Frexp(x) // 52 bits of randomness
		return frac*2 - 1
	}
}