internal/upload/dates_test.go

// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package upload

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"golang.org/x/telemetry/counter"
	"golang.org/x/telemetry/internal/configtest"
	"golang.org/x/telemetry/internal/regtest"
	"golang.org/x/telemetry/internal/telemetry"
	"golang.org/x/telemetry/internal/testenv"
)

// CreateTestUploadServer creates a test server that records the uploaded data.
// The server is closed as part of cleaning up t.
func CreateTestUploadServer(t *testing.T) (*httptest.Server, func() [][]byte) {
	s := &uploadQueue{}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buf, err := io.ReadAll(r.Body)
		if err != nil {
			t.Errorf("invalid request received: %v", err)
			http.Error(w, "read failed", http.StatusBadRequest)
			return
		}
		s.Append(buf)
	}))
	t.Cleanup(srv.Close)
	return srv, s.Get
}

type uploadQueue struct {
	mu   sync.Mutex
	data [][]byte
}

func (s *uploadQueue) Append(data []byte) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.data = append(s.data, data)
}

func (s *uploadQueue) Get() [][]byte {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.data
}

func CreateTestUploadConfig(t *testing.T, counterNames, stackCounterNames []string) *telemetry.UploadConfig {
	goVersion, progPath, progVersion := regtest.ProgramInfo(t)
	GOOS, GOARCH := runtime.GOOS, runtime.GOARCH
	programConfig := &telemetry.ProgramConfig{
		Name:     progPath,
		Versions: []string{progVersion},
	}
	for _, c := range counterNames {
		programConfig.Counters = append(programConfig.Counters, telemetry.CounterConfig{Name: c, Rate: 1})
	}
	for _, c := range stackCounterNames {
		programConfig.Stacks = append(programConfig.Stacks, telemetry.CounterConfig{Name: c, Rate: 1, Depth: 16})
	}
	return &telemetry.UploadConfig{
		GOOS:       []string{GOOS},
		GOARCH:     []string{GOARCH},
		SampleRate: 1.0,
		GoVersion:  []string{goVersion},
		Programs:   []*telemetry.ProgramConfig{programConfig},
	}
}

func TestDates(t *testing.T) {
	testenv.SkipIfUnsupportedPlatform(t)

	prog := regtest.NewProgram(t, "prog", func() int {
		counter.Inc("testing")
		counter.NewStack("aStack", 4).Inc()
		return 0
	})

	// Run a fake program that produces a counter file in the telemetryDir.
	// readCountFileInfo will give us a template counter file content
	// based on the counter file.
	telemetryDir := t.TempDir()
	if out, err := regtest.RunProg(t, telemetryDir, prog); err != nil {
		t.Fatalf("failed to run program: %s", out)
	}
	cs := readCountFileInfo(t, filepath.Join(telemetryDir, "local"))
	uc := CreateTestUploadConfig(t, nil, []string{"aStack"})
	env := configtest.LocalProxyEnv(t, uc, "v1.2.3")

	const today = "2020-01-24"
	const yesterday = "2020-01-23"
	telemetryEnableTime, _ := time.Parse(dateFormat, "2019-12-01") // back-date the telemetry acceptance
	tests := []Test{                                               // each date must be different to subvert the parse cache
		{ // test that existing counters and ready files are not uploaded if they span data before telemetry was enabled
			name:   "beforefirstupload",
			today:  "2019-12-04",
			date:   "2019-12-03",
			begins: "2019-12-01",
			ends:   "2019-12-03",
			readys: []string{"2019-12-01", "2019-12-02"},
			// We get one local report: the newly created report.
			// It is not ready as it begins on the same day that telemetry was
			// enabled, and we err on the side of assuming it holds data from before
			// the user turned on uploading.
			wantLocal: 1,
			// The report for 2019-12-01 is still ready, because it was not uploaded.
			// This could happen in practice if the user disabled and then reenabled
			// telmetry.
			wantReady: 1,
			// The report for 2019-12-02 was uploaded.
			wantUploadeds: 1,
		},
		{ // test that existing counters and ready files are uploaded they only contain data after telemetry was enabled
			name:          "oktoupload",
			today:         "2019-12-10",
			date:          "2019-12-09",
			begins:        "2019-12-02",
			ends:          "2019-12-09",
			readys:        []string{"2019-12-07"},
			wantLocal:     1,
			wantUploadeds: 2, // Both new report and existing report are uploaded.
		},
		{ // test that an old countfile is removed and no reports generated
			name:   "oldcountfile",
			today:  today,
			date:   "2020-01-01",
			begins: "2020-01-01",
			ends:   olderThan(t, today, distantPast, "oldcountfile"),
			// one local; readys, uploads are empty, and there should be nothing left
			wantLocal: 1,
		},
		{ // test that a count file expiring today is left alone
			name:       "todayscountfile",
			today:      today,
			date:       "2020-01-02",
			begins:     "2020-01-08",
			ends:       today,
			wantCounts: 1,
		},
		{ // test that a count file expiring yesterday generates reports
			name:          "yesterdaycountfile",
			today:         today,
			date:          "2020-01-03",
			begins:        "2020-01-16",
			ends:          yesterday,
			wantLocal:     1,
			wantUploadeds: 1,
		},
		{ // count file already has local report, remove count file
			name:       "alreadydonelocal",
			today:      today,
			date:       "2020-01-04",
			begins:     "2020-01-16",
			ends:       yesterday,
			locals:     []string{yesterday},
			wantCounts: 0,
			wantLocal:  1,
		},
		{ // count file already has upload report, remove count file
			name:          "alreadydoneuploaded",
			today:         today,
			date:          "2020-01-05",
			begins:        "2020-01-16",
			ends:          "2020-01-23",
			uploads:       []string{"2020-01-23"},
			wantCounts:    0, // count file has been used, remove it
			wantLocal:     0, // no local report generated
			wantUploadeds: 1, // the existing uploaded report
		},
		{ // for some reason there's a ready file in the future, don't upload it
			name:       "futurereadyfile",
			today:      "2020-01-24",
			date:       "2020-01-06",
			begins:     "2020-01-16",
			ends:       "2020-01-24", // count file not expired
			readys:     []string{"2020-01-25"},
			wantCounts: 1, // active count file
			wantReady:  1, // existing premature ready file
		},
	}

	for _, tx := range tests {
		t.Run(tx.name, func(t *testing.T) {
			telemetryDir := t.TempDir()

			srv, uploaded := CreateTestUploadServer(t)

			dbg := filepath.Join(telemetryDir, "debug")
			if err := os.MkdirAll(dbg, 0777); err != nil {
				t.Fatal(err)
			}
			uploader, err := newUploader(RunConfig{
				TelemetryDir: telemetryDir,
				UploadURL:    srv.URL,
				Env:          env,
			})
			if err != nil {
				t.Fatal(err)
			}
			defer uploader.Close()
			if err := uploader.dir.SetModeAsOf("on", telemetryEnableTime); err != nil {
				t.Fatal(err)
			}
			uploader.startTime = mustParseDate(tx.today)

			wantUploadCount := doTest(t, uploader, &tx, cs)
			if got := len(uploaded()); wantUploadCount != got {
				t.Errorf("server got %d upload requests, want %d", got, wantUploadCount)
			}
		})
	}
}

func mustParseDate(d string) time.Time {
	x, err := time.Parse("2006-01-02", d)
	if err != nil {
		log.Fatalf("couldn't parse time %s", d)
	}
	return x
}

// return a day more than 'old' before 'today'
func olderThan(t *testing.T, today string, old time.Duration, nm string) string {
	x, err := time.Parse("2006-01-02", today)
	if err != nil {
		t.Errorf("%q not a day in test %s (%v)", today, nm, err)
		return today // so test should fail
	}
	ans := x.Add(-old - 24*time.Hour)
	msg := ans.Format("2006-01-02")
	return msg
}

// Test is a single test.
//
// All dates are in YYYY-MM-DD format.
type Test struct {
	name  string // the test name; only used for descriptive output
	today string // the date of the fake upload
	// count file
	date         string // the date in of the upload file name; must be unique among tests
	begins, ends string // the begin and end date stored in the counter metadata

	// Dates of load reports in the local dir.
	locals []string

	// Dates of upload reports in the local dir.
	readys []string

	// Dates of reports already uploaded.
	uploads []string

	// number of expected results
	wantCounts    int
	wantReady     int
	wantLocal     int
	wantUploadeds int
}

// Information from the counter file so its contents can be
// modified for tests
type countFileInfo struct {
	beginOffset, endOffset int    // where the dates are in the file
	buf                    []byte // counter file contents
	namePrefix             string // the part of its name before the date
	originalName           string // its original name
}

// return useful information from the counter file to be used
// in creating tests. also compute and return the UploadConfig
// Note that there must be exactly one counter file in localDir.
func readCountFileInfo(t *testing.T, localDir string) *countFileInfo {
	fis, err := os.ReadDir(localDir)
	if err != nil {
		t.Fatal(err)
	}
	var countFileName string
	var countFileBuf []byte
	for _, f := range fis {
		if strings.HasSuffix(f.Name(), ".count") {
			countFileName = filepath.Join(localDir, f.Name())
			buf, err := os.ReadFile(countFileName)
			if err != nil {
				t.Fatal(err)
			}
			countFileBuf = buf
			break
		}
	}
	if len(countFileBuf) == 0 {
		t.Fatalf("no contents read for %s", countFileName)
	}

	var cfilename string = countFileName
	cfilename = filepath.Base(cfilename)
	flds := strings.Split(cfilename, "-")
	if len(flds) != 7 {
		t.Fatalf("got %d fields, expected 7 (%q)", len(flds), cfilename)
	}
	pr := strings.Join(flds[:4], "-") + "-"

	ans := countFileInfo{
		buf:          countFileBuf,
		namePrefix:   pr,
		originalName: countFileName,
	}
	idx := bytes.Index(countFileBuf, []byte("TimeEnd: "))
	if idx < 0 {
		t.Fatalf("couldn't find TimeEnd in count file %q", countFileBuf[:100])
	}
	ans.endOffset = idx + len("TimeEnd: ")
	idx = bytes.Index(countFileBuf, []byte("TimeBegin: "))
	if idx < 0 {
		t.Fatalf("couldn't find TimeBegin in countfile %q", countFileBuf[:100])
	}
	ans.beginOffset = idx + len("TimeBegin: ")
	return &ans
}

func doTest(t *testing.T, u *uploader, test *Test, known *countFileInfo) int {
	// set up directory contents
	contents := bytes.Join([][]byte{
		known.buf[:known.beginOffset],
		[]byte(test.begins),
		known.buf[known.beginOffset+len("YYYY-MM-DD") : known.endOffset],
		[]byte(test.ends),
		known.buf[known.endOffset+len("YYYY-MM-DD"):],
	}, nil)
	filename := known.namePrefix + test.date + ".v1.count"
	if err := os.MkdirAll(u.dir.LocalDir(), 0777); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(u.dir.LocalDir(), filename), contents, 0666); err != nil {
		t.Fatalf("writing count file for %s (%s): %v", test.name, filename, err)
	}
	for _, x := range test.locals {
		nm := fmt.Sprintf("local.%s.json", x)
		if err := os.WriteFile(filepath.Join(u.dir.LocalDir(), nm), []byte{}, 0666); err != nil {
			t.Fatalf("%v writing local file %s", err, nm)
		}
	}
	for _, x := range test.readys {
		nm := fmt.Sprintf("%s.json", x)
		if err := os.WriteFile(filepath.Join(u.dir.LocalDir(), nm), []byte{}, 0666); err != nil {
			t.Fatalf("%v writing ready file %s", err, nm)
		}
	}
	if len(test.uploads) > 0 {
		os.MkdirAll(u.dir.UploadDir(), 0777)
	}
	for _, x := range test.uploads {
		nm := fmt.Sprintf("%s.json", x)
		if err := os.WriteFile(filepath.Join(u.dir.UploadDir(), nm), []byte{}, 0666); err != nil {
			t.Fatalf("%v writing upload %s", err, nm)
		}
	}

	// run
	u.Run()

	// check results
	var cfiles, rfiles, lfiles, ufiles int
	fis, err := os.ReadDir(u.dir.LocalDir())
	if err != nil {
		t.Errorf("%v reading localdir %s", err, u.dir.LocalDir())
		return 0
	}
	for _, f := range fis {
		switch {
		case strings.HasSuffix(f.Name(), ".v1.count"):
			cfiles++
		case f.Name() == "weekends": // ok
		case strings.HasPrefix(f.Name(), "local."):
			lfiles++
		case strings.HasSuffix(f.Name(), ".json"):
			rfiles++
		default:
			t.Errorf("for %s, unexpected local file %s", test.name, f.Name())
		}
	}

	var logcnt int
	logs, err := os.ReadDir(u.dir.DebugDir())
	if err == nil {
		logcnt = len(logs)
	}
	if logcnt != 1 {
		t.Errorf("expected 1 log file, got %d", logcnt)
	}

	fis, err = os.ReadDir(u.dir.UploadDir())
	if err != nil {
		t.Errorf("%v reading uploaddir %s", err, u.dir.UploadDir())
		return 0
	}
	ufiles = len(fis) // assume there's nothing but .json reports
	if test.wantCounts != cfiles {
		t.Errorf("%s: got %d countfiles, wanted %d", test.name, cfiles, test.wantCounts)
	}
	if test.wantReady != rfiles {
		t.Errorf("%s: got %d ready files, wanted %d", test.name, rfiles, test.wantReady)
	}
	if test.wantLocal != lfiles {
		t.Errorf("%s: got %d localfiles, wanted %d", test.name, lfiles, test.wantLocal)
	}
	if test.wantUploadeds != ufiles {
		t.Errorf("%s: got %d uploaded files, wanted %d", test.name, ufiles, test.wantUploadeds)
	}
	return ufiles - len(test.uploads)
}