internal/configstore: add a package for upload config download
Telemetry upload config is a Go module (golang.org/x/telemetry/config).
that can be downloaded with `go mod download`. This allows telemetry
configs to be cacheable, and verifiable like other Go modules.
Moreover, 'go mod download' can download the config directly from
the source repository, so we don't need a separate config serving
infra.
internal/proxy is a helper that builds a file-system based Go
module proxy used for testing.
Change-Id: I299946943fce05561879dfb05addec47404d6a32
Reviewed-on: https://go-review.googlesource.com/c/telemetry/+/499255
Reviewed-by: Jamal Carvalho <jamal@golang.org>
Run-TryBot: Hyang-Ah Hana Kim <hyangah@gmail.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Auto-Submit: Hyang-Ah Hana Kim <hyangah@gmail.com>
diff --git a/go.mod b/go.mod
index f427f76..b0876d2 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,5 @@
module golang.org/x/telemetry
go 1.20
+
+require golang.org/x/mod v0.10.0
diff --git a/go.sum b/go.sum
index e69de29..0b96b7e 100644
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,2 @@
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
diff --git a/internal/configstore/download.go b/internal/configstore/download.go
new file mode 100644
index 0000000..f8c657c
--- /dev/null
+++ b/internal/configstore/download.go
@@ -0,0 +1,70 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package configstore abstracts interaction with the telemetry config server.
+// Telemetry config (golang.org/x/telemetry/config) is distributed as a go
+// module containing go.mod and config.json. Programs that upload collected
+// counters download the latest config using `go mod download`. This provides
+// verification of downloaded configuration and cacheability.
+package configstore
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+
+ "golang.org/x/telemetry"
+)
+
+const (
+ configModulePath = "golang.org/x/telemetry/config"
+ configFileName = "config.json"
+)
+
+// DownloadOption is an option for Download.
+type DownloadOption struct {
+ // Env holds the environment variables used when downloading the configuration.
+ // If nil, the process's environment variables are used.
+ Env []string
+}
+
+// Download fetches the requested telemetry UploadConfig using "go mod download".
+func Download(version string, opts *DownloadOption) (telemetry.UploadConfig, error) {
+ if version == "" {
+ version = "latest"
+ }
+ if opts == nil {
+ opts = &DownloadOption{}
+ }
+ modVer := configModulePath + "@" + version
+ var stdout, stderr bytes.Buffer
+ cmd := exec.Command("go", "mod", "download", "-json", modVer)
+ cmd.Env = opts.Env
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ return telemetry.UploadConfig{}, fmt.Errorf("failed to download config module: %w\n%s", err, &stderr)
+ }
+
+ var info struct {
+ Dir string
+ Version string
+ }
+ if err := json.Unmarshal(stdout.Bytes(), &info); err != nil || info.Dir == "" {
+ return telemetry.UploadConfig{}, fmt.Errorf("failed to download config module (invalid JSON): %w", err)
+ }
+ data, err := os.ReadFile(filepath.Join(info.Dir, configFileName))
+ if err != nil {
+ return telemetry.UploadConfig{}, fmt.Errorf("invalid config module: %w", err)
+ }
+ var cfg telemetry.UploadConfig
+ if err := json.Unmarshal(data, &cfg); err != nil {
+ return telemetry.UploadConfig{}, fmt.Errorf("invalid config: %w", err)
+ }
+ cfg.Version = info.Version
+ return cfg, nil
+}
diff --git a/internal/configstore/download_test.go b/internal/configstore/download_test.go
new file mode 100644
index 0000000..ec18880
--- /dev/null
+++ b/internal/configstore/download_test.go
@@ -0,0 +1,102 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package configstore
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "reflect"
+ "testing"
+
+ "golang.org/x/telemetry"
+ "golang.org/x/telemetry/internal/proxy"
+ "golang.org/x/telemetry/internal/testenv"
+)
+
+func TestDownload(t *testing.T) {
+ testenv.NeedsGo(t)
+ tmpdir := t.TempDir()
+ defer cleanModuleCache(t, tmpdir)
+
+ configVersion := "v0.0.0-20230526221463-e8d11ddaba41"
+ in := telemetry.UploadConfig{
+ GOOS: []string{"darwin"},
+ GOARCH: []string{"amd64", "arm64"},
+ GoVersion: []string{"1.20.3", "1.20.4"},
+ Programs: []*telemetry.ProgramConfig{{
+ Name: "gopls",
+ Versions: []string{"v0.11.0"},
+ Counters: []telemetry.CounterConfig{{
+ Name: "foobar",
+ Rate: 2,
+ }},
+ }},
+ }
+
+ proxyURI, err := writeConfig(tmpdir, in, configVersion)
+ if err != nil {
+ t.Fatal("failed to prepare proxy:", err)
+ }
+
+ opts := testDownloadOption(proxyURI, tmpdir)
+
+ got, err := Download("latest", opts)
+ if err != nil {
+ t.Fatal("failed to download the latest version:", err)
+ }
+
+ want := in
+ want.Version = configVersion // want the Version field to be populated with the module version.
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("Download(latest, _) = %v\nwant %v", stringify(got), stringify(want))
+ }
+}
+
+func stringify(x any) string {
+ ret, err := json.MarshalIndent(x, "", " ")
+ if err != nil {
+ return fmt.Sprintf("json.Marshal failed - %v", err)
+ }
+ return string(ret)
+}
+
+// writeConfig adds cfg to the module proxy used for testing.
+func writeConfig(dir string, cfg telemetry.UploadConfig, version string) (proxyURI string, _ error) {
+ encoded, err := json.Marshal(cfg)
+ if err != nil {
+ return "", err
+ }
+ dirPath := fmt.Sprintf("%v@%v/", configModulePath, version)
+ files := map[string][]byte{
+ dirPath + "go.mod": []byte("module " + configModulePath + "\n\ngo 1.20\n"),
+ dirPath + "config.json": encoded,
+ }
+ return proxy.WriteProxy(dir, files)
+}
+
+func testDownloadOption(proxyURI, tmpDir string) *DownloadOption {
+ var env []string
+ env = append(env, os.Environ()...)
+ env = append(env,
+ "GOPROXY="+proxyURI, // Use the fake proxy.
+ "GONOSUMDB=*", // Skip verifying checksum against sum.golang.org.
+ "GOMODCACHE="+tmpDir, // Don't pollute system module cache.
+ )
+ return &DownloadOption{
+ Env: env,
+ }
+}
+
+func cleanModuleCache(t *testing.T, tmpDir string) {
+ t.Helper()
+ cmd := exec.Command("go", "clean", "-modcache")
+ cmd.Env = append(cmd.Environ(), "GOMODCACHE="+tmpDir)
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Errorf("go clean -modcache failed: %v\n%s", err, out)
+ }
+}
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
new file mode 100644
index 0000000..9f5b677
--- /dev/null
+++ b/internal/proxy/proxy.go
@@ -0,0 +1,144 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package proxy provides functions for writing module data to a directory
+// in proxy format, so that it can be used as a module proxy by setting
+// GOPROXY="file://<dir>".
+// This is copied from golang.org/x/tools/gopls/internal/{proxydir,proxy}.
+package proxy
+
+import (
+ "archive/zip"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "golang.org/x/mod/module"
+)
+
+// WriteProxy creates a new proxy file tree using the txtar-encoded content,
+// and returns its URL.
+func WriteProxy(tmpdir string, files map[string][]byte) (string, error) {
+ type moduleVersion struct {
+ modulePath, version string
+ }
+ // Transform into the format expected by the proxydir package.
+ filesByModule := make(map[moduleVersion]map[string][]byte)
+ for name, data := range files {
+ modulePath, version, suffix := splitModuleVersionPath(name)
+ mv := moduleVersion{modulePath, version}
+ if _, ok := filesByModule[mv]; !ok {
+ filesByModule[mv] = make(map[string][]byte)
+ }
+ filesByModule[mv][suffix] = data
+ }
+ for mv, files := range filesByModule {
+ if err := writeModuleVersion(tmpdir, mv.modulePath, mv.version, files); err != nil {
+ return "", fmt.Errorf("error writing %s@%s: %v", mv.modulePath, mv.version, err)
+ }
+ }
+ return toURL(tmpdir), nil
+}
+
+// splitModuleVersionPath extracts module information from files stored in the
+// directory structure modulePath@version/suffix.
+// For example:
+//
+// splitModuleVersionPath("mod.com@v1.2.3/package") = ("mod.com", "v1.2.3", "package")
+func splitModuleVersionPath(path string) (modulePath, version, suffix string) {
+ parts := strings.Split(path, "/")
+ var modulePathParts []string
+ for i, p := range parts {
+ if strings.Contains(p, "@") {
+ mv := strings.SplitN(p, "@", 2)
+ modulePathParts = append(modulePathParts, mv[0])
+ return strings.Join(modulePathParts, "/"), mv[1], strings.Join(parts[i+1:], "/")
+ }
+ modulePathParts = append(modulePathParts, p)
+ }
+ // Default behavior: this is just a module path.
+ return path, "", ""
+}
+
+// writeModuleVersion creates a directory in the proxy dir for a module.
+func writeModuleVersion(rootDir, mod, ver string, files map[string][]byte) (rerr error) {
+ dir := filepath.Join(rootDir, mod, "@v")
+ if err := os.MkdirAll(dir, 0755); err != nil {
+ return err
+ }
+
+ // The go command checks for versions by looking at the "list" file. Since
+ // we are supporting multiple versions, create this file if it does not exist
+ // or append the version number to the preexisting file.
+
+ f, err := os.OpenFile(filepath.Join(dir, "list"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer checkClose("list file", f, &rerr)
+ if _, err := f.WriteString(ver + "\n"); err != nil {
+ return err
+ }
+
+ // Serve the go.mod file on the <version>.mod url, if it exists. Otherwise,
+ // serve a stub.
+ modContents, ok := files["go.mod"]
+ if !ok {
+ modContents = []byte("module " + mod)
+ }
+ if err := os.WriteFile(filepath.Join(dir, ver+".mod"), modContents, 0644); err != nil {
+ return err
+ }
+
+ // info file, just the bare bones.
+ infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, ver))
+ if err := os.WriteFile(filepath.Join(dir, ver+".info"), infoContents, 0644); err != nil {
+ return err
+ }
+
+ // zip of all the source files.
+ f, err = os.OpenFile(filepath.Join(dir, ver+".zip"), os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer checkClose("zip file", f, &rerr)
+ z := zip.NewWriter(f)
+ defer checkClose("zip writer", z, &rerr)
+ for name, contents := range files {
+ zf, err := z.Create(mod + "@" + ver + "/" + name)
+ if err != nil {
+ return err
+ }
+ if _, err := zf.Write(contents); err != nil {
+ return err
+ }
+ }
+
+ // Populate the /module/path/@latest that is used by @latest query.
+ if module.IsPseudoVersion(ver) {
+ latestFile := filepath.Join(rootDir, mod, "@latest")
+ if err := os.WriteFile(latestFile, infoContents, 0644); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func checkClose(name string, closer io.Closer, err *error) {
+ if cerr := closer.Close(); cerr != nil && *err == nil {
+ *err = fmt.Errorf("closing %s: %v", name, cerr)
+ }
+}
+
+// toURL returns the file uri for a proxy directory.
+func toURL(dir string) string {
+ // file URLs on Windows must start with file:///. See golang.org/issue/6027.
+ path := filepath.ToSlash(dir)
+ if !strings.HasPrefix(path, "/") {
+ path = "/" + path
+ }
+ return "file://" + path
+}
diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go
new file mode 100644
index 0000000..79ded1d
--- /dev/null
+++ b/internal/proxy/proxy_test.go
@@ -0,0 +1,109 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+ "archive/zip"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestWriteModuleVersion(t *testing.T) {
+ tests := []struct {
+ modulePath, version string
+ files map[string][]byte
+ }{
+ {
+ modulePath: "mod.test/module",
+ version: "v1.2.3",
+ files: map[string][]byte{
+ "go.mod": []byte("module mod.com\n\ngo 1.12"),
+ "const.go": []byte("package module\n\nconst Answer = 42"),
+ },
+ },
+ {
+ modulePath: "mod.test/module",
+ version: "v1.2.4",
+ files: map[string][]byte{
+ "go.mod": []byte("module mod.com\n\ngo 1.12"),
+ "const.go": []byte("package module\n\nconst Answer = 43"),
+ },
+ },
+ {
+ modulePath: "mod.test/nogomod",
+ version: "v0.9.0",
+ files: map[string][]byte{
+ "const.go": []byte("package module\n\nconst Other = \"Other\""),
+ },
+ },
+ }
+ dir := t.TempDir()
+ defer os.RemoveAll(dir)
+ for _, test := range tests {
+ // Since we later assert on the contents of /list, don't use subtests.
+ if err := writeModuleVersion(dir, test.modulePath, test.version, test.files); err != nil {
+ t.Fatal(err)
+ }
+ rootDir := filepath.Join(dir, filepath.FromSlash(test.modulePath), "@v")
+ gomod, err := os.ReadFile(filepath.Join(rootDir, test.version+".mod"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantMod, ok := test.files["go.mod"]
+ if !ok {
+ wantMod = []byte("module " + test.modulePath)
+ }
+ if got, want := string(gomod), string(wantMod); got != want {
+ t.Errorf("reading %s/@v/%s.mod: got %q, want %q", test.modulePath, test.version, got, want)
+ }
+ zr, err := zip.OpenReader(filepath.Join(rootDir, test.version+".zip"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer zr.Close()
+
+ for _, zf := range zr.File {
+ r, err := zf.Open()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer r.Close()
+ content, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatal(err)
+ }
+ name := strings.TrimPrefix(zf.Name, fmt.Sprintf("%s@%s/", test.modulePath, test.version))
+ if got, want := string(content), string(test.files[name]); got != want {
+ t.Errorf("unzipping %q: got %q, want %q", zf.Name, got, want)
+ }
+ delete(test.files, name)
+ }
+ for name := range test.files {
+ t.Errorf("file %q not present in the module zip", name)
+ }
+ }
+
+ lists := []struct {
+ modulePath, want string
+ }{
+ {"mod.test/module", "v1.2.3\nv1.2.4\n"},
+ {"mod.test/nogomod", "v0.9.0\n"},
+ }
+
+ for _, test := range lists {
+ fp := filepath.Join(dir, filepath.FromSlash(test.modulePath), "@v", "list")
+ list, err := os.ReadFile(fp)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := string(list); got != test.want {
+ t.Errorf("%q/@v/list: got %q, want %q", test.modulePath, got, test.want)
+ }
+ }
+}
