internal/relui: support uploading to CDN and publishing to go.dev
Port over the -upload mode of cmd/release. We'll need a website change
to allow the "relui" user to publish.
For golang/go#51797.
Change-Id: Ic9e059ef14253f972a1709ea6c37d2a73f8622fd
Reviewed-on: https://go-review.googlesource.com/c/build/+/410234
Run-TryBot: Heschi Kreinick <heschi@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Alex Rakoczy <alex@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Auto-Submit: Heschi Kreinick <heschi@google.com>
diff --git a/cmd/relui/deployment-prod.yaml b/cmd/relui/deployment-prod.yaml
index 9d052a4..09c8bef 100644
--- a/cmd/relui/deployment-prod.yaml
+++ b/cmd/relui/deployment-prod.yaml
@@ -37,6 +37,9 @@
- "--builder-master-key=secret:symbolic-datum-552/builder-master-key"
- "--scratch-files-base=gs://golang-release-staging/relui-scratch"
- "--staging-files-base=gs://golang-release-staging"
+ - "--serving-files-base=gs://golang"
+ - "--edge-cache-url=https://dl.google.com/go"
+ - "--website-upload-url=https://go.dev/dl/upload"
ports:
- containerPort: 444
env:
diff --git a/cmd/relui/main.go b/cmd/relui/main.go
index 9380c96..671dc67 100644
--- a/cmd/relui/main.go
+++ b/cmd/relui/main.go
@@ -6,14 +6,18 @@
package main
import (
+ "bytes"
"context"
"crypto/hmac"
"crypto/md5"
+ "encoding/json"
"flag"
"fmt"
"io"
+ "io/ioutil"
"log"
"math/rand"
+ "net/http"
"net/url"
"time"
@@ -39,6 +43,9 @@
scratchFilesBase = flag.String("scratch-files-base", "", "Storage for scratch files. gs://bucket/path or file:///path/to/scratch.")
stagingFilesBase = flag.String("staging-files-base", "", "Storage for staging files. gs://bucket/path or file:///path/to/staging.")
+ servingFilesBase = flag.String("serving-files-base", "", "Storage for serving files. gs://bucket/path or file:///path/to/serving.")
+ edgeCacheURL = flag.String("edge-cache-url", "", "URL release files appear at when published to the CDN, e.g. https://dl.google.com/go.")
+ websiteUploadURL = flag.String("website-upload-url", "", "URL to POST website file data to, e.g. https://go.dev/dl/upload.")
)
func main() {
@@ -89,11 +96,12 @@
dh := relui.NewDefinitionHolder()
relui.RegisterMailDLCLDefinition(dh, extCfg)
relui.RegisterTweetDefinitions(dh, extCfg)
+ userPassAuth := buildlet.UserPass{
+ Username: "user-relui",
+ Password: key(*masterKey, "user-relui"),
+ }
coordinator := &buildlet.CoordinatorClient{
- Auth: buildlet.UserPass{
- Username: "user-relui",
- Password: key(*masterKey, "user-relui"),
- },
+ Auth: userPassAuth,
Instance: build.ProdCoordinator,
}
if _, err := coordinator.RemoteBuildlets(); err != nil {
@@ -109,6 +117,11 @@
GCSClient: gcsClient,
ScratchURL: *scratchFilesBase,
StagingURL: *stagingFilesBase,
+ ServingURL: *servingFilesBase,
+ DownloadURL: *edgeCacheURL,
+ PublishFile: func(f *relui.WebsiteFile) error {
+ return publishFile(*websiteUploadURL, userPassAuth, f)
+ },
}
releaseTasks.RegisterBuildReleaseWorkflows(dh)
db, err := pgxpool.Connect(ctx, *pgConnect)
@@ -140,3 +153,26 @@
io.WriteString(h, principal)
return fmt.Sprintf("%x", h.Sum(nil))
}
+
+func publishFile(uploadURL string, auth buildlet.UserPass, f *relui.WebsiteFile) error {
+ req, err := json.Marshal(f)
+ if err != nil {
+ return err
+ }
+ u, err := url.Parse(uploadURL)
+ if err != nil {
+ return fmt.Errorf("invalid website upload URL %q: %v", *websiteUploadURL, err)
+ }
+ u.Query().Set("user", auth.Username)
+ u.Query().Set("key", auth.Password)
+ resp, err := http.Post(u.String(), "application/json", bytes.NewReader(req))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ b, _ := ioutil.ReadAll(resp.Body)
+ return fmt.Errorf("upload failed to %q: %v\n%s", uploadURL, resp.Status, b)
+ }
+ return nil
+}
diff --git a/internal/relui/buildrelease_test.go b/internal/relui/buildrelease_test.go
index 333251e..669ce14 100644
--- a/internal/relui/buildrelease_test.go
+++ b/internal/relui/buildrelease_test.go
@@ -27,8 +27,10 @@
"time"
"github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/uuid"
"golang.org/x/build/buildlet"
+ "golang.org/x/build/internal"
"golang.org/x/build/internal/untar"
"golang.org/x/build/internal/workflow"
)
@@ -39,23 +41,49 @@
if runtime.GOOS != "linux" {
t.Skip("Requires bash shell scripting support.")
}
- s := httptest.NewServer(http.HandlerFunc(serveTarballs))
- defer s.Close()
+
+ // Set up a server that will be used to serve inputs to the build.
+ tarballServer := httptest.NewServer(http.HandlerFunc(serveTarballs))
+ defer tarballServer.Close()
fakeBuildlets := &fakeBuildlets{
t: t,
dir: t.TempDir(),
- httpURL: s.URL,
+ httpURL: tarballServer.URL,
logs: map[string][]*[]string{},
}
+
+ // Set up the fake signing process.
stagingDir := t.TempDir()
go fakeSign(ctx, t, filepath.Join(stagingDir, "go1.18releasetest1"))
signingPollDuration = 100 * time.Millisecond
+
+ // Set up the fake CDN publishing process.
+ servingDir := t.TempDir()
+ dlDir := t.TempDir()
+ dlServer := httptest.NewServer(http.FileServer(http.FS(os.DirFS(dlDir))))
+ defer dlServer.Close()
+ go fakeCDNLoad(ctx, t, servingDir, dlDir)
+ uploadPollDuration = 100 * time.Millisecond
+
+ // Set up the fake website to publish to.
+ var filesMu sync.Mutex
+ files := map[string]*WebsiteFile{}
+ publishFile := func(f *WebsiteFile) error {
+ filesMu.Lock()
+ defer filesMu.Unlock()
+ files[f.Filename] = f
+ return nil
+ }
+
tasks := BuildReleaseTasks{
- GerritURL: s.URL,
+ GerritURL: tarballServer.URL,
GCSClient: nil,
ScratchURL: "file://" + filepath.ToSlash(t.TempDir()),
StagingURL: "file://" + filepath.ToSlash(stagingDir),
+ ServingURL: "file://" + filepath.ToSlash(servingDir),
CreateBuildlet: fakeBuildlets.createBuildlet,
+ DownloadURL: dlServer.URL,
+ PublishFile: publishFile,
}
wd, err := tasks.newBuildReleaseWorkflow("go1.18")
if err != nil {
@@ -69,38 +97,59 @@
if err != nil {
t.Fatal(err)
}
- out, err := w.Run(ctx, &verboseListener{t})
+ _, err = w.Run(ctx, &verboseListener{t})
if err != nil {
t.Fatal(err)
}
- artifacts := out["Signed artifacts"].([]artifact)
- byName := map[string]artifact{}
- for _, a := range artifacts {
- if a.sha256 == "" || a.size < 1 || a.suffix == "" || a.filename == "" {
- t.Errorf("release process produced an invalid artifact: %#v", a)
+ for _, f := range files {
+ if f.ChecksumSHA256 == "" || f.Size < 1 || f.Filename == "" || f.Kind == "" {
+ t.Errorf("release process produced an invalid artifact: %#v", f)
}
- byName[a.filename] = a
}
- checkTGZ(t, stagingDir, byName["go1.18releasetest1.src.tar.gz"], map[string]string{
+ checkTGZ(t, dlDir, files, "go1.18releasetest1.src.tar.gz", &WebsiteFile{
+ OS: "",
+ Arch: "",
+ Kind: "source",
+ }, map[string]string{
"go/VERSION": "go1.18releasetest1",
"go/src/make.bash": makeScript,
})
- checkContents(t, stagingDir, byName["go1.18releasetest1.windows-amd64.msi"], "I'm an MSI!\n")
- checkTGZ(t, stagingDir, byName["go1.18releasetest1.linux-amd64.tar.gz"], map[string]string{
+ checkContents(t, dlDir, files, "go1.18releasetest1.windows-amd64.msi", &WebsiteFile{
+ OS: "windows",
+ Arch: "amd64",
+ Kind: "installer",
+ }, "I'm an MSI!\n")
+ checkTGZ(t, dlDir, files, "go1.18releasetest1.linux-amd64.tar.gz", &WebsiteFile{
+ OS: "linux",
+ Arch: "amd64",
+ Kind: "archive",
+ }, map[string]string{
"go/VERSION": "go1.18releasetest1",
"go/tool/something_orother/compile": "",
"go/pkg/something_orother/race.a": "",
})
- checkZip(t, stagingDir, byName["go1.18releasetest1.windows-arm64.zip"], map[string]string{
+ checkZip(t, dlDir, files, "go1.18releasetest1.windows-arm64.zip", &WebsiteFile{
+ OS: "windows",
+ Arch: "arm64",
+ Kind: "archive",
+ }, map[string]string{
"go/VERSION": "go1.18releasetest1",
"go/tool/something_orother/compile": "",
})
- checkZip(t, stagingDir, byName["go1.18releasetest1.windows-386.zip"], map[string]string{
+ checkTGZ(t, dlDir, files, "go1.18releasetest1.linux-armv6l.tar.gz", &WebsiteFile{
+ OS: "linux",
+ Arch: "armv6l",
+ Kind: "archive",
+ }, map[string]string{
"go/VERSION": "go1.18releasetest1",
"go/tool/something_orother/compile": "",
})
- checkContents(t, stagingDir, byName["go1.18releasetest1.darwin-amd64.pkg"], "I'm a .pkg!\n")
+ checkContents(t, dlDir, files, "go1.18releasetest1.darwin-amd64.pkg", &WebsiteFile{
+ OS: "darwin",
+ Arch: "amd64",
+ Kind: "installer",
+ }, "I'm a .pkg!\n")
// TODO: consider logging this to golden files?
for name, logs := range fakeBuildlets.logs {
@@ -195,27 +244,36 @@
}
}
-func checkContents(t *testing.T, stagingDir string, a artifact, contents string) {
- t.Run(a.filename, func(t *testing.T) {
- b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.signedPath))
- if err != nil {
- t.Fatalf("reading %v: %v", a.filename, err)
+func checkFile(t *testing.T, dlDir string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, check func([]byte)) {
+ t.Run(filename, func(t *testing.T) {
+ f, ok := files[filename]
+ if !ok {
+ t.Fatalf("file %q not published", filename)
}
+ if diff := cmp.Diff(meta, f, cmpopts.IgnoreFields(WebsiteFile{}, "Filename", "Version", "ChecksumSHA256", "Size")); diff != "" {
+ t.Errorf("file metadata mismatch (-want +got):\n%v", diff)
+ }
+ b, err := ioutil.ReadFile(filepath.Join(dlDir, f.Filename))
+ if err != nil {
+ t.Fatalf("reading %v: %v", f.Filename, err)
+ }
+ check(b)
+ })
+}
+
+func checkContents(t *testing.T, dlDir string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents string) {
+ checkFile(t, dlDir, files, filename, meta, func(b []byte) {
if got, want := string(b), contents; got != want {
- t.Errorf("%v contains %q, want %q", a.filename, got, want)
+ t.Errorf("%v contains %q, want %q", filename, got, want)
}
})
}
-func checkTGZ(t *testing.T, stagingDir string, a artifact, contents map[string]string) {
- t.Run(a.filename, func(t *testing.T) {
- b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.signedPath))
- if err != nil {
- t.Fatalf("reading %v: %v", a.filename, err)
- }
+func checkTGZ(t *testing.T, dlDir string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) {
+ checkFile(t, dlDir, files, filename, meta, func(b []byte) {
gzr, err := gzip.NewReader(bytes.NewReader(b))
if err != nil {
- t.Fatalf("unzipping %v: %v", a.filename, err)
+ t.Fatal(err)
}
tr := tar.NewReader(gzr)
for {
@@ -245,12 +303,8 @@
})
}
-func checkZip(t *testing.T, stagingDir string, a artifact, contents map[string]string) {
- t.Run(a.filename, func(t *testing.T) {
- b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.signedPath))
- if err != nil {
- t.Fatalf("reading %v: %v", a.filename, err)
- }
+func checkZip(t *testing.T, dlDir string, files map[string]*WebsiteFile, filename string, meta *WebsiteFile, contents map[string]string) {
+ checkFile(t, dlDir, files, filename, meta, func(b []byte) {
zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
if err != nil {
t.Fatal(err)
@@ -489,15 +543,9 @@
// fakeSign acts like a human running the signbinaries job periodically.
func fakeSign(ctx context.Context, t *testing.T, dir string) {
seen := map[string]bool{}
- for {
- select {
- case <-ctx.Done():
- return
- case <-time.After(500 * time.Millisecond):
- }
-
+ internal.PeriodicallyDo(ctx, 100*time.Millisecond, func(_ context.Context, _ time.Time) {
fakeSignOnce(t, dir, seen)
- }
+ })
}
func fakeSignOnce(t *testing.T, dir string, seen map[string]bool) {
@@ -645,3 +693,26 @@
t.Errorf("signed outputs mismatch (-want +got):\n%v", diff)
}
}
+
+func fakeCDNLoad(ctx context.Context, t *testing.T, from, to string) {
+ seen := map[string]bool{}
+ internal.PeriodicallyDo(ctx, 100*time.Millisecond, func(_ context.Context, _ time.Time) {
+ files, err := os.ReadDir(from)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, f := range files {
+ if seen[f.Name()] {
+ continue
+ }
+ seen[f.Name()] = true
+ contents, err := os.ReadFile(filepath.Join(from, f.Name()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(to, f.Name()), contents, 0777); err != nil {
+ t.Fatal(err)
+ }
+ }
+ })
+}
diff --git a/internal/relui/workflows.go b/internal/relui/workflows.go
index 620cb1a..deec414 100644
--- a/internal/relui/workflows.go
+++ b/internal/relui/workflows.go
@@ -10,6 +10,7 @@
"io"
"io/fs"
"math/rand"
+ "net/http"
"path"
"strings"
"sync"
@@ -244,17 +245,21 @@
stagedArtifacts := wd.Task("Stage artifacts for signing", tasks.copyToStaging, version, wd.Slice(artifacts))
signedArtifacts := wd.Task("Wait for signed artifacts", tasks.awaitSigned, version, wd.Constant(darwinTargets), stagedArtifacts)
wd.Output("Signed artifacts", signedArtifacts)
- results := wd.Task("Combine results", combineResults, stagedArtifacts, wd.Slice(testResults))
- wd.Output("Build results", results)
+ wait := wd.Task("Wait for signing and tests", combineResults, signedArtifacts, wd.Slice(testResults))
+ uploaded := wd.Task("Upload artifacts to CDN", tasks.uploadArtifacts, signedArtifacts, wait)
+ published := wd.Task("Publish to website", tasks.publishArtifacts, version, signedArtifacts, uploaded)
+ wd.Output("Published", published)
return wd, nil
}
// BuildReleaseTasks serves as an adapter to the various build tasks in the task package.
type BuildReleaseTasks struct {
- GerritURL string
- GCSClient *storage.Client
- ScratchURL, StagingURL string
- CreateBuildlet func(string) (buildlet.Client, error)
+ GerritURL string
+ GCSClient *storage.Client
+ ScratchURL, StagingURL, ServingURL string
+ DownloadURL string
+ PublishFile func(*WebsiteFile) error
+ CreateBuildlet func(string) (buildlet.Client, error)
}
func (b *BuildReleaseTasks) buildSource(ctx *workflow.TaskContext, revision, version string) (artifact, error) {
@@ -584,3 +589,136 @@
}
return signed, true, nil
}
+
+var uploadPollDuration = 30 * time.Second
+
+func (tasks *BuildReleaseTasks) uploadArtifacts(ctx *workflow.TaskContext, artifacts []artifact, _ string) (string, error) {
+ stagingFS, err := gcsfs.FromURL(ctx, tasks.GCSClient, tasks.StagingURL)
+ if err != nil {
+ return "", err
+ }
+ servingFS, err := gcsfs.FromURL(ctx, tasks.GCSClient, tasks.ServingURL)
+ if err != nil {
+ return "", err
+ }
+
+ todo := map[artifact]bool{}
+ for _, a := range artifacts {
+ if err := uploadArtifact(stagingFS, servingFS, a); err != nil {
+ return "", err
+ }
+ todo[a] = true
+ }
+
+ for {
+ for _, a := range artifacts {
+ resp, err := http.Head(tasks.DownloadURL + "/" + a.filename)
+ if err != nil {
+ return "", err
+ }
+ resp.Body.Close()
+ if resp.StatusCode == http.StatusOK {
+ delete(todo, a)
+ }
+ }
+
+ if len(todo) == 0 {
+ return "", nil
+ }
+ select {
+ case <-ctx.Done():
+ return "", ctx.Err()
+ case <-time.After(uploadPollDuration):
+ ctx.Printf("Still waiting for %v artifacts to be published", len(todo))
+ }
+ }
+}
+
+func uploadArtifact(stagingFS, servingFS fs.FS, a artifact) error {
+ in, err := stagingFS.Open(a.signedPath)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+
+ out, err := gcsfs.Create(servingFS, a.filename)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+ if _, err := io.Copy(out, in); err != nil {
+ return err
+ }
+ if err := out.Close(); err != nil {
+ return err
+ }
+
+ sha256, err := gcsfs.Create(servingFS, a.filename+".sha256")
+ if err != nil {
+ return err
+ }
+ defer sha256.Close()
+ if _, err := sha256.Write([]byte(a.sha256)); err != nil {
+ return err
+ }
+ if err := sha256.Close(); err != nil {
+ return err
+ }
+
+ if a.gpgSignature != "" {
+ asc, err := gcsfs.Create(servingFS, a.filename+".asc")
+ if err != nil {
+ return err
+ }
+ defer asc.Close()
+ if _, err := asc.Write([]byte(a.gpgSignature)); err != nil {
+ return err
+ }
+ if err := asc.Close(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (tasks *BuildReleaseTasks) publishArtifacts(ctx *workflow.TaskContext, version string, artifacts []artifact, _ string) (string, error) {
+ for _, a := range artifacts {
+ f := &WebsiteFile{
+ Filename: a.filename,
+ Version: version,
+ ChecksumSHA256: a.sha256,
+ Size: int64(a.size),
+ }
+ if a.target != nil {
+ f.OS = a.target.GOOS
+ f.Arch = a.target.GOARCH
+ if a.target.GOARCH == "arm" {
+ f.Arch = "armv6l"
+ }
+ }
+ switch a.suffix {
+ case "src.tar.gz":
+ f.Kind = "source"
+ case "tar.gz", "zip":
+ f.Kind = "archive"
+ case "msi", "pkg":
+ f.Kind = "installer"
+ }
+ if err := tasks.PublishFile(f); err != nil {
+ return "", err
+ }
+ }
+ return "", nil
+}
+
+// WebsiteFile represents a file on the go.dev downloads page.
+// It should be kept in sync with the download code in x/website/internal/dl.
+type WebsiteFile struct {
+ Filename string `json:"filename"`
+ OS string `json:"os"`
+ Arch string `json:"arch"`
+ Version string `json:"version"`
+ ChecksumSHA256 string `json:"sha256"`
+ Size int64 `json:"size"`
+ Kind string `json:"kind"` // "archive", "installer", "source"
+}
