cmd/relui: add build release workflow
Adds the core of the build release workflow to relui. We build
artifacts and run tests.
Adds the necessary dependencies, currently just the coordinator client,
to relui's main function. The filesystem stuff will be replaced in the
next CL.
For golang/go#51797.
Change-Id: I4eea478d026b08540ac3bb48a684a545d862de6c
Reviewed-on: https://go-review.googlesource.com/c/build/+/398694
Run-TryBot: Heschi Kreinick <heschi@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Alex Rakoczy <alex@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/cmd/relui/deployment-prod.yaml b/cmd/relui/deployment-prod.yaml
index d6fd075..16ba7e2 100644
--- a/cmd/relui/deployment-prod.yaml
+++ b/cmd/relui/deployment-prod.yaml
@@ -30,6 +30,7 @@
- "--site-header-css=Site-header--production"
- "--gerrit-api-secret=secret:symbolic-datum-552/gobot-password"
- "--twitter-api-secret=secret:symbolic-datum-552/twitter-api-secret"
+ - "--builder-master-key=secret:symbolic-datum-552/builder-master-key"
ports:
- containerPort: 444
env:
diff --git a/cmd/relui/main.go b/cmd/relui/main.go
index 3c84dab..8d85533 100644
--- a/cmd/relui/main.go
+++ b/cmd/relui/main.go
@@ -7,11 +7,19 @@
import (
"context"
+ "crypto/hmac"
+ "crypto/md5"
"flag"
+ "fmt"
+ "io"
"log"
"net/url"
+ "os"
+ "path/filepath"
"github.com/jackc/pgx/v4/pgxpool"
+ "golang.org/x/build"
+ "golang.org/x/build/buildlet"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal/https"
"golang.org/x/build/internal/relui"
@@ -36,6 +44,7 @@
gerritAPIFlag := secret.Flag("gerrit-api-secret", "Gerrit API secret to use for workflows that interact with Gerrit.")
var twitterAPI secret.TwitterCredentials
secret.JSONVarFlag(&twitterAPI, "twitter-api-secret", "Twitter API secret to use for workflows involving tweeting.")
+ masterKey := secret.Flag("builder-master-key", "Builder master key")
https.RegisterFlags(flag.CommandLine)
flag.Parse()
@@ -75,6 +84,18 @@
dh := relui.NewDefinitionHolder()
relui.RegisterMailDLCLDefinition(dh, extCfg)
relui.RegisterTweetDefinitions(dh, extCfg)
+ coordinator := &buildlet.CoordinatorClient{
+ Auth: buildlet.UserPass{
+ Username: "user-relui",
+ Password: key(*masterKey, "user-relui"),
+ },
+ Instance: build.ProdCoordinator,
+ }
+ log.Printf("Coordinator client: %#v", coordinator)
+ if _, err := coordinator.RemoteBuildlets(); err != nil {
+ log.Printf("Broken coordinator client: %v", err)
+ }
+ relui.RegisterBuildReleaseWorkflows(dh, &osFiles{"/tmp"}, coordinator.CreateBuildlet)
db, err := pgxpool.Connect(ctx, *pgConnect)
if err != nil {
log.Fatal(err)
@@ -98,3 +119,21 @@
}
log.Fatalln(https.ListenAndServe(ctx, s))
}
+
+func key(masterKey, principal string) string {
+ h := hmac.New(md5.New, []byte(masterKey))
+ io.WriteString(h, principal)
+ return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+type osFiles struct {
+ basePath string
+}
+
+func (f *osFiles) Create(path string) (io.WriteCloser, error) {
+ return os.Create(filepath.Join(f.basePath, path))
+}
+
+func (f *osFiles) Open(path string) (io.ReadCloser, error) {
+ return os.Open(filepath.Join(f.basePath, path))
+}
diff --git a/internal/relui/listener.go b/internal/relui/listener.go
index ac16c91..8d819dc 100644
--- a/internal/relui/listener.go
+++ b/internal/relui/listener.go
@@ -32,7 +32,7 @@
// workflow. The workflow.TaskState is persisted as a db.Task,
// creating or updating a row as necessary.
func (l *PGListener) TaskStateChanged(workflowID uuid.UUID, taskName string, state *workflow.TaskState) error {
- log.Printf("TaskStateChanged(%q, %q, %v)", workflowID, taskName, state)
+ log.Printf("TaskStateChanged(%q, %q, %#v)", workflowID, taskName, state)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
result, err := json.Marshal(state.Result)
@@ -54,7 +54,7 @@
return err
})
if err != nil {
- log.Printf("TaskStateChanged(%q, %q, %v) = %v", workflowID, taskName, state, err)
+ log.Printf("TaskStateChanged(%q, %q, %#v) = %v", workflowID, taskName, state, err)
}
return err
}
diff --git a/internal/relui/workflows.go b/internal/relui/workflows.go
index 9e78439..7e11c19 100644
--- a/internal/relui/workflows.go
+++ b/internal/relui/workflows.go
@@ -5,8 +5,14 @@
package relui
import (
+ "fmt"
+ "io"
+ "strings"
"sync"
+ "golang.org/x/build/buildlet"
+ "golang.org/x/build/dashboard"
+ "golang.org/x/build/internal/releasetargets"
"golang.org/x/build/internal/task"
"golang.org/x/build/internal/workflow"
)
@@ -169,3 +175,191 @@
ctx.Printf("echo(%v, %q)", ctx, arg)
return arg, nil
}
+
+func RegisterBuildReleaseWorkflows(h *DefinitionHolder, files WorkingFiles, createBuildlet func(string) (buildlet.Client, error)) {
+ go117, err := newBuildReleaseWorkflow("go1.17", files, createBuildlet)
+ if err != nil {
+ panic(err)
+ }
+ h.RegisterDefinition("Release Go 1.17", go117)
+ go118, err := newBuildReleaseWorkflow("go1.18", files, createBuildlet)
+ if err != nil {
+ panic(err)
+ }
+ h.RegisterDefinition("Release Go 1.18", go118)
+
+}
+
+func newBuildReleaseWorkflow(majorVersion string, files WorkingFiles, createBuildlet func(string) (buildlet.Client, error)) (*workflow.Definition, error) {
+ wd := workflow.New()
+ targets, ok := releasetargets.TargetsForVersion(majorVersion)
+ if !ok {
+ return nil, fmt.Errorf("malformed/unknown version %q", majorVersion)
+ }
+ version := wd.Parameter(workflow.Parameter{Name: "Version", Example: "go1.10.1"})
+ revision := wd.Parameter(workflow.Parameter{Name: "Revision", Example: "release-branch.go1.10"})
+ skipTests := wd.Parameter(workflow.Parameter{Name: "Targets to skip testing (space-separated target names or 'all') (optional)"})
+ tasks := buildReleaseTasks{files, createBuildlet}
+
+ source := wd.Task("Build source archive", tasks.buildSource, revision, version)
+ // Artifact file paths.
+ var artifacts []workflow.Value
+ // Empty values that represent the dependency on tests passing.
+ var testResults []workflow.Value
+ for _, target := range targets {
+ targetVal := wd.Constant(target)
+ taskName := func(step string) string { return target.Name + ": " + step }
+
+ // Build release artifacts for the platform.
+ bin := wd.Task(taskName("Build binary archive"), tasks.buildBinary, targetVal, source)
+ if target.GOOS == "windows" {
+ zip := wd.Task(taskName("Convert to .zip"), tasks.convertToZip, targetVal, bin)
+ msi := wd.Task(taskName("Build MSI"), tasks.buildMSI, targetVal, bin)
+ artifacts = append(artifacts, msi, zip)
+ } else {
+ artifacts = append(artifacts, bin)
+ }
+
+ if target.BuildOnly {
+ continue
+ }
+ short := wd.Task(taskName("Run short tests"), tasks.runTests, targetVal, wd.Constant(target.Builder), skipTests, bin)
+ testResults = append(testResults, short)
+ if target.LongTestBuilder != "" {
+ long := wd.Task(taskName("Run long tests"), tasks.runTests, targetVal, wd.Constant(target.LongTestBuilder), skipTests, bin)
+ testResults = append(testResults, long)
+ }
+ }
+ // Eventually we need to upload artifacts and perhaps summarize test results.
+ // For now, just mush them all together.
+ results := wd.Task("Combine results", combineResults, wd.Slice(artifacts), wd.Slice(testResults))
+ wd.Output("Build results", results)
+ return wd, nil
+}
+
+// buildReleaseTasks serves as an adapter to the various build tasks in the task package.
+type buildReleaseTasks struct {
+ fs WorkingFiles
+ createBuildlet func(string) (buildlet.Client, error)
+}
+
+func (b *buildReleaseTasks) buildSource(ctx *workflow.TaskContext, revision, version string) (string, error) {
+ return b.runBuildStep(ctx, nil, "", "", "source.tar.gz", func(_ *task.BuildletStep, _ io.Reader, w io.Writer) error {
+ return task.WriteSourceArchive(ctx, revision, version, w)
+ })
+}
+
+func (b *buildReleaseTasks) buildBinary(ctx *workflow.TaskContext, target *releasetargets.Target, source string) (string, error) {
+ return b.runBuildStep(ctx, target, target.Builder, source, target.Name+"-binary.tar.gz", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
+ return bs.BuildBinary(ctx, r, w)
+ })
+}
+
+func (b *buildReleaseTasks) buildMSI(ctx *workflow.TaskContext, target *releasetargets.Target, binary string) (string, error) {
+ return b.runBuildStep(ctx, target, target.Builder, binary, target.Name+".msi", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
+ return bs.BuildMSI(ctx, r, w)
+ })
+}
+
+func (b *buildReleaseTasks) convertToZip(ctx *workflow.TaskContext, target *releasetargets.Target, binary string) (string, error) {
+ return b.runBuildStep(ctx, nil, "", binary, target.Name+".zip", func(_ *task.BuildletStep, r io.Reader, w io.Writer) error {
+ return task.ConvertTGZToZIP(r, w)
+ })
+}
+
+func (b *buildReleaseTasks) runTests(ctx *workflow.TaskContext, target *releasetargets.Target, buildlet, skipTests, binary string) (string, error) {
+ skipped := skipTests == "all"
+ skipTargets := strings.Fields(skipTests)
+ for _, skip := range skipTargets {
+ if target.Name == skip {
+ skipped = true
+ }
+ }
+ if skipped {
+ ctx.Printf("Skipping test")
+ return "", nil
+ }
+ return b.runBuildStep(ctx, target, buildlet, binary, "", func(bs *task.BuildletStep, r io.Reader, _ io.Writer) error {
+ return bs.TestTarget(ctx, r)
+ })
+}
+
+// runBuildStep is a convenience function that manages resources a build step might need.
+// If target and buildlet name are specified, a BuildletStep will be passed to f.
+// If inputName is specified, it will be opened and passed as a Reader to f.
+// If outputName is specified, a unique filename will be generated based off it, the file
+// will be opened and passed as a Writer to f, and its name will be returned as the result.
+func (b *buildReleaseTasks) runBuildStep(
+ ctx *workflow.TaskContext,
+ target *releasetargets.Target,
+ buildletName, inputName, outputName string,
+ f func(*task.BuildletStep, io.Reader, io.Writer) error,
+) (string, error) {
+ if (target == nil) != (buildletName == "") {
+ return "", fmt.Errorf("target and buildlet must be specified together")
+ }
+
+ var step *task.BuildletStep
+ if target != nil {
+ ctx.Printf("Creating buildlet %v.", buildletName)
+ client, err := b.createBuildlet(buildletName)
+ if err != nil {
+ return "", err
+ }
+ defer client.Close()
+ buildConfig, ok := dashboard.Builders[buildletName]
+ if !ok {
+ return "", fmt.Errorf("unknown builder: %v", buildConfig)
+ }
+ step = &task.BuildletStep{
+ Target: target,
+ Buildlet: client,
+ BuildConfig: buildConfig,
+ Watch: true,
+ }
+ ctx.Printf("Buildlet ready.")
+ }
+
+ var in io.ReadCloser
+ var err error
+ if inputName != "" {
+ in, err = b.fs.Open(inputName)
+ if err != nil {
+ return "", err
+ }
+ defer in.Close()
+ }
+ var out io.WriteCloser
+ if outputName != "" {
+ out, err = b.fs.Create(outputName)
+ if err != nil {
+ return "", err
+ }
+ defer out.Close()
+ }
+ if err := f(step, in, out); err != nil {
+ return "", err
+ }
+ if step != nil {
+ if err := step.Buildlet.Close(); err != nil {
+ return "", err
+ }
+ }
+ // Don't check the error from in.Close: the steps may assert it down to
+ // Closer and close it themselves. (See https://pkg.go.dev/net/http#NewRequestWithContext.)
+ if out != nil {
+ if err := out.Close(); err != nil {
+ return "", err
+ }
+ }
+ return outputName, nil
+}
+
+func combineResults(ctx *workflow.TaskContext, artifacts, tests []string) (string, error) {
+ return strings.Join(artifacts, "\n") + strings.Join(tests, "\n"), nil
+}
+
+type WorkingFiles interface {
+ Create(string) (io.WriteCloser, error)
+ Open(string) (io.ReadCloser, error)
+}
