internal/relui: add a step to wait for signing
For the moment, signing will be run out-of-band with the release
process. Add a step that waits for it to complete.
Because our signing process is rough around the edges there are a lot of
subtleties to deal with: we don't produce signatures for zip files,
files only appear in /signed/ if they were modified by the signing
process, etc. Most inconveniently, the .pkg files are produced by the
signing process, which doesn't fit the model I was going for and
requires them to be injected at signing time.
For golang/go#51797.
Change-Id: I6d0b5ffe8b75f4a0d66de3f059d166c843cd0209
Reviewed-on: https://go-review.googlesource.com/c/build/+/410014
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Run-TryBot: Heschi Kreinick <heschi@google.com>
Auto-Submit: Heschi Kreinick <heschi@google.com>
diff --git a/internal/relui/buildrelease_test.go b/internal/relui/buildrelease_test.go
index a5e477f..333251e 100644
--- a/internal/relui/buildrelease_test.go
+++ b/internal/relui/buildrelease_test.go
@@ -10,6 +10,7 @@
"bytes"
"compress/gzip"
"context"
+ "crypto/sha256"
"fmt"
"io"
"io/ioutil"
@@ -25,6 +26,7 @@
"testing"
"time"
+ "github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"golang.org/x/build/buildlet"
"golang.org/x/build/internal/untar"
@@ -32,6 +34,8 @@
)
func TestRelease(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
if runtime.GOOS != "linux" {
t.Skip("Requires bash shell scripting support.")
}
@@ -44,6 +48,8 @@
logs: map[string][]*[]string{},
}
stagingDir := t.TempDir()
+ go fakeSign(ctx, t, filepath.Join(stagingDir, "go1.18releasetest1"))
+ signingPollDuration = 100 * time.Millisecond
tasks := BuildReleaseTasks{
GerritURL: s.URL,
GCSClient: nil,
@@ -63,13 +69,16 @@
if err != nil {
t.Fatal(err)
}
- out, err := w.Run(context.TODO(), &verboseListener{t})
+ out, err := w.Run(ctx, &verboseListener{t})
if err != nil {
t.Fatal(err)
}
- artifacts := out["Staged artifacts"].([]artifact)
+ artifacts := out["Signed artifacts"].([]artifact)
byName := map[string]artifact{}
for _, a := range artifacts {
+ if a.sha256 == "" || a.size < 1 || a.suffix == "" || a.filename == "" {
+ t.Errorf("release process produced an invalid artifact: %#v", a)
+ }
byName[a.filename] = a
}
@@ -77,7 +86,7 @@
"go/VERSION": "go1.18releasetest1",
"go/src/make.bash": makeScript,
})
- checkMSI(t, stagingDir, byName["go1.18releasetest1.windows-amd64.msi"])
+ checkContents(t, stagingDir, byName["go1.18releasetest1.windows-amd64.msi"], "I'm an MSI!\n")
checkTGZ(t, stagingDir, byName["go1.18releasetest1.linux-amd64.tar.gz"], map[string]string{
"go/VERSION": "go1.18releasetest1",
"go/tool/something_orother/compile": "",
@@ -91,6 +100,7 @@
"go/VERSION": "go1.18releasetest1",
"go/tool/something_orother/compile": "",
})
+ checkContents(t, stagingDir, byName["go1.18releasetest1.darwin-amd64.pkg"], "I'm a .pkg!\n")
// TODO: consider logging this to golden files?
for name, logs := range fakeBuildlets.logs {
@@ -185,21 +195,21 @@
}
}
-func checkMSI(t *testing.T, stagingDir string, a artifact) {
+func checkContents(t *testing.T, stagingDir string, a artifact, contents string) {
t.Run(a.filename, func(t *testing.T) {
- b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.stagingPath))
+ b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.signedPath))
if err != nil {
t.Fatalf("reading %v: %v", a.filename, err)
}
- if got, want := string(b), "I'm an MSI!\n"; got != want {
- t.Fatalf("%v contains %q, want %q", a.filename, got, want)
+ if got, want := string(b), contents; got != want {
+ t.Errorf("%v contains %q, want %q", a.filename, got, want)
}
})
}
func checkTGZ(t *testing.T, stagingDir string, a artifact, contents map[string]string) {
t.Run(a.filename, func(t *testing.T) {
- b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.stagingPath))
+ b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.signedPath))
if err != nil {
t.Fatalf("reading %v: %v", a.filename, err)
}
@@ -230,14 +240,14 @@
}
}
if len(contents) != 0 {
- t.Fatalf("not all files were found: missing %v", contents)
+ t.Errorf("not all files were found: missing %v", contents)
}
})
}
func checkZip(t *testing.T, stagingDir string, a artifact, contents map[string]string) {
t.Run(a.filename, func(t *testing.T) {
- b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.stagingPath))
+ b, err := ioutil.ReadFile(filepath.Join(stagingDir, a.signedPath))
if err != nil {
t.Fatalf("reading %v: %v", a.filename, err)
}
@@ -264,7 +274,7 @@
}
}
if len(contents) != 0 {
- t.Fatalf("not all files were found: missing %v", contents)
+ t.Errorf("not all files were found: missing %v", contents)
}
})
}
@@ -475,3 +485,163 @@
func (l *testLogger) Printf(format string, v ...interface{}) {
l.t.Logf("task %-10v: LOG: %s", l.task, fmt.Sprintf(format, v...))
}
+
+// fakeSign acts like a human running the signbinaries job periodically.
+func fakeSign(ctx context.Context, t *testing.T, dir string) {
+ seen := map[string]bool{}
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(500 * time.Millisecond):
+ }
+
+ fakeSignOnce(t, dir, seen)
+ }
+}
+
+func fakeSignOnce(t *testing.T, dir string, seen map[string]bool) {
+ contents, err := os.ReadDir(dir)
+ if os.IsNotExist(err) {
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, fi := range contents {
+ fn := fi.Name()
+ if fn == "signed" || seen[fn] {
+ continue
+ }
+ var copy, gpgSign, makePkg bool
+ hasSuffix := func(suffix string) bool { return strings.HasSuffix(fn, suffix) }
+ switch {
+ case strings.Contains(fn, "darwin") && hasSuffix(".tar.gz"):
+ copy = true
+ gpgSign = true
+ makePkg = true
+ case strings.Contains(fn, "darwin") && hasSuffix(".pkg"):
+ copy = true
+ case hasSuffix(".tar.gz"):
+ gpgSign = true
+ case hasSuffix("msi"):
+ copy = true
+ }
+
+ if err := os.MkdirAll(filepath.Join(dir, "signed"), 0777); err != nil {
+ t.Fatal(err)
+ }
+
+ writeSignedWithHash := func(filename string, contents []byte) {
+ path := filepath.Join(dir, "signed", filename)
+ if err := ioutil.WriteFile(path, contents, 0777); err != nil {
+ t.Fatal(err)
+ }
+ hash := fmt.Sprintf("%x", sha256.Sum256(contents))
+ if err := ioutil.WriteFile(path+".sha256", []byte(hash), 0777); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if copy {
+ bytes, err := ioutil.ReadFile(filepath.Join(dir, fn))
+ if err != nil {
+ t.Fatal(err)
+ }
+ writeSignedWithHash(fn, bytes)
+ }
+ if makePkg {
+ writeSignedWithHash(strings.ReplaceAll(fn, ".tar.gz", ".pkg"), []byte("I'm a .pkg!\n"))
+ }
+ if gpgSign {
+ writeSignedWithHash(fn+".asc", []byte("gpg signature"))
+ }
+ seen[fn] = true
+ }
+}
+
+// These are the files created by the Go 1.18 release.
+const inputs = `
+go1.18.darwin-amd64.tar.gz
+go1.18.darwin-arm64.tar.gz
+go1.18.freebsd-386.tar.gz
+go1.18.freebsd-amd64.tar.gz
+go1.18.linux-386.tar.gz
+go1.18.linux-amd64.tar.gz
+go1.18.linux-arm64.tar.gz
+go1.18.linux-armv6l.tar.gz
+go1.18.linux-ppc64le.tar.gz
+go1.18.linux-s390x.tar.gz
+go1.18.src.tar.gz
+go1.18.windows-386.msi
+go1.18.windows-386.zip
+go1.18.windows-amd64.msi
+go1.18.windows-amd64.zip
+go1.18.windows-arm64.msi
+go1.18.windows-arm64.zip
+`
+
+// These are the files created in the "signed" folder by the signing run for Go 1.18.
+const outputs = `
+go1.18.darwin-amd64.pkg
+go1.18.darwin-amd64.pkg.sha256
+go1.18.darwin-amd64.tar.gz
+go1.18.darwin-amd64.tar.gz.asc
+go1.18.darwin-amd64.tar.gz.asc.sha256
+go1.18.darwin-amd64.tar.gz.sha256
+go1.18.darwin-arm64.pkg
+go1.18.darwin-arm64.pkg.sha256
+go1.18.darwin-arm64.tar.gz
+go1.18.darwin-arm64.tar.gz.asc
+go1.18.darwin-arm64.tar.gz.asc.sha256
+go1.18.darwin-arm64.tar.gz.sha256
+go1.18.freebsd-386.tar.gz.asc
+go1.18.freebsd-386.tar.gz.asc.sha256
+go1.18.freebsd-amd64.tar.gz.asc
+go1.18.freebsd-amd64.tar.gz.asc.sha256
+go1.18.linux-386.tar.gz.asc
+go1.18.linux-386.tar.gz.asc.sha256
+go1.18.linux-amd64.tar.gz.asc
+go1.18.linux-amd64.tar.gz.asc.sha256
+go1.18.linux-arm64.tar.gz.asc
+go1.18.linux-arm64.tar.gz.asc.sha256
+go1.18.linux-armv6l.tar.gz.asc
+go1.18.linux-armv6l.tar.gz.asc.sha256
+go1.18.linux-ppc64le.tar.gz.asc
+go1.18.linux-ppc64le.tar.gz.asc.sha256
+go1.18.linux-s390x.tar.gz.asc
+go1.18.linux-s390x.tar.gz.asc.sha256
+go1.18.src.tar.gz.asc
+go1.18.src.tar.gz.asc.sha256
+go1.18.windows-386.msi
+go1.18.windows-386.msi.sha256
+go1.18.windows-amd64.msi
+go1.18.windows-amd64.msi.sha256
+go1.18.windows-arm64.msi
+go1.18.windows-arm64.msi.sha256
+`
+
+func TestFakeSign(t *testing.T) {
+ dir := t.TempDir()
+ for _, f := range strings.Split(strings.TrimSpace(inputs), "\n") {
+ if err := ioutil.WriteFile(filepath.Join(dir, f), []byte("hi"), 0777); err != nil {
+ t.Fatal(err)
+ }
+ }
+ fakeSignOnce(t, dir, map[string]bool{})
+ want := map[string]bool{}
+ for _, f := range strings.Split(strings.TrimSpace(outputs), "\n") {
+ want[f] = true
+ }
+ got := map[string]bool{}
+ files, err := ioutil.ReadDir(filepath.Join(dir, "signed"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, f := range files {
+ got[f.Name()] = true
+ }
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("signed outputs mismatch (-want +got):\n%v", diff)
+ }
+}
diff --git a/internal/relui/workflows.go b/internal/relui/workflows.go
index f1bffb2..620cb1a 100644
--- a/internal/relui/workflows.go
+++ b/internal/relui/workflows.go
@@ -8,10 +8,12 @@
"crypto/sha256"
"fmt"
"io"
+ "io/fs"
"math/rand"
"path"
"strings"
"sync"
+ "time"
"cloud.google.com/go/storage"
"golang.org/x/build/buildlet"
@@ -208,6 +210,7 @@
source := wd.Task("Build source archive", tasks.buildSource, revision, version)
// Artifact file paths.
artifacts := []workflow.Value{source}
+ var darwinTargets []*releasetargets.Target
// Empty values that represent the dependency on tests passing.
var testResults []workflow.Value
for _, target := range targets {
@@ -216,11 +219,15 @@
// Build release artifacts for the platform.
bin := wd.Task(taskName("Build binary archive"), tasks.buildBinary, targetVal, source)
- if target.GOOS == "windows" {
+ switch target.GOOS {
+ case "windows":
zip := wd.Task(taskName("Convert to .zip"), tasks.convertToZip, targetVal, bin)
msi := wd.Task(taskName("Build MSI"), tasks.buildMSI, targetVal, bin)
artifacts = append(artifacts, msi, zip)
- } else {
+ case "darwin":
+ artifacts = append(artifacts, bin)
+ darwinTargets = append(darwinTargets, target)
+ default:
artifacts = append(artifacts, bin)
}
@@ -234,10 +241,9 @@
testResults = append(testResults, long)
}
}
- // Eventually we need to sign artifacts and perhaps summarize test results.
- // For now, just mush them all together.
stagedArtifacts := wd.Task("Stage artifacts for signing", tasks.copyToStaging, version, wd.Slice(artifacts))
- wd.Output("Staged artifacts", stagedArtifacts)
+ signedArtifacts := wd.Task("Wait for signed artifacts", tasks.awaitSigned, version, wd.Constant(darwinTargets), stagedArtifacts)
+ wd.Output("Signed artifacts", signedArtifacts)
results := wd.Task("Combine results", combineResults, stagedArtifacts, wd.Slice(testResults))
wd.Output("Build results", results)
return wd, nil
@@ -397,6 +403,11 @@
scratchPath string
// The path the artifact was staged to for the signing process.
stagingPath string
+ // The path artifact can be found at after the signing process. It may be
+ // the same as the staging path for artifacts that are externally signed.
+ signedPath string
+ // The contents of the GPG signature for this artifact (.asc file).
+ gpgSignature string
// The filename suffix of the artifact, e.g. "tar.gz" or "src.tar.gz",
// combined with the version and target name to produce filename.
suffix string
@@ -459,3 +470,117 @@
}
return stagedArtifacts, nil
}
+
+var signingPollDuration = 30 * time.Second
+
+// awaitSigned waits for all of artifacts to be signed, plus the pkgs for
+// darwinTargets.
+func (tasks *BuildReleaseTasks) awaitSigned(ctx *workflow.TaskContext, version string, darwinTargets []*releasetargets.Target, artifacts []artifact) ([]artifact, error) {
+ // .pkg artifacts are created by the signing process. Create placeholders,
+ // to be filled out once the files exist.
+ for _, t := range darwinTargets {
+ artifacts = append(artifacts, artifact{
+ target: t,
+ suffix: "pkg",
+ filename: version + "." + t.Name + ".pkg",
+ size: -1,
+ })
+ }
+
+ stagingFS, err := gcsfs.FromURL(ctx, tasks.GCSClient, tasks.StagingURL)
+ if err != nil {
+ return nil, err
+ }
+
+ todo := map[artifact]bool{}
+ for _, a := range artifacts {
+ todo[a] = true
+ }
+ var signedArtifacts []artifact
+ for {
+ for a := range todo {
+ signed, ok, err := readSignedArtifact(stagingFS, version, a)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ continue
+ }
+
+ signedArtifacts = append(signedArtifacts, signed)
+ delete(todo, a)
+ }
+
+ if len(todo) == 0 {
+ return signedArtifacts, nil
+ }
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(signingPollDuration):
+ ctx.Printf("Still waiting for %v artifacts to be signed", len(todo))
+ }
+ }
+}
+
+func readSignedArtifact(stagingFS fs.FS, version string, a artifact) (_ artifact, ok bool, _ error) {
+ // Our signing process has somewhat uneven behavior. In general, for things
+ // that contain their own signature, such as MSIs and .pkgs, we don't
+ // produce a GPG signature, just the new file. On macOS, tars can be signed
+ // too, but we GPG sign them anyway.
+ modifiedBySigning := false
+ hasGPG := false
+ suffix := func(suffix string) bool { return a.suffix == suffix }
+ switch {
+ case suffix("src.tar.gz"):
+ hasGPG = true
+ case a.target.GOOS == "darwin" && suffix("tar.gz"):
+ modifiedBySigning = true
+ hasGPG = true
+ case a.target.GOOS == "darwin" && suffix("pkg"):
+ modifiedBySigning = true
+ case suffix("tar.gz"):
+ hasGPG = true
+ case suffix("msi"):
+ modifiedBySigning = true
+ case suffix("zip"):
+ // For reasons unclear, we don't sign zip files.
+ default:
+ return artifact{}, false, fmt.Errorf("unhandled file type %q", a.suffix)
+ }
+
+ signed := artifact{
+ target: a.target,
+ filename: a.filename,
+ suffix: a.suffix,
+ }
+ if modifiedBySigning {
+ signed.signedPath = version + "/signed/" + a.filename
+ } else {
+ signed.signedPath = version + "/" + a.filename
+ }
+
+ fi, err := fs.Stat(stagingFS, signed.signedPath)
+ if err != nil {
+ return artifact{}, false, nil
+ }
+ if modifiedBySigning {
+ hash, err := fs.ReadFile(stagingFS, version+"/signed/"+a.filename+".sha256")
+ if err != nil {
+ return artifact{}, false, err
+ }
+ signed.size = int(fi.Size())
+ signed.sha256 = string(hash)
+ } else {
+ signed.sha256 = a.sha256
+ signed.size = a.size
+ }
+ if hasGPG {
+ sig, err := fs.ReadFile(stagingFS, version+"/signed/"+a.filename+".asc")
+ if err != nil {
+ return artifact{}, false, nil
+ }
+ signed.gpgSignature = string(sig)
+ }
+ return signed, true, nil
+}
