blob: 2c9b37d126bd095c37bf3a764b77da90163fd9f7 [file] [log] [blame]
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package task
import (
"bytes"
"errors"
"fmt"
"io/fs"
"path"
"slices"
"strconv"
"strings"
texttemplate "text/template"
"time"
"github.com/google/go-github/v48/github"
"golang.org/x/build/gerrit"
"golang.org/x/build/internal/gitfs"
wf "golang.org/x/build/internal/workflow"
"golang.org/x/build/relnote"
"rsc.io/markdown"
)
// ReleaseCycleTasks implements tasks related to the Go release cycle (go.dev/s/release).
type ReleaseCycleTasks struct {
Gerrit GerritClient
GitHub GitHubClientInterface
}
// PromotedAPI holds information about APIs promoted to an api/go1.N.txt file.
type PromotedAPI struct {
// APIs holds promoted API lines, like
// "pkg math/big, method (*Rat) FloatPrec() (int, bool) #50489",
// "pkg iter, func Pull[$0 interface{}](Seq[$0]) (func() ($0, bool), func()) #61897",
// with no empty lines and no newlines, and in sorted order.
APIs []string
// PromotionCL holds the CL number that promoted api/next to api/go1.N.txt.
PromotionCL int
}
// PromoteNextAPI promotes api under api/next to api/go1.{version}.txt
// by mailing a Gerrit CL that does this and waiting for it to be submitted,
// then returns the promoted API.
//
// version is a value like 24 representing that Go 1.24 is the major version which
// is entering release freeze, in anticipation of pre-release versions and eventual release versions.
func (t ReleaseCycleTasks) PromoteNextAPI(ctx *wf.TaskContext, version int, reviewers []string) (PromotedAPI, error) {
// Read branch head.
commit, err := t.Gerrit.ReadBranchHead(ctx, "go", "master")
if err != nil {
return PromotedAPI{}, err
}
ctx.Printf("Using commit %q as the branch head.", commit)
// Confirm that "api/go1.{version}.txt" hasn't already been made.
promotedAPIFile := path.Join("api", fmt.Sprintf("go1.%d.txt", version))
_, err = t.Gerrit.ReadFile(ctx, "go", commit, promotedAPIFile)
if err == nil {
ctx.Printf("The %s file is already created, so refusing to run this task (and tasks that depend on it).", promotedAPIFile)
return PromotedAPI{}, fmt.Errorf("%s already exists; the scope of this task is to create that file only", promotedAPIFile)
} else if errors.Is(err, gerrit.ErrResourceNotExist) {
// OK. We'll be creating it below.
} else if err != nil {
return PromotedAPI{}, err
}
// Read api/next files at commit.
var files = make(map[string]string)
des, err := t.Gerrit.ReadDir(ctx, "go", commit, "api/next")
if err != nil {
return PromotedAPI{}, err
}
var promoted PromotedAPI
for _, de := range des {
if ext := path.Ext(de.Name); ext != ".txt" {
return PromotedAPI{}, fmt.Errorf("file %q in api/next has a non-.txt extension %q", de.Name, ext)
}
b, err := t.Gerrit.ReadFile(ctx, "go", commit, path.Join("api/next", de.Name))
if err != nil {
return PromotedAPI{}, err
}
// TODO(dmitshur): After Go 1.24, consider simplifying bytes.CutSuffix(…, []byte("\n")) + bytes.Split(…, []byte("\n")) in favor of iterating over bytes.Lines or so.
b, ok := bytes.CutSuffix(b, []byte("\n"))
if !ok {
return PromotedAPI{}, fmt.Errorf("API file %s doesn't have a trailing newline", de.Name)
}
for _, l := range bytes.Split(b, []byte("\n")) {
if len(l) == 0 {
return PromotedAPI{}, fmt.Errorf("API file %s has a blank line", de.Name)
}
promoted.APIs = append(promoted.APIs, string(l))
}
files[path.Join("api/next", de.Name)] = "" // Delete the file.
}
slices.Sort(promoted.APIs)
var buf strings.Builder
for _, api := range promoted.APIs {
fmt.Fprintln(&buf, api)
}
files[promotedAPIFile] = buf.String()
// Beyond this point we want retries to be done manually, not automatically.
ctx.DisableRetries()
// Create the promotion CL and await its submission.
changeID, err := t.Gerrit.CreateAutoSubmitChange(ctx, gerrit.ChangeInput{
Project: "go", Branch: "master",
Subject: fmt.Sprintf("api: promote next to go1.%d", version),
}, reviewers, files)
if err != nil {
return PromotedAPI{}, err
}
ctx.Printf("Awaiting review/submit of %v.", ChangeLink(changeID))
if _, err := AwaitCondition(ctx, 10*time.Second, func() (string, bool, error) {
return t.Gerrit.Submitted(ctx, changeID, "")
}); err != nil {
return PromotedAPI{}, err
}
ctx.Printf("The api/next fragments were promoted to api/go1.%d.txt in %v.", version, ChangeLink(changeID))
clNumber, err := strconv.Atoi(strings.TrimPrefix(changeID, "go~"))
if err != nil {
return PromotedAPI{}, err
}
promoted.PromotionCL = clNumber
return promoted, nil
}
func (t ReleaseCycleTasks) OpenAPIAuditIssue(ctx *wf.TaskContext, version int, nextAPI PromotedAPI) (openedIssue int, _ error) {
// TODO(go.dev/issue/70655): Determine programmatically.
const (
devVerMilestone = 322 // Go1.24 milestone (https://github.com/golang/go/milestone/322).
)
// Parse individual lines into sorted groups by package.
parseLine := func(line string) (pkg, feature string, proposal int, _ error) {
pkgAndFeature, proposalStr, ok := strings.Cut(line, " #")
if !ok {
return "", "", 0, fmt.Errorf("no ' #'")
}
proposal, err := strconv.Atoi(proposalStr)
if err != nil {
return "", "", 0, fmt.Errorf("parsing %q to an int: %v", proposalStr, err)
} else if proposal <= 0 {
return "", "", 0, fmt.Errorf("non-positive proposal %d", proposal)
}
pkgAndFeature, ok = strings.CutPrefix(pkgAndFeature, "pkg ")
if !ok {
return "", "", 0, fmt.Errorf("no 'pkg ' prefix")
}
pkg, feature, ok = strings.Cut(pkgAndFeature, ", ")
if !ok {
return "", "", 0, fmt.Errorf("no ', ' after package")
}
if pkg == "" {
return "", "", 0, fmt.Errorf("package is empty string")
} else if feature == "" {
return "", "", 0, fmt.Errorf("feature is empty string")
}
return pkg, feature, proposal, nil
}
type APIAndProposal struct {
API string
Proposal int
}
var byPackage = make(map[string][]APIAndProposal)
for _, line := range nextAPI.APIs {
pkg, feature, proposal, err := parseLine(line)
if err != nil {
return 0, fmt.Errorf("line %q has a problem: %v", line, err)
}
byPackage[pkg] = append(byPackage[pkg], APIAndProposal{feature, proposal})
}
type PackageAndAPI struct {
Package string
APIs []APIAndProposal
}
var nextAPIByPackage []PackageAndAPI
for pkg, features := range byPackage {
slices.SortFunc(features, func(a, b APIAndProposal) int { return strings.Compare(a.API, b.API) })
nextAPIByPackage = append(nextAPIByPackage, PackageAndAPI{
Package: pkg,
APIs: features,
})
}
slices.SortFunc(nextAPIByPackage, func(a, b PackageAndAPI) int { return strings.Compare(a.Package, b.Package) })
// Beyond this point we want retries to be done manually, not automatically.
ctx.DisableRetries()
// Create the API audit issue.
tmpl := texttemplate.Must(texttemplate.New("").
Parse(`This is a tracking issue for doing an audit of API additions for Go 1.{{.Version}} as of [CL {{.PromotionCL}}](https://go.dev/cl/{{.PromotionCL}}).
## New API changes for Go 1.{{.Version}}
{{range .NextAPIByPackage}}
### {{.Package}}
{{range .APIs}}
- ` + "`" + `{{.API}}` + "`" + ` #{{.Proposal}}{{end}}
{{end}}
CC @aclements, @ianlancetaylor, @golang/release.`))
title := fmt.Sprintf("api: audit for Go 1.%d", version)
var body bytes.Buffer
if err := tmpl.Execute(&body, map[string]any{
"Version": version,
"PromotionCL": nextAPI.PromotionCL,
"NextAPIByPackage": nextAPIByPackage,
}); err != nil {
return 0, err
}
issue, _, err := t.GitHub.CreateIssue(ctx, "golang", "go", &github.IssueRequest{
Title: github.String(title),
Body: github.String(body.String()),
Labels: &[]string{"NeedsDecision", "release-blocker", "ExpertNeeded"},
Milestone: github.Int(devVerMilestone),
})
if err != nil {
return 0, err
}
return issue.GetNumber(), nil
}
// NextRelnote holds information about merged release notes.
type NextRelnote struct {
AddMergedToWebsiteCL int // CL number that added merged release notes to x/website.
}
// MergeNextRelnoteAndAddToWebsite merges the release fragments found in
// doc/next of the main Go repository, and adds the merged release notes
// to _content/doc of the x/website repository.
func (t ReleaseCycleTasks) MergeNextRelnoteAndAddToWebsite(ctx *wf.TaskContext, version int, reviewers []string) (NextRelnote, error) {
// TODO(go.dev/issue/70655): Determine programmatically.
const (
releaseNotesIssue = 68545 // "doc: write release notes for Go 1.24"
)
// Read branch head.
goRepo, err := gitfs.NewRepo(t.Gerrit.GitilesURL() + "/" + "go")
if err != nil {
return NextRelnote{}, err
}
commit, err := goRepo.Resolve("refs/heads/master")
if err != nil {
return NextRelnote{}, err
}
ctx.Printf("Using commit %q as the branch head.", commit)
// Confirm that fragments haven't been merged into "_content/doc/go1.{version}.md" yet.
const (
// knownSubstringForPlaceholder is a substring used to identify when go1.N.md is still
// just a placeholder template like in CL 600179, without release note fragments merged in.
// We need some way of detecting whether the merge happened, and this will have to do.
knownSubstringForPlaceholder = "Eventually the release note fragments in doc/next will be merged"
)
mergedRelnoteFile := fmt.Sprintf("_content/doc/go1.%d.md", version)
b, err := t.Gerrit.ReadFile(ctx, "website", commit.String(), mergedRelnoteFile)
if err == nil && !bytes.Contains(b, []byte(knownSubstringForPlaceholder)) {
ctx.Printf("Release note fragments seem to be merged into %s in x/website, so refusing to run this task (and tasks that depend on it).", mergedRelnoteFile)
return NextRelnote{}, fmt.Errorf("%s already has merged fragments; the scope of this task is to merge fragments into that file only", mergedRelnoteFile)
} else if errors.Is(err, gerrit.ErrResourceNotExist) {
// OK. We'll be creating it below.
} else if err != nil {
return NextRelnote{}, err
}
// Collect all doc/next files to merge.
root, err := goRepo.CloneHash(commit)
if err != nil {
return NextRelnote{}, err
}
next, err := fs.Sub(root, "doc/next")
if err != nil {
return NextRelnote{}, err
}
doc, err := relnote.Merge(next)
if err != nil {
return NextRelnote{}, fmt.Errorf("relnote.Merge: %v", err)
}
mergedRelnoteContent := fmt.Sprintf(`---
title: Go 1.%d Release Notes
template: false
---
`, version) + markdown.ToMarkdown(doc)
// Beyond this point we want retries to be done manually, not automatically.
ctx.DisableRetries()
// Create the add-merged CL and await its submission.
changeID, err := t.Gerrit.CreateAutoSubmitChange(ctx, gerrit.ChangeInput{
Project: "website", Branch: "master",
Subject: fmt.Sprintf(`_content/doc: add merged go1.%d.md
Using doc/next content as of %s (commit %s).
For golang/go#%d.`, version, time.Now().Format(time.DateOnly), commit, releaseNotesIssue),
}, reviewers, map[string]string{mergedRelnoteFile: mergedRelnoteContent})
if err != nil {
return NextRelnote{}, err
}
ctx.Printf("Awaiting review/submit of %v.", ChangeLink(changeID))
if _, err := AwaitCondition(ctx, 10*time.Second, func() (string, bool, error) {
return t.Gerrit.Submitted(ctx, changeID, "")
}); err != nil {
return NextRelnote{}, err
}
clNumber, err := strconv.Atoi(strings.TrimPrefix(changeID, "website~"))
if err != nil {
return NextRelnote{}, err
}
return NextRelnote{AddMergedToWebsiteCL: clNumber}, nil
}
// RemoveNextRelnoteFromMainRepo removes release note fragments
// from doc/next in the main Go repository.
func (t ReleaseCycleTasks) RemoveNextRelnoteFromMainRepo(ctx *wf.TaskContext, version int, nr NextRelnote, reviewers []string) error {
// TODO(go.dev/issue/70655): Determine programmatically.
const (
releaseNotesIssue = 68545 // "doc: write release notes for Go 1.24"
)
// Read branch head.
goRepo, err := gitfs.NewRepo(t.Gerrit.GitilesURL() + "/" + "go")
if err != nil {
return err
}
commit, err := goRepo.Resolve("refs/heads/master")
if err != nil {
return err
}
ctx.Printf("Using commit %q as the branch head.", commit)
// Collect all doc/next files to delete.
root, err := goRepo.CloneHash(commit)
if err != nil {
return err
}
var files = make(map[string]string)
err = fs.WalkDir(root, "doc/next", func(path string, de fs.DirEntry, err error) error {
if err != nil {
return err
}
if !de.IsDir() {
files[path] = "" // Delete the file.
}
return nil
})
if err != nil {
return err
}
// Beyond this point we want retries to be done manually, not automatically.
ctx.DisableRetries()
// Create the remove-fragments CL and await its submission.
changeID, err := t.Gerrit.CreateAutoSubmitChange(ctx, gerrit.ChangeInput{
Project: "go", Branch: "master",
Subject: fmt.Sprintf(`doc/next: delete
The release note fragments have been merged and added
as _content/doc/go1.%d.md in x/website in CL %d.
For #%d.`, version, nr.AddMergedToWebsiteCL, releaseNotesIssue),
}, reviewers, files)
if err != nil {
return err
}
ctx.Printf("Awaiting review/submit of %v.", ChangeLink(changeID))
if _, err := AwaitCondition(ctx, 10*time.Second, func() (string, bool, error) {
return t.Gerrit.Submitted(ctx, changeID, "")
}); err != nil {
return err
}
ctx.Printf("Release note fragments were removed in %v.", ChangeLink(changeID))
return nil
}
// ApplyWaitReleaseCLs applies a "wait-release" hashtag to remaining open CLs that are
// adding new APIs and should wait for next release. This is done once at the start of
// the release freeze.
func (t ReleaseCycleTasks) ApplyWaitReleaseCLs(ctx *wf.TaskContext) (result struct{}, _ error) {
clsToWait, err := t.Gerrit.QueryChanges(ctx, "repo:go status:open -is:wip dir:api/next -hashtag:wait-release")
if err != nil {
return struct{}{}, err
}
ctx.Printf("Processing %d open Gerrit CLs to be marked with wait-release hashtag.", len(clsToWait))
for _, cl := range clsToWait {
const dryRun = false
if dryRun {
ctx.Printf("[dry run] Would've waited CL %d (%.32s…).", cl.ChangeNumber, cl.Subject)
continue
}
err := t.Gerrit.SetHashtags(ctx, cl.ID, gerrit.HashtagsInput{Add: []string{"wait-release"}})
if err != nil {
return struct{}{}, err
}
ctx.Printf("Waited CL %d (%.32s…).", cl.ChangeNumber, cl.Subject)
time.Sleep(3 * time.Second) // Take a moment between updating CLs to avoid a high rate of modify operations.
}
return struct{}{}, nil
}
// UnwaitWaitReleaseCLs changes all open Gerrit CLs with hashtag "wait-release" into "ex-wait-release".
// This is done once at the opening of a release cycle, currently via a standalone workflow.
func (t ReleaseCycleTasks) UnwaitWaitReleaseCLs(ctx *wf.TaskContext) (result struct{}, _ error) {
waitingCLs, err := t.Gerrit.QueryChanges(ctx, "status:open hashtag:wait-release")
if err != nil {
return struct{}{}, err
}
ctx.Printf("Processing %d open Gerrit CL with wait-release hashtag.", len(waitingCLs))
for _, cl := range waitingCLs {
const dryRun = false
if dryRun {
ctx.Printf("[dry run] Would've unwaited CL %d (%.32s…).", cl.ChangeNumber, cl.Subject)
continue
}
err := t.Gerrit.SetHashtags(ctx, cl.ID, gerrit.HashtagsInput{
Remove: []string{"wait-release"},
Add: []string{"ex-wait-release"},
})
if err != nil {
return struct{}{}, err
}
ctx.Printf("Unwaited CL %d (%.32s…).", cl.ChangeNumber, cl.Subject)
time.Sleep(3 * time.Second) // Take a moment between updating CLs to avoid a high rate of modify operations.
}
return struct{}{}, nil
}