cmd/relui, internal/task: add release announcement email workflows
Add workflows for automatically sending Go release announcement emails
to the relevant Google Groups mailing lists. Use the Markdown email
templates from the previous CL for determining the email content.
In the future, these workflows will be merged with the tweet workflows
and into workflows that build and publish the release artifacts.
Fixes golang/go#47405.
Updates golang/go#47404.
Fixes golang/go#50864.
Change-Id: Id82cd4c18e569a19b9d807125765a31773c5bcca
Reviewed-on: https://go-review.googlesource.com/c/build/+/411575
Run-TryBot: Dmitri Shuralyov <dmitshur@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Auto-Submit: Dmitri Shuralyov <dmitshur@golang.org>
Reviewed-by: Heschi Kreinick <heschi@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
diff --git a/cmd/relui/deployment-prod.yaml b/cmd/relui/deployment-prod.yaml
index 2b4c40b..f5822f4 100644
--- a/cmd/relui/deployment-prod.yaml
+++ b/cmd/relui/deployment-prod.yaml
@@ -33,6 +33,10 @@
- "--site-title=Go Releases"
- "--site-header-css=Site-header--production"
- "--gerrit-api-secret=secret:symbolic-datum-552/gobot-password"
+ - "--sendgrid-api-key=secret:symbolic-datum-552/sendgrid-sendonly-api-key"
+ - "--announce-mail-from=announce@golang.org"
+ - "--announce-mail-to=golang-nuts@googlegroups.com",
+ - "--announce-mail-bcc=golang-announce@googlegroups.com, golang-dev@googlegroups.com",
- "--twitter-api-secret=secret:symbolic-datum-552/twitter-api-secret"
- "--builder-master-key=secret:symbolic-datum-552/builder-master-key"
- "--github-token=secret:symbolic-datum-552/maintner-github-token"
diff --git a/cmd/relui/main.go b/cmd/relui/main.go
index 4d031c3..934a3a9 100644
--- a/cmd/relui/main.go
+++ b/cmd/relui/main.go
@@ -18,6 +18,7 @@
"log"
"math/rand"
"net/http"
+ "net/mail"
"net/url"
"strings"
"time"
@@ -57,6 +58,11 @@
log.Fatalln(err)
}
gerritAPIFlag := secret.Flag("gerrit-api-secret", "Gerrit API secret to use for workflows that interact with Gerrit.")
+ var annMail task.AnnounceMailTasks
+ secret.FlagVar(&annMail.SendGridAPIKey, "sendgrid-api-key", "SendGrid API key for workflows involving sending email.")
+ addressVarFlag(&annMail.From, "announce-mail-from", "The From address to use for the announcement mail.")
+ addressVarFlag(&annMail.To, "announce-mail-to", "The To address to use for the announcement mail.")
+ addressListVarFlag(&annMail.BCC, "announce-mail-bcc", "The BCC address list to use for the announcement mail.")
var twitterAPI secret.TwitterCredentials
secret.JSONVarFlag(&twitterAPI, "twitter-api-secret", "Twitter API secret to use for workflows involving tweeting.")
masterKey := secret.Flag("builder-master-key", "Builder master key")
@@ -102,6 +108,7 @@
}
dh := relui.NewDefinitionHolder()
relui.RegisterMailDLCLDefinition(dh, versionTasks)
+ relui.RegisterAnnounceDefinitions(dh, annMail)
relui.RegisterTweetDefinitions(dh, extCfg)
userPassAuth := buildlet.UserPass{
Username: "user-relui",
@@ -196,3 +203,32 @@
}
return nil
}
+
+// addressVarFlag defines an address flag with specified name and usage string.
+// The argument p points to a mail.Address variable in which to store the value of the flag.
+func addressVarFlag(p *mail.Address, name, usage string) {
+ flag.Func(name, usage, func(s string) error {
+ a, err := mail.ParseAddress(s)
+ if err != nil {
+ return err
+ }
+ *p = *a
+ return nil
+ })
+}
+
+// addressListVarFlag defines an address list flag with specified name and usage string.
+// The argument p points to a []mail.Address variable in which to store the value of the flag.
+func addressListVarFlag(p *[]mail.Address, name, usage string) {
+ flag.Func(name, usage, func(s string) error {
+ as, err := mail.ParseAddressList(s)
+ if err != nil {
+ return err
+ }
+ *p = nil // Clear out the list before appending.
+ for _, a := range as {
+ *p = append(*p, *a)
+ }
+ return nil
+ })
+}
diff --git a/go.mod b/go.mod
index f378f9e..08f76d4 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@
github.com/go-sql-driver/mysql v1.5.0
github.com/golang-migrate/migrate/v4 v4.15.0-beta.3
github.com/golang/protobuf v1.5.2
- github.com/google/go-cmp v0.5.7
+ github.com/google/go-cmp v0.5.8
github.com/google/go-github v17.0.0+incompatible
github.com/google/uuid v1.2.0
github.com/googleapis/gax-go/v2 v2.1.1
@@ -39,8 +39,10 @@
github.com/julienschmidt/httprouter v1.3.0
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/mattn/go-sqlite3 v1.14.6
+ github.com/sendgrid/sendgrid-go v3.11.1+incompatible
github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07
+ github.com/yuin/goldmark v1.4.12
go.opencensus.io v0.23.0
go4.org v0.0.0-20180809161055-417644f6feb5
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70
@@ -108,6 +110,7 @@
github.com/prometheus/common v0.15.0 // indirect
github.com/prometheus/procfs v0.2.0 // indirect
github.com/prometheus/statsd_exporter v0.20.0 // indirect
+ github.com/sendgrid/rest v2.6.9+incompatible // indirect
github.com/shurcooL/graphql v0.0.0-20220520033453-bdb1221e171e // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
go.uber.org/atomic v1.6.0 // indirect
diff --git a/go.sum b/go.sum
index b58de2c..913edec 100644
--- a/go.sum
+++ b/go.sum
@@ -405,8 +405,9 @@
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v35 v35.2.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs=
@@ -783,6 +784,10 @@
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
+github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=
+github.com/sendgrid/sendgrid-go v3.11.1+incompatible h1:ai0+woZ3r/+tKLQExznak5XerOFoD6S7ePO0lMV8WXo=
+github.com/sendgrid/sendgrid-go v3.11.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
@@ -843,6 +848,8 @@
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.12 h1:6hffw6vALvEDqJ19dOJvJKOoAOKe4NDaTqvd2sktGN0=
+github.com/yuin/goldmark v1.4.12/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
diff --git a/internal/relui/workflows.go b/internal/relui/workflows.go
index 47bf095..9990ff5 100644
--- a/internal/relui/workflows.go
+++ b/internal/relui/workflows.go
@@ -74,7 +74,7 @@
}
// RegisterMailDLCLDefinition registers a workflow definition for mailing a golang.org/dl CL
-// onto h, using e for the external service configuration.
+// onto h.
func RegisterMailDLCLDefinition(h *DefinitionHolder, tasks *task.VersionTasks) {
versions := workflow.Parameter{
Name: "Versions",
@@ -101,6 +101,108 @@
h.RegisterDefinition("mail-dl-cl", wd)
}
+// RegisterAnnounceDefinitions registers workflow definitions involving announcing
+// onto h.
+func RegisterAnnounceDefinitions(h *DefinitionHolder, tasks task.AnnounceMailTasks) {
+ version := workflow.Parameter{
+ Name: "Version",
+ Doc: `Version is the Go version that has been released.
+
+The version string must use the same format as Go tags.`,
+ }
+ security := workflow.Parameter{
+ Name: "Security (optional)",
+ ParameterType: workflow.SliceLong,
+ Doc: `Security is a list of descriptions, one for each distinct security fix included in this release, in Markdown format.
+
+The empty list means there are no security fixes included.
+
+This field applies only to minor releases.
+
+Past examples:
+• "encoding/pem: fix stack overflow in Decode
+
+ A large (more than 5 MB) PEM input can cause a stack overflow in Decode,
+ leading the program to crash.
+
+ Thanks to Juho Nurminen of Mattermost who reported the error.
+
+ This is CVE-2022-24675 and Go issue https://go.dev/issue/51853."
+• "crypto/elliptic: tolerate all oversized scalars in generic P-256
+
+ A crafted scalar input longer than 32 bytes can cause P256().ScalarMult
+ or P256().ScalarBaseMult to panic. Indirect uses through crypto/ecdsa and
+ crypto/tls are unaffected. amd64, arm64, ppc64le, and s390x are unaffected.
+
+ This was discovered thanks to a Project Wycheproof test vector.
+
+ This is CVE-2022-28327 and Go issue https://go.dev/issue/52075."`,
+ Example: `encoding/pem: fix stack overflow in Decode
+
+A large (more than 5 MB) PEM input can cause a stack overflow in Decode,
+leading the program to crash.
+
+Thanks to Juho Nurminen of Mattermost who reported the error.
+
+This is CVE-2022-24675 and Go issue https://go.dev/issue/51853.`,
+ }
+ names := workflow.Parameter{
+ Name: "Names (optional)",
+ ParameterType: workflow.SliceShort,
+ Doc: `Names is an optional list of release coordinator names to include in the sign-off message.`,
+ }
+
+ {
+ minorVersion := version
+ minorVersion.Example = "go1.18.2"
+ secondaryVersion := workflow.Parameter{
+ Name: "SecondaryVersion",
+ Doc: `SecondaryVersion is an older Go version that was also released.`,
+ Example: "go1.17.10",
+ }
+
+ wd := workflow.New()
+ sentMail := wd.Task("mail-announcement", func(ctx *workflow.TaskContext, v1, v2 string, sec, names []string) (task.SentMail, error) {
+ return tasks.AnnounceMinorRelease(ctx, task.ReleaseAnnouncement{Version: v1, SecondaryVersion: v2, Security: sec, Names: names})
+ }, wd.Parameter(minorVersion), wd.Parameter(secondaryVersion), wd.Parameter(security), wd.Parameter(names))
+ wd.Output("AnnouncementURL", wd.Task("await-announcement", tasks.AwaitAnnounceMail, sentMail))
+ h.RegisterDefinition("announce-minor", wd)
+ }
+ {
+ betaVersion := version
+ betaVersion.Example = "go1.19beta1"
+
+ wd := workflow.New()
+ sentMail := wd.Task("mail-announcement", func(ctx *workflow.TaskContext, v string, names []string) (task.SentMail, error) {
+ return tasks.AnnounceBetaRelease(ctx, task.ReleaseAnnouncement{Version: v, Names: names})
+ }, wd.Parameter(betaVersion), wd.Parameter(names))
+ wd.Output("AnnouncementURL", wd.Task("await-announcement", tasks.AwaitAnnounceMail, sentMail))
+ h.RegisterDefinition("announce-beta", wd)
+ }
+ {
+ rcVersion := version
+ rcVersion.Example = "go1.19rc1"
+
+ wd := workflow.New()
+ sentMail := wd.Task("mail-announcement", func(ctx *workflow.TaskContext, v string, names []string) (task.SentMail, error) {
+ return tasks.AnnounceRCRelease(ctx, task.ReleaseAnnouncement{Version: v, Names: names})
+ }, wd.Parameter(rcVersion), wd.Parameter(names))
+ wd.Output("AnnouncementURL", wd.Task("await-announcement", tasks.AwaitAnnounceMail, sentMail))
+ h.RegisterDefinition("announce-rc", wd)
+ }
+ {
+ majorVersion := version
+ majorVersion.Example = "go1.19"
+
+ wd := workflow.New()
+ sentMail := wd.Task("mail-announcement", func(ctx *workflow.TaskContext, v string, names []string) (task.SentMail, error) {
+ return tasks.AnnounceMajorRelease(ctx, task.ReleaseAnnouncement{Version: v, Names: names})
+ }, wd.Parameter(majorVersion), wd.Parameter(names))
+ wd.Output("AnnouncementURL", wd.Task("await-announcement", tasks.AwaitAnnounceMail, sentMail))
+ h.RegisterDefinition("announce-major", wd)
+ }
+}
+
// RegisterTweetDefinitions registers workflow definitions involving tweeting
// onto h, using e for the external service configuration.
func RegisterTweetDefinitions(h *DefinitionHolder, e task.ExternalConfig) {
diff --git a/internal/task/announce.go b/internal/task/announce.go
index 553af28..6af44ed 100644
--- a/internal/task/announce.go
+++ b/internal/task/announce.go
@@ -5,12 +5,29 @@
package task
import (
+ "bytes"
"embed"
"fmt"
+ "io"
+ "mime"
+ "net/http"
+ "net/mail"
+ "net/url"
"strings"
"text/template"
+ "time"
+ sendgrid "github.com/sendgrid/sendgrid-go"
+ sendgridmail "github.com/sendgrid/sendgrid-go/helpers/mail"
+ "github.com/yuin/goldmark"
+ "github.com/yuin/goldmark/ast"
+ "github.com/yuin/goldmark/extension"
+ "github.com/yuin/goldmark/renderer"
+ goldmarkhtml "github.com/yuin/goldmark/renderer/html"
+ goldmarktext "github.com/yuin/goldmark/text"
+ "golang.org/x/build/internal/workflow"
"golang.org/x/build/maintner/maintnerd/maintapi/version"
+ "golang.org/x/net/html"
)
type ReleaseAnnouncement struct {
@@ -39,6 +56,105 @@
Names []string
}
+// AnnounceMailTasks contains tasks related to the release announcement email.
+type AnnounceMailTasks struct {
+ SendGridAPIKey string
+
+ From mail.Address // An RFC 5322 address. For example, "Barry Gibbs <bg@example.com>".
+ To mail.Address
+ BCC []mail.Address
+}
+
+// SentMail represents an email that was sent.
+type SentMail struct {
+ Subject string // Subject of the email. Expected to be unique so it can be used to identify the email.
+}
+
+// AnnounceMinorRelease sends an email announcing a minor Go release to Google Groups.
+func (t AnnounceMailTasks) AnnounceMinorRelease(ctx *workflow.TaskContext, r ReleaseAnnouncement) (SentMail, error) {
+ if err := verifyGoVersions(r.Version, r.SecondaryVersion); err != nil {
+ return SentMail{}, err
+ }
+
+ return t.announceRelease(ctx, r)
+}
+
+// AnnounceBetaRelease sends an email announcing a beta Go release to Google Groups.
+func (t AnnounceMailTasks) AnnounceBetaRelease(ctx *workflow.TaskContext, r ReleaseAnnouncement) (SentMail, error) {
+ if r.SecondaryVersion != "" {
+ return SentMail{}, fmt.Errorf("got 2 Go versions, want 1")
+ }
+ if err := verifyGoVersions(r.Version); err != nil {
+ return SentMail{}, err
+ }
+
+ return t.announceRelease(ctx, r)
+}
+
+// AnnounceRCRelease sends an email announcing a Go release candidate to Google Groups.
+func (t AnnounceMailTasks) AnnounceRCRelease(ctx *workflow.TaskContext, r ReleaseAnnouncement) (SentMail, error) {
+ if r.SecondaryVersion != "" {
+ return SentMail{}, fmt.Errorf("got 2 Go versions, want 1")
+ }
+ if err := verifyGoVersions(r.Version); err != nil {
+ return SentMail{}, err
+ }
+
+ return t.announceRelease(ctx, r)
+}
+
+// AnnounceMajorRelease sends an email announcing a major Go release to Google Groups.
+func (t AnnounceMailTasks) AnnounceMajorRelease(ctx *workflow.TaskContext, r ReleaseAnnouncement) (SentMail, error) {
+ if r.SecondaryVersion != "" {
+ return SentMail{}, fmt.Errorf("got 2 Go versions, want 1")
+ }
+ if err := verifyGoVersions(r.Version); err != nil {
+ return SentMail{}, err
+ }
+
+ return t.announceRelease(ctx, r)
+}
+
+// announceRelease sends an email announcing a Go release.
+func (t AnnounceMailTasks) announceRelease(ctx *workflow.TaskContext, r ReleaseAnnouncement) (SentMail, error) {
+ if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
+ return SentMail{}, fmt.Errorf("insufficient time for announce release task; a minimum of a minute left on context is required")
+ }
+
+ // Generate the announcement email.
+ m, err := announcementMail(r)
+ if err != nil {
+ return SentMail{}, err
+ }
+ if log := ctx.Logger; log != nil {
+ log.Printf("announcement subject: %s\n", m.Subject)
+ log.Printf("\nannouncement body HTML:\n%s", m.BodyHTML)
+ log.Printf("\nannouncement body text:\n%s", m.BodyText)
+ }
+
+ // Confirm that this announcement doesn't already exist.
+ if threadURL, err := findGoogleGroupsThread(ctx, m.Subject); err != nil {
+ // Proceeding would risk sending a duplicate email, so error out instead.
+ return SentMail{}, fmt.Errorf("stopping early due to error checking for an existing Google Groups thread: %v", err)
+ } else if threadURL != "" {
+ // TODO(go.dev/issue/47406): Once this task is a part of a larger workflow (which may need
+ // to tolerate resuming, restarting, and so on), the case of the matching subject already
+ // being there should become considered as "success, keep going" rather than "error, stop".
+ return SentMail{}, fmt.Errorf("a Google Groups thread with matching subject %q already exists at %q, stopping", m.Subject, threadURL)
+ }
+
+ // Send the announcement email to the destination mailing lists.
+ if t.SendGridAPIKey == "" {
+ return SentMail{Subject: "[dry-run] " + m.Subject}, nil
+ }
+ err = t.sendMailViaSendGrid(m)
+ if err != nil {
+ return SentMail{}, err
+ }
+
+ return SentMail{m.Subject}, nil
+}
+
type mailContent struct {
Subject string
BodyHTML string
@@ -71,10 +187,33 @@
return mailContent{}, fmt.Errorf("email template %q doesn't support the Security field; this field can only be used in minor releases", name)
}
- // TODO(go.dev/issue/47405): Render the announcement template.
- // Get the email subject.
- // Render the email body.
- return mailContent{}, fmt.Errorf("not implemented yet")
+ // Render the announcement email template.
+ //
+ // It'll produce a valid message with a MIME header and a body, so parse it as such.
+ var buf bytes.Buffer
+ if err := announceTmpl.ExecuteTemplate(&buf, name, r); err != nil {
+ return mailContent{}, err
+ }
+ m, err := mail.ReadMessage(&buf)
+ if err != nil {
+ return mailContent{}, fmt.Errorf(`email template must be formatted like a mail message, but reading it failed: %v`, err)
+ }
+
+ // Get the email subject (it's a plain string, no further processing needed).
+ if _, ok := m.Header["Subject"]; !ok {
+ return mailContent{}, fmt.Errorf(`email template must have a "Subject" key in its MIME header, but it's not found`)
+ } else if n := len(m.Header["Subject"]); n != 1 {
+ return mailContent{}, fmt.Errorf(`email template must have a single "Subject" value in its MIME header, but have %d values`, n)
+ }
+ subject := m.Header.Get("Subject")
+
+ // Render the email body, in Markdown format at this point, to HTML and plain text.
+ html, text, err := renderMarkdown(m.Body)
+ if err != nil {
+ return mailContent{}, err
+ }
+
+ return mailContent{subject, html, text}, nil
}
// announceTmpl holds templates for Go release announcement emails.
@@ -137,3 +276,277 @@
//go:embed template
var tmplDir embed.FS
+
+// sendMailViaSendGrid sends an email by making
+// an authenticated request to the SendGrid API.
+func (t AnnounceMailTasks) sendMailViaSendGrid(m mailContent) error {
+ from, to := sendgridmail.Email(t.From), sendgridmail.Email(t.To)
+ req := sendgridmail.NewSingleEmail(&from, m.Subject, &to, m.BodyText, m.BodyHTML)
+ if len(req.Personalizations) != 1 {
+ return fmt.Errorf("internal error: len(req.Personalizations) is %d, want 1", len(req.Personalizations))
+ }
+ for _, bcc := range t.BCC {
+ bcc := sendgridmail.Email(bcc)
+ req.Personalizations[0].AddBCCs(&bcc)
+ }
+
+ no := false
+ req.TrackingSettings = &sendgridmail.TrackingSettings{
+ ClickTracking: &sendgridmail.ClickTrackingSetting{Enable: &no},
+ OpenTracking: &sendgridmail.OpenTrackingSetting{Enable: &no},
+ SubscriptionTracking: &sendgridmail.SubscriptionTrackingSetting{Enable: &no},
+ }
+
+ sg := sendgrid.NewSendClient(t.SendGridAPIKey)
+ resp, err := sg.Send(req)
+ if err != nil {
+ return err
+ } else if resp.StatusCode != http.StatusAccepted {
+ return fmt.Errorf("unexpected status %d %s, want 202 Accepted; body = %s", resp.StatusCode, http.StatusText(resp.StatusCode), resp.Body)
+ }
+ return nil
+}
+
+// AwaitAnnounceMail waits for an announcement email with the specified subject
+// to show up on Google Groups, and returns its canonical URL.
+func (t AnnounceMailTasks) AwaitAnnounceMail(ctx *workflow.TaskContext, m SentMail) (announcementURL string, _ error) {
+ // Find the URL for the announcement while giving the email a chance to be received and moderated.
+ started := time.Now()
+ poll := time.NewTicker(10 * time.Second)
+ defer poll.Stop()
+ updateLog := time.NewTicker(time.Minute)
+ defer updateLog.Stop()
+ for {
+ // Wait a bit, updating the log periodically.
+ select {
+ case <-ctx.Done():
+ return "", ctx.Err()
+ case <-poll.C:
+ case t := <-updateLog.C:
+ if log := ctx.Logger; log != nil {
+ log.Printf("... still waiting for %q to appear after %v ...\n", m.Subject, t.Sub(started))
+ }
+ continue
+ }
+
+ // See if our email is available by now.
+ threadURL, err := findGoogleGroupsThread(ctx, m.Subject)
+ if err != nil {
+ if log := ctx.Logger; log != nil {
+ log.Printf("findGoogleGroupsThread: %v", err)
+ }
+ continue
+ } else if threadURL == "" {
+ // Our email hasn't yet shown up. Wait more and try again.
+ continue
+ }
+ return threadURL, nil
+ }
+}
+
+// findGoogleGroupsThread fetches the first page of threads from the golang-announce
+// Google Groups mailing list, and looks for a thread with the matching subject line.
+// It returns its URL if found or the empty string if not found.
+func findGoogleGroupsThread(ctx *workflow.TaskContext, subject string) (threadURL string, _ error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://groups.google.com/g/golang-announce", nil)
+ if err != nil {
+ return "", err
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+ return "", fmt.Errorf("did not get acceptable status code: %v body: %q", resp.Status, body)
+ }
+ if ct, want := resp.Header.Get("Content-Type"), "text/html; charset=utf-8"; ct != want {
+ if log := ctx.Logger; log != nil {
+ log.Printf("findGoogleGroupsThread: got error response with non-'text/html; charset=utf-8' Content-Type header %q\n", ct)
+ }
+ if mediaType, _, err := mime.ParseMediaType(ct); err != nil {
+ return "", fmt.Errorf("bad Content-Type header %q: %v", ct, err)
+ } else if mediaType != "text/html" {
+ return "", fmt.Errorf("got media type %q, want %q", mediaType, "text/html")
+ }
+ }
+ doc, err := html.Parse(io.LimitReader(resp.Body, 5<<20))
+ if err != nil {
+ return "", err
+ }
+ var baseHref string
+ var linkHref string
+ var found bool
+ var f func(*html.Node)
+ f = func(n *html.Node) {
+ if n.Type == html.ElementNode && n.Data == "base" {
+ baseHref = href(n)
+ } else if n.Type == html.ElementNode && n.Data == "a" {
+ linkHref = href(n)
+ } else if n.Type == html.TextNode && n.Data == subject {
+ // Found our link. Break out.
+ found = true
+ return
+ }
+ for c := n.FirstChild; c != nil && !found; c = c.NextSibling {
+ f(c)
+ }
+ }
+ f(doc)
+ if !found {
+ // Thread not found on the first page.
+ return "", nil
+ }
+ base, err := url.Parse(baseHref)
+ if err != nil {
+ return "", err
+ }
+ link, err := url.Parse(linkHref)
+ if err != nil {
+ return "", err
+ }
+ threadURL = base.ResolveReference(link).String()
+ if !strings.HasPrefix(threadURL, announcementPrefix) {
+ return "", fmt.Errorf("found URL %q, but it doesn't have the expected prefix %q", threadURL, announcementPrefix)
+ }
+ return threadURL, nil
+}
+
+func href(n *html.Node) string {
+ for _, a := range n.Attr {
+ if a.Key == "href" {
+ return a.Val
+ }
+ }
+ return ""
+}
+
+// renderMarkdown parses Markdown source
+// and renders it to HTML and plain text.
+//
+// The Markdown specification and its various extensions are vast.
+// Here we support a small, simple set of Markdown features
+// that satisfies the needs of the announcement mail tasks.
+func renderMarkdown(r io.Reader) (html, text string, _ error) {
+ source, err := io.ReadAll(r)
+ if err != nil {
+ return "", "", err
+ }
+ // Configure a Markdown parser and HTML renderer fairly closely
+ // to how it's done in x/website, just without raw HTML support
+ // and other extensions we don't need for announcement emails.
+ md := goldmark.New(
+ goldmark.WithRendererOptions(goldmarkhtml.WithHardWraps()),
+ goldmark.WithExtensions(
+ extension.NewLinkify(extension.WithLinkifyAllowedProtocols([][]byte{[]byte("https:")})),
+ ),
+ )
+ doc := md.Parser().Parse(goldmarktext.NewReader(source))
+ var htmlBuf, textBuf bytes.Buffer
+ err = md.Renderer().Render(&htmlBuf, source, doc)
+ if err != nil {
+ return "", "", err
+ }
+ err = (markdownToTextRenderer{}).Render(&textBuf, source, doc)
+ if err != nil {
+ return "", "", err
+ }
+ return htmlBuf.String(), textBuf.String(), nil
+}
+
+// markdownToTextRenderer is a simple goldmark/renderer.Renderer implementation
+// that renders Markdown to plain text for the needs of announcement mail tasks.
+//
+// It produces an output suitable for email clients that cannot (or choose not to)
+// display the HTML version of the email. (It also helps a bit with the readability
+// of our test data, since without a browser plain text is more readable than HTML.)
+//
+// The output is mostly plain text that doesn't preserve Markdown syntax (for example,
+// `code` is rendered without backticks), though there is very lightweight formatting
+// applied (links are written as "text <URL>").
+//
+// We can in theory choose to delete this renderer at any time if its maintenance costs
+// start to outweight its benefits, since Markdown by definition is designed to be human
+// readable and can be used as plain text without any processing.
+type markdownToTextRenderer struct{}
+
+func (markdownToTextRenderer) Render(w io.Writer, source []byte, n ast.Node) error {
+ const debug = false
+ if debug {
+ n.Dump(source, 0)
+ }
+
+ var (
+ markers []byte // Stack of list markers, from outermost to innermost.
+ )
+ walk := func(n ast.Node, entering bool) (ast.WalkStatus, error) {
+ if entering {
+ if n.Type() == ast.TypeBlock && n.PreviousSibling() != nil {
+ // Print a blank line between block nodes.
+ switch n.PreviousSibling().Kind() {
+ default:
+ fmt.Fprint(w, "\n\n")
+ case ast.KindCodeBlock:
+ // A code block always ends with a newline, so only need one more.
+ fmt.Fprintln(w)
+ }
+
+ // If we're in a list, indent accordingly.
+ if n.Kind() != ast.KindListItem {
+ fmt.Fprint(w, strings.Repeat("\t", len(markers)))
+ }
+ }
+
+ switch n := n.(type) {
+ case *ast.Text:
+ fmt.Fprintf(w, "%s", n.Text(source))
+
+ // Print a line break.
+ if n.SoftLineBreak() || n.HardLineBreak() {
+ fmt.Fprintln(w)
+
+ // If we're in a list, indent accordingly.
+ fmt.Fprint(w, strings.Repeat("\t", len(markers)))
+ }
+ case *ast.CodeBlock:
+ indent := strings.Repeat("\t", len(markers)+1) // Indent if in a list, plus one more since it's a code block.
+ for i := 0; i < n.Lines().Len(); i++ {
+ s := n.Lines().At(i)
+ fmt.Fprint(w, indent, string(source[s.Start:s.Stop]))
+ }
+ case *ast.AutoLink:
+ // Auto-links are printed as is in plain text.
+ //
+ // For example, the Markdown "https://go.dev/issue/123"
+ // is rendered as plain text "https://go.dev/issue/123".
+ fmt.Fprint(w, string(n.Label(source)))
+ case *ast.List:
+ // Push list marker on the stack.
+ markers = append(markers, n.Marker)
+ case *ast.ListItem:
+ fmt.Fprintf(w, "%s%c\t", strings.Repeat("\t", len(markers)-1), markers[len(markers)-1])
+ }
+ } else {
+ switch n := n.(type) {
+ case *ast.Link:
+ // Append the link's URL after its text.
+ //
+ // For example, the Markdown "[security policy](https://go.dev/security)"
+ // is rendered as plain text "security policy <https://go.dev/security>".
+ fmt.Fprintf(w, " <%s>", n.Destination)
+ case *ast.List:
+ // Pop list marker off the stack.
+ markers = markers[:len(markers)-1]
+ }
+
+ if n.Type() == ast.TypeDocument && n.ChildCount() != 0 {
+ // Print a newline at the end of the document, if it's not empty.
+ fmt.Fprintln(w)
+ }
+ }
+ return ast.WalkContinue, nil
+ }
+ return ast.Walk(n, walk)
+}
+func (markdownToTextRenderer) AddOptions(...renderer.Option) {}
diff --git a/internal/task/announce_test.go b/internal/task/announce_test.go
index fc90ee1..569a317 100644
--- a/internal/task/announce_test.go
+++ b/internal/task/announce_test.go
@@ -5,13 +5,32 @@
package task
import (
+ "bytes"
+ "context"
"os"
"path/filepath"
+ "strings"
"testing"
+ "time"
"github.com/google/go-cmp/cmp"
+ "golang.org/x/build/internal/workflow"
)
+// Test that the task doesn't start running if the provided
+// context doesn't have sufficient time for the task to run.
+func TestAnnounceReleaseShortContext(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ r := ReleaseAnnouncement{Version: "go1.18.1", SecondaryVersion: "go1.17.8"}
+ _, err := (AnnounceMailTasks{}).AnnounceMinorRelease(&workflow.TaskContext{Context: ctx}, r)
+ if err == nil {
+ t.Errorf("want non-nil error")
+ } else if !strings.HasPrefix(err.Error(), "insufficient time") {
+ t.Errorf("want error that starts with 'insufficient time' instead of: %s", err)
+ }
+}
+
func TestAnnouncementMail(t *testing.T) {
tests := [...]struct {
name string
@@ -86,7 +105,6 @@
t.Run(tc.name, func(t *testing.T) {
m, err := announcementMail(tc.in)
if err != nil {
- t.Skip("announcementMail is not implemented yet") // TODO(go.dev/issue/47405): Implement.
t.Fatal("announcementMail returned non-nil error:", err)
}
if *updateFlag {
@@ -128,3 +146,188 @@
t.Fatal(err)
}
}
+
+func TestAnnounceRelease(t *testing.T) {
+ if testing.Short() {
+ t.Skip("not running test that uses internet in short mode")
+ }
+
+ tests := [...]struct {
+ name string
+ taskFn func(*workflow.TaskContext, ReleaseAnnouncement) (SentMail, error)
+ in ReleaseAnnouncement
+ want SentMail
+ wantLog string
+ }{
+ {
+ name: "minor",
+ taskFn: (AnnounceMailTasks{}).AnnounceMinorRelease,
+ in: ReleaseAnnouncement{
+ Version: "go1.18.1",
+ SecondaryVersion: "go1.17.8", // Intentionally not 1.17.9 so the real email doesn't get in the way.
+ Names: []string{"Alice", "Bob", "Charlie"},
+ },
+ want: SentMail{Subject: "[dry-run] Go 1.18.1 and Go 1.17.8 are released"},
+ wantLog: `announcement subject: Go 1.18.1 and Go 1.17.8 are released
+
+announcement body HTML:
+<p>Hello gophers,</p>
+<p>We have just released Go versions 1.18.1 and 1.17.8, minor point releases.</p>
+<p>View the release notes for more information:<br>
+<a href="https://go.dev/doc/devel/release#go1.18.1">https://go.dev/doc/devel/release#go1.18.1</a></p>
+<p>You can download binary and source distributions from the Go website:<br>
+<a href="https://go.dev/dl/">https://go.dev/dl/</a></p>
+<p>To compile from source using a Git clone, update to the release with<br>
+<code>git checkout go1.18.1</code> and build as usual.</p>
+<p>Thanks to everyone who contributed to the releases.</p>
+<p>Cheers,<br>
+Alice, Bob, and Charlie for the Go team</p>
+
+announcement body text:
+Hello gophers,
+
+We have just released Go versions 1.18.1 and 1.17.8, minor point releases.
+
+View the release notes for more information:
+https://go.dev/doc/devel/release#go1.18.1
+
+You can download binary and source distributions from the Go website:
+https://go.dev/dl/
+
+To compile from source using a Git clone, update to the release with
+git checkout go1.18.1 and build as usual.
+
+Thanks to everyone who contributed to the releases.
+
+Cheers,
+Alice, Bob, and Charlie for the Go team` + "\n",
+ },
+ // Just one test case is enough, since TestAnnouncementMail
+ // has very thorough coverage for all release types.
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ // Call the announce task function in dry-run mode so it
+ // doesn't actually try to announce, but capture its log.
+ var buf bytes.Buffer
+ ctx := &workflow.TaskContext{Context: context.Background(), Logger: fmtWriter{&buf}}
+ sentMail, err := tc.taskFn(ctx, tc.in)
+ if err != nil {
+ t.Fatal("task function returned non-nil error:", err)
+ }
+ if diff := cmp.Diff(tc.want, sentMail); diff != "" {
+ t.Errorf("sent mail mismatch (-want +got):\n%s", diff)
+ }
+ if diff := cmp.Diff(tc.wantLog, buf.String()); diff != "" {
+ t.Errorf("log mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
+func TestFindGoogleGroupsThread(t *testing.T) {
+ if testing.Short() {
+ t.Skip("not running test that uses internet in short mode")
+ }
+
+ threadURL, err := findGoogleGroupsThread(&workflow.TaskContext{
+ Context: context.Background(),
+ }, "[security] Go 1.18.3 and Go 1.17.11 are released")
+ if err != nil {
+ // Note: These test failures are only actionable if the error is not
+ // a transient network one.
+ t.Fatalf("findGoogleGroupsThread returned a non-nil error: %v", err)
+ }
+ // Just log the threadURL since we can't rely on stable output.
+ // This test is mostly for debugging if we need to.
+ t.Logf("threadURL: %q\n", threadURL)
+}
+
+func TestMarkdownToText(t *testing.T) {
+ const in = `Hello gophers,
+
+This is a simple Markdown document that exercises
+a limited set of features used in email templates.
+
+There may be security fixes following the [security policy](https://go.dev/security):
+
+- abc: Read hangs on extremely large input
+
+ On an operating system, ` + "`Read`" + ` will hang indefinitely if
+ the buffer size is larger than 1 << 64 - 1 bytes.
+
+ Thanks to Gopher A for reporting the issue.
+
+ This is CVE-123 and Go issue https://go.dev/issue/123.
+
+- xyz: Clean("X") returns "Y" when Z
+
+ Some description of the problem here.
+
+ Markdown allows one to use backslash escapes, like \_underscore\_
+ or \*literal asterisks\*, so we might encounter that.
+
+View release notes:
+https://go.dev/doc/devel/release#go1.18.3
+
+You can download binaries:
+https://go.dev/dl/
+
+To builds from source, use
+` + "`git checkout`" + `.
+
+An easy way to try go1.19beta1
+is by using the go command:
+$ go install example.org@latest
+$ example download
+
+That's all for now.
+`
+ _, got, err := renderMarkdown(strings.NewReader(in))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ const want = `Hello gophers,
+
+This is a simple Markdown document that exercises
+a limited set of features used in email templates.
+
+There may be security fixes following the security policy <https://go.dev/security>:
+
+- abc: Read hangs on extremely large input
+
+ On an operating system, Read will hang indefinitely if
+ the buffer size is larger than 1 << 64 - 1 bytes.
+
+ Thanks to Gopher A for reporting the issue.
+
+ This is CVE-123 and Go issue https://go.dev/issue/123.
+
+- xyz: Clean("X") returns "Y" when Z
+
+ Some description of the problem here.
+
+ Markdown allows one to use backslash escapes, like \_underscore\_
+ or \*literal asterisks\*, so we might encounter that.
+
+View release notes:
+https://go.dev/doc/devel/release#go1.18.3
+
+You can download binaries:
+https://go.dev/dl/
+
+To builds from source, use
+git checkout.
+
+An easy way to try go1.19beta1
+is by using the go command:
+$ go install example.org@latest
+$ example download
+
+That's all for now.
+`
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("plain text rendering mismatch (-want +got):\n%s", diff)
+ }
+}
