blob: d5423b392910c3432a0cfef535e4a09f65e4875b [file] [log] [blame]
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package task
import (
"bytes"
"embed"
"fmt"
"io"
"mime"
"net/http"
"net/mail"
"net/url"
"strings"
"text/template"
"time"
sendgrid "github.com/sendgrid/sendgrid-go"
sendgridmail "github.com/sendgrid/sendgrid-go/helpers/mail"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/renderer"
goldmarkhtml "github.com/yuin/goldmark/renderer/html"
goldmarktext "github.com/yuin/goldmark/text"
"golang.org/x/build/internal/gophers"
"golang.org/x/build/internal/workflow"
"golang.org/x/build/maintner/maintnerd/maintapi/version"
"golang.org/x/net/html"
)
type releaseAnnouncement struct {
// Version is the Go version that has been released.
//
// The version string must use the same format as Go tags. For example:
// • "go1.17.2" for a minor Go release
// • "go1.18" for a major Go release
// • "go1.18beta1" or "go1.18rc1" for a pre-release
Version string
// SecondaryVersion is an older Go version that was also released.
// This only applies to minor releases when two releases are made.
// For example, "go1.16.10".
SecondaryVersion string
// Security is a list of descriptions, one for each distinct
// security fix included in this release, in Markdown format.
//
// The empty list means there are no security fixes included.
//
// This field applies only to minor releases; it is an error
// to try to use it another release type.
Security []string
// Names is an optional list of release coordinator names to
// include in the sign-off message.
Names []string
}
type releasePreAnnouncement struct {
// Target is the planned date for the release.
Target Date
// Version is the Go version that will be released.
//
// The version string must use the same format as Go tags. For example, "go1.17.2".
Version string
// SecondaryVersion is an older Go version that will also be released.
// This only applies when two releases are planned. For example, "go1.16.10".
SecondaryVersion string
// Security is the security content to be included in the
// release pre-announcement. It should not reveal details
// beyond what's allowed by the security policy.
Security string
// Names is an optional list of release coordinator names to
// include in the sign-off message.
Names []string
}
// A Date represents a single calendar day (year, month, day).
//
// This type does not include location information, and
// therefore does not describe a unique 24-hour timespan.
//
// TODO(go.dev/issue/19700): Start using time.Day or so when available.
type Date struct {
Year int // Year (for example, 2009).
Month time.Month // Month of the year (January = 1, ...).
Day int // Day of the month, starting at 1.
}
func (d Date) String() string { return d.Format("2006-01-02") }
func (d Date) Format(layout string) string {
return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, time.UTC).Format(layout)
}
func (d Date) After(year int, month time.Month, day int) bool {
return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, time.UTC).
After(time.Date(year, month, day, 0, 0, 0, 0, time.UTC))
}
// AnnounceMailTasks contains tasks related to the release (pre-)announcement email.
type AnnounceMailTasks struct {
// SendMail sends an email with the given header and content
// using an externally-provided implementation.
//
// Email delivery happens asynchronously, so SendMail returns a nil error
// if the transmission was started successfully, but that error value
// doesn't indicate anything about the status of the delivery.
SendMail func(MailHeader, MailContent) error
// AnnounceMailHeader is the header to use for the release (pre-)announcement email.
AnnounceMailHeader MailHeader
// testHookNow is optionally set by tests to override time.Now.
testHookNow func() time.Time
}
// SentMail represents an email that was sent.
type SentMail struct {
Subject string // Subject of the email. Expected to be unique so it can be used to identify the email.
}
// AnnounceRelease sends an email announcing a Go release to Google Groups.
func (t AnnounceMailTasks) AnnounceRelease(ctx *workflow.TaskContext, versions []string, security []string, users []string) (SentMail, error) {
if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
return SentMail{}, fmt.Errorf("insufficient time for announce release task; a minimum of a minute left on context is required")
}
if err := oneOrTwoGoVersions(versions); err != nil {
return SentMail{}, err
}
names, err := coordinatorFirstNames(users)
if err != nil {
return SentMail{}, err
}
r := releaseAnnouncement{
Version: versions[0],
Security: security,
Names: names,
}
if len(versions) == 2 {
r.SecondaryVersion = versions[1]
}
// Generate the announcement email.
m, err := announcementMail(r)
if err != nil {
return SentMail{}, err
}
ctx.Printf("announcement subject: %s\n\n", m.Subject)
ctx.Printf("announcement body HTML:\n%s\n", m.BodyHTML)
ctx.Printf("announcement body text:\n%s", m.BodyText)
// Before sending, check to see if this announcement already exists.
if threadURL, err := findGoogleGroupsThread(ctx, m.Subject); err != nil {
// Proceeding would risk sending a duplicate email, so error out instead.
return SentMail{}, fmt.Errorf("stopping early due to error checking for an existing Google Groups thread: %v", err)
} else if threadURL != "" {
// This should never happen since this task runs once per release.
// It can happen under unusual circumstances, for example if the task crashes after
// mailing but before completion, or if parts of the release workflow are restarted,
// or if a human mails the announcement email manually out of band.
//
// So if we see that the email exists, consider it as "task completed successfully"
// and pretend we were the ones that sent it, so the high level workflow can keep going.
ctx.Printf("a Google Groups thread with matching subject %q already exists at %q, so we'll consider that as it being sent successfully", m.Subject, threadURL)
return SentMail{m.Subject}, nil
}
// Send the announcement email to the destination mailing lists.
if t.SendMail == nil {
return SentMail{Subject: "[dry-run] " + m.Subject}, nil
}
ctx.DisableRetries()
err = t.SendMail(t.AnnounceMailHeader, m)
if err != nil {
return SentMail{}, err
}
return SentMail{m.Subject}, nil
}
// PreAnnounceRelease sends an email pre-announcing a Go release
// containing PRIVATE track security fixes planned for the target date.
func (t AnnounceMailTasks) PreAnnounceRelease(ctx *workflow.TaskContext, versions []string, target Date, security string, users []string) (SentMail, error) {
if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
return SentMail{}, fmt.Errorf("insufficient time for pre-announce release task; a minimum of a minute left on context is required")
}
if err := oneOrTwoGoVersions(versions); err != nil {
return SentMail{}, err
}
now := time.Now().UTC()
if t.testHookNow != nil {
now = t.testHookNow()
}
if !target.After(now.Year(), now.Month(), now.Day()) { // A very simple check. Improve as needed.
return SentMail{}, fmt.Errorf("target release date is not in the future")
}
if security == "" {
return SentMail{}, fmt.Errorf("security content is not specified")
}
names, err := coordinatorFirstNames(users)
if err != nil {
return SentMail{}, err
}
r := releasePreAnnouncement{
Target: target,
Version: versions[0],
Security: security,
Names: names,
}
if len(versions) == 2 {
r.SecondaryVersion = versions[1]
}
// Generate the pre-announcement email.
m, err := announcementMail(r)
if err != nil {
return SentMail{}, err
}
ctx.Printf("pre-announcement subject: %s\n\n", m.Subject)
ctx.Printf("pre-announcement body HTML:\n%s\n", m.BodyHTML)
ctx.Printf("pre-announcement body text:\n%s", m.BodyText)
// Before sending, check to see if this pre-announcement already exists.
if threadURL, err := findGoogleGroupsThread(ctx, m.Subject); err != nil {
return SentMail{}, fmt.Errorf("stopping early due to error checking for an existing Google Groups thread: %v", err)
} else if threadURL != "" {
ctx.Printf("a Google Groups thread with matching subject %q already exists at %q, so we'll consider that as it being sent successfully", m.Subject, threadURL)
return SentMail{m.Subject}, nil
}
// Send the pre-announcement email to the destination mailing lists.
if t.SendMail == nil {
return SentMail{Subject: "[dry-run] " + m.Subject}, nil
}
ctx.DisableRetries()
err = t.SendMail(t.AnnounceMailHeader, m)
if err != nil {
return SentMail{}, err
}
return SentMail{m.Subject}, nil
}
func coordinatorFirstNames(users []string) ([]string, error) {
return mapCoordinators(users, func(p *gophers.Person) string {
name, _, _ := strings.Cut(p.Name, " ")
return name
})
}
func coordinatorEmails(users []string) ([]string, error) {
return mapCoordinators(users, func(p *gophers.Person) string {
return p.Gerrit
})
}
func mapCoordinators(users []string, f func(*gophers.Person) string) ([]string, error) {
var outs []string
for _, user := range users {
person := gophers.GetPerson(user + "@golang.org")
if person == nil {
person = gophers.GetPerson(user + "@google.com")
}
if person == nil {
return nil, fmt.Errorf("unknown username %q: no @golang or @google account", user)
}
outs = append(outs, f(person))
}
return outs, nil
}
// A MailHeader is an email header.
type MailHeader struct {
From mail.Address // An RFC 5322 address. For example, "Barry Gibbs <bg@example.com>".
To mail.Address
BCC []mail.Address
}
// A MailContent holds the content of an email.
type MailContent struct {
Subject string
BodyHTML string
BodyText string
}
// announcementMail generates the (pre-)announcement email using data,
// which must be one of these types:
// - releaseAnnouncement for a release announcement
// - releasePreAnnouncement for a release pre-announcement
func announcementMail(data any) (MailContent, error) {
// Select the appropriate template name.
var name string
switch r := data.(type) {
case releaseAnnouncement:
if i := strings.Index(r.Version, "beta"); i != -1 { // A beta release.
name = "announce-beta.md"
} else if i := strings.Index(r.Version, "rc"); i != -1 { // Release Candidate.
name = "announce-rc.md"
} else if strings.Count(r.Version, ".") == 1 { // Major release like "go1.X".
name = "announce-major.md"
} else if strings.Count(r.Version, ".") == 2 { // Minor release like "go1.X.Y".
name = "announce-minor.md"
} else {
return MailContent{}, fmt.Errorf("unknown version format: %q", r.Version)
}
if len(r.Security) > 0 && name != "announce-minor.md" {
// The Security field isn't supported in templates other than minor,
// so report an error instead of silently dropping it.
//
// Note: Maybe in the future we'd want to consider support for including sentences like
// "This beta release includes the same security fixes as in Go X.Y.Z and Go A.B.C.",
// but we'll have a better idea after these initial templates get more practical use.
return MailContent{}, fmt.Errorf("email template %q doesn't support the Security field; this field can only be used in minor releases", name)
} else if r.SecondaryVersion != "" && name != "announce-minor.md" {
return MailContent{}, fmt.Errorf("email template %q doesn't support more than one release; the SecondaryVersion field can only be used in minor releases", name)
}
case releasePreAnnouncement:
name = "pre-announce-minor.md"
default:
return MailContent{}, fmt.Errorf("unknown template data type %T", data)
}
// Render the (pre-)announcement email template.
//
// It'll produce a valid message with a MIME header and a body, so parse it as such.
var buf bytes.Buffer
if err := announceTmpl.ExecuteTemplate(&buf, name, data); err != nil {
return MailContent{}, err
}
m, err := mail.ReadMessage(&buf)
if err != nil {
return MailContent{}, fmt.Errorf(`email template must be formatted like a mail message, but reading it failed: %v`, err)
}
// Get the email subject (it's a plain string, no further processing needed).
if _, ok := m.Header["Subject"]; !ok {
return MailContent{}, fmt.Errorf(`email template must have a "Subject" key in its MIME header, but it's not found`)
} else if n := len(m.Header["Subject"]); n != 1 {
return MailContent{}, fmt.Errorf(`email template must have a single "Subject" value in its MIME header, but have %d values`, n)
}
subject := m.Header.Get("Subject")
// Render the email body, in Markdown format at this point, to HTML and plain text.
html, text, err := renderMarkdown(m.Body)
if err != nil {
return MailContent{}, err
}
return MailContent{subject, html, text}, nil
}
// announceTmpl holds templates for Go release announcement emails.
//
// Each email template starts with a MIME-style header with a Subject key,
// and the rest of it is Markdown for the email body.
var announceTmpl = template.Must(template.New("").Funcs(template.FuncMap{
"join": func(s []string) string {
switch len(s) {
case 0:
return ""
case 1:
return s[0]
case 2:
return s[0] + " and " + s[1]
default:
return strings.Join(s[:len(s)-1], ", ") + ", and " + s[len(s)-1]
}
},
"indent": func(s string) string { return "\t" + strings.ReplaceAll(s, "\n", "\n\t") },
// subjectPrefix returns the email subject prefix for release r, if any.
"subjectPrefix": func(r releaseAnnouncement) string {
switch {
case len(r.Security) > 0:
// Include a security prefix as documented at https://go.dev/security#receiving-security-updates:
//
// > The best way to receive security announcements is to subscribe to the golang-announce mailing list.
// > Any messages pertaining to a security issue will be prefixed with [security].
//
return "[security]"
default:
return ""
}
},
// short and helpers below manipulate valid Go version strings
// for the current needs of the announcement templates.
"short": func(v string) string { return strings.TrimPrefix(v, "go") },
// major extracts the major part of a valid Go version.
// For example, major("go1.18.4") == "1.18".
"major": func(v string) (string, error) {
x, ok := version.Go1PointX(v)
if !ok {
return "", fmt.Errorf("internal error: version.Go1PointX(%q) is not ok", v)
}
return fmt.Sprintf("1.%d", x), nil
},
// build extracts the pre-release build number of a valid Go version.
// For example, build("go1.19beta2") == "2".
"build": func(v string) (string, error) {
if i := strings.Index(v, "beta"); i != -1 {
return v[i+len("beta"):], nil
} else if i := strings.Index(v, "rc"); i != -1 {
return v[i+len("rc"):], nil
}
return "", fmt.Errorf("internal error: unhandled pre-release Go version %q", v)
},
}).ParseFS(tmplDir, "template/announce-*.md", "template/pre-announce-minor.md"))
//go:embed template
var tmplDir embed.FS
type realSendGridMailClient struct {
sg *sendgrid.Client
}
// NewSendGridMailClient creates a SendGrid mail client
// authenticated with the given API key.
func NewSendGridMailClient(sendgridAPIKey string) realSendGridMailClient {
return realSendGridMailClient{sg: sendgrid.NewSendClient(sendgridAPIKey)}
}
// SendMail sends an email by making an authenticated request to the SendGrid API.
func (c realSendGridMailClient) SendMail(h MailHeader, m MailContent) error {
from, to := sendgridmail.Email(h.From), sendgridmail.Email(h.To)
req := sendgridmail.NewSingleEmail(&from, m.Subject, &to, m.BodyText, m.BodyHTML)
if len(req.Personalizations) != 1 {
return fmt.Errorf("internal error: len(req.Personalizations) is %d, want 1", len(req.Personalizations))
}
for _, bcc := range h.BCC {
bcc := sendgridmail.Email(bcc)
req.Personalizations[0].AddBCCs(&bcc)
}
no := false
req.TrackingSettings = &sendgridmail.TrackingSettings{
ClickTracking: &sendgridmail.ClickTrackingSetting{Enable: &no},
OpenTracking: &sendgridmail.OpenTrackingSetting{Enable: &no},
SubscriptionTracking: &sendgridmail.SubscriptionTrackingSetting{Enable: &no},
}
resp, err := c.sg.Send(req)
if err != nil {
return err
} else if resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("unexpected status %d %s, want 202 Accepted; body = %s", resp.StatusCode, http.StatusText(resp.StatusCode), resp.Body)
}
return nil
}
// AwaitAnnounceMail waits for an announcement email with the specified subject
// to show up on Google Groups, and returns its canonical URL.
func (t AnnounceMailTasks) AwaitAnnounceMail(ctx *workflow.TaskContext, m SentMail) (announcementURL string, _ error) {
// Find the URL for the announcement while giving the email a chance to be received and moderated.
check := func() (string, bool, error) {
// See if our email is available by now.
threadURL, err := findGoogleGroupsThread(ctx, m.Subject)
if err != nil {
ctx.Printf("findGoogleGroupsThread: %v", err)
return "", false, nil
}
return threadURL, threadURL != "", nil
}
return AwaitCondition(ctx, 10*time.Second, check)
}
// findGoogleGroupsThread fetches the first page of threads from the golang-announce
// Google Groups mailing list, and looks for a thread with the matching subject line.
// It returns its URL if found or the empty string if not found.
func findGoogleGroupsThread(ctx *workflow.TaskContext, subject string) (threadURL string, _ error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://groups.google.com/g/golang-announce", nil)
if err != nil {
return "", err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return "", fmt.Errorf("did not get acceptable status code: %v body: %q", resp.Status, body)
}
if ct, want := resp.Header.Get("Content-Type"), "text/html; charset=utf-8"; ct != want {
ctx.Printf("findGoogleGroupsThread: got response with non-'text/html; charset=utf-8' Content-Type header %q\n", ct)
if mediaType, _, err := mime.ParseMediaType(ct); err != nil {
return "", fmt.Errorf("bad Content-Type header %q: %v", ct, err)
} else if mediaType != "text/html" {
return "", fmt.Errorf("got media type %q, want %q", mediaType, "text/html")
}
}
doc, err := html.Parse(io.LimitReader(resp.Body, 5<<20))
if err != nil {
return "", err
}
var baseHref string
var linkHref string
var found bool
var f func(*html.Node)
f = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "base" {
baseHref = href(n)
} else if n.Type == html.ElementNode && n.Data == "a" {
linkHref = href(n)
} else if n.Type == html.TextNode && n.Data == subject {
// Found our link. Break out.
found = true
return
}
for c := n.FirstChild; c != nil && !found; c = c.NextSibling {
f(c)
}
}
f(doc)
if !found {
// Thread not found on the first page.
return "", nil
}
base, err := url.Parse(baseHref)
if err != nil {
return "", err
}
link, err := url.Parse(linkHref)
if err != nil {
return "", err
}
threadURL = base.ResolveReference(link).String()
const announcementPrefix = "https://groups.google.com/g/golang-announce/c/"
if !strings.HasPrefix(threadURL, announcementPrefix) {
return "", fmt.Errorf("found URL %q, but it doesn't have the expected prefix %q", threadURL, announcementPrefix)
}
return threadURL, nil
}
func href(n *html.Node) string {
for _, a := range n.Attr {
if a.Key == "href" {
return a.Val
}
}
return ""
}
// renderMarkdown parses Markdown source
// and renders it to HTML and plain text.
//
// The Markdown specification and its various extensions are vast.
// Here we support a small, simple set of Markdown features
// that satisfies the needs of the announcement mail tasks.
func renderMarkdown(r io.Reader) (html, text string, _ error) {
source, err := io.ReadAll(r)
if err != nil {
return "", "", err
}
// Configure a Markdown parser and HTML renderer fairly closely
// to how it's done in x/website, just without raw HTML support
// and other extensions we don't need for announcement emails.
md := goldmark.New(
goldmark.WithRendererOptions(goldmarkhtml.WithHardWraps()),
goldmark.WithExtensions(
extension.NewLinkify(extension.WithLinkifyAllowedProtocols([][]byte{[]byte("https:")})),
),
)
doc := md.Parser().Parse(goldmarktext.NewReader(source))
var htmlBuf, textBuf bytes.Buffer
err = md.Renderer().Render(&htmlBuf, source, doc)
if err != nil {
return "", "", err
}
err = (markdownToTextRenderer{}).Render(&textBuf, source, doc)
if err != nil {
return "", "", err
}
return htmlBuf.String(), textBuf.String(), nil
}
// markdownToTextRenderer is a simple goldmark/renderer.Renderer implementation
// that renders Markdown to plain text for the needs of announcement mail tasks.
//
// It produces an output suitable for email clients that cannot (or choose not to)
// display the HTML version of the email. (It also helps a bit with the readability
// of our test data, since without a browser plain text is more readable than HTML.)
//
// The output is mostly plain text that doesn't preserve Markdown syntax (for example,
// `code` is rendered without backticks), though there is very lightweight formatting
// applied (links are written as "text <URL>").
//
// We can in theory choose to delete this renderer at any time if its maintenance costs
// start to outweight its benefits, since Markdown by definition is designed to be human
// readable and can be used as plain text without any processing.
type markdownToTextRenderer struct{}
func (markdownToTextRenderer) Render(w io.Writer, source []byte, n ast.Node) error {
const debug = false
if debug {
n.Dump(source, 0)
}
var (
markers []byte // Stack of list markers, from outermost to innermost.
)
walk := func(n ast.Node, entering bool) (ast.WalkStatus, error) {
if entering {
if n.Type() == ast.TypeBlock && n.PreviousSibling() != nil {
// Print a blank line between block nodes.
switch n.PreviousSibling().Kind() {
default:
fmt.Fprint(w, "\n\n")
case ast.KindCodeBlock:
// A code block always ends with a newline, so only need one more.
fmt.Fprintln(w)
}
// If we're in a list, indent accordingly.
if n.Kind() != ast.KindListItem {
fmt.Fprint(w, strings.Repeat("\t", len(markers)))
}
}
switch n := n.(type) {
case *ast.Text:
fmt.Fprintf(w, "%s", n.Text(source))
// Print a line break.
if n.SoftLineBreak() || n.HardLineBreak() {
fmt.Fprintln(w)
// If we're in a list, indent accordingly.
fmt.Fprint(w, strings.Repeat("\t", len(markers)))
}
case *ast.CodeBlock:
indent := strings.Repeat("\t", len(markers)+1) // Indent if in a list, plus one more since it's a code block.
for i := 0; i < n.Lines().Len(); i++ {
s := n.Lines().At(i)
fmt.Fprint(w, indent, string(source[s.Start:s.Stop]))
}
case *ast.AutoLink:
// Auto-links are printed as is in plain text.
//
// For example, the Markdown "https://go.dev/issue/123"
// is rendered as plain text "https://go.dev/issue/123".
fmt.Fprint(w, string(n.Label(source)))
case *ast.List:
// Push list marker on the stack.
markers = append(markers, n.Marker)
case *ast.ListItem:
fmt.Fprintf(w, "%s%c\t", strings.Repeat("\t", len(markers)-1), markers[len(markers)-1])
}
} else {
switch n := n.(type) {
case *ast.Link:
// Append the link's URL after its text.
//
// For example, the Markdown "[security policy](https://go.dev/security)"
// is rendered as plain text "security policy <https://go.dev/security>".
fmt.Fprintf(w, " <%s>", n.Destination)
case *ast.List:
// Pop list marker off the stack.
markers = markers[:len(markers)-1]
}
if n.Type() == ast.TypeDocument && n.ChildCount() != 0 {
// Print a newline at the end of the document, if it's not empty.
fmt.Fprintln(w)
}
}
return ast.WalkContinue, nil
}
return ast.Walk(n, walk)
}
func (markdownToTextRenderer) AddOptions(...renderer.Option) {}