internal/task/announce.go

// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package task

import (
	"bytes"
	"context"
	"embed"
	"errors"
	"fmt"
	goversion "go/version"
	"io"
	"mime"
	"net/http"
	"net/mail"
	"net/url"
	"path"
	"regexp"
	"strings"
	"text/template"
	"time"

	"github.com/mailjet/mailjet-apiv3-go/v4"
	sendgrid "github.com/sendgrid/sendgrid-go"
	sendgridmail "github.com/sendgrid/sendgrid-go/helpers/mail"
	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/renderer"
	goldmarkhtml "github.com/yuin/goldmark/renderer/html"
	goldmarktext "github.com/yuin/goldmark/text"
	"golang.org/x/build/internal/gophers"
	"golang.org/x/build/internal/secret"
	"golang.org/x/build/internal/workflow"
	"golang.org/x/build/maintner/maintnerd/maintapi/version"
	"golang.org/x/build/relmeta"
	"golang.org/x/net/html"
	yaml "gopkg.in/yaml.v3"
)

type releaseAnnouncement struct {
	// Kind is the kind of release being announced.
	Kind ReleaseKind

	// Version is the Go version that has been released.
	//
	// The version string must use the same format as Go tags. For example:
	//   - "go1.21rc2" for a pre-release
	//   - "go1.21.0" for a major Go release
	//   - "go1.21.1" for a minor Go release
	Version string
	// SecondaryVersion is an older Go version that was also released.
	// This only applies to minor releases when two releases are made.
	// For example, "go1.20.9".
	SecondaryVersion string

	// Security is a list of descriptions, one for each distinct
	// security fix included in this release, in Markdown format.
	//
	// The empty list means there are no security fixes included.
	//
	// This field applies only to minor releases; it is an error
	// to try to use it another release type.
	Security []string

	// Names is an optional list of release coordinator names to
	// include in the sign-off message.
	Names []string
}

type releasePreAnnouncement struct {
	// Target is the planned date for the release.
	Target Date

	// Version is the Go version that will be released.
	//
	// The version string must use the same format as Go tags. For example, "go1.17.2".
	Version string
	// SecondaryVersion is an older Go version that will also be released.
	// This only applies when two releases are planned. For example, "go1.16.10".
	SecondaryVersion string

	// Security is the security content to be included in the
	// release pre-announcement. It should not reveal details
	// beyond what's allowed by the security policy.
	Security string

	// CVEs is the list of CVEs for PRIVATE track fixes to
	// be included in the release pre-announcement.
	CVEs []string

	// Names is an optional list of release coordinator names to
	// include in the sign-off message.
	Names []string
}

type golangOrgXPreAnnouncement struct {
	// Target is the planned date for the release.
	Target Date

	// Module is the module the security fixes are in.
	Module string

	// Package is the package the security fix is in.
	Packages []string

	// CVEs is the list of CVEs for PRIVATE track fixes to
	// be included in the release pre-announcement.
	CVEs []string

	// Names is an optional list of release coordinator names to
	// include in the sign-off message.
	Names []string
}

// A Date represents a single calendar day (year, month, day).
//
// This type does not include location information, and
// therefore does not describe a unique 24-hour timespan.
//
// TODO(go.dev/issue/19700): Start using time.Day or so when available.
type Date struct {
	Year  int        // Year (for example, 2009).
	Month time.Month // Month of the year (January = 1, ...).
	Day   int        // Day of the month, starting at 1.
}

func (d Date) String() string { return d.Format("2006-01-02") }
func (d Date) Format(layout string) string {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, time.UTC).Format(layout)
}

func (d Date) After(year int, month time.Month, day int) bool {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, time.UTC).
		After(time.Date(year, month, day, 0, 0, 0, 0, time.UTC))
}

// AnnounceMailTasks contains tasks related to the release (pre-)announcement email.
type AnnounceMailTasks struct {
	// SendMail sends an email with the given header and content
	// using an externally-provided implementation.
	//
	// Email delivery happens asynchronously, so SendMail returns a nil error
	// if the transmission was started successfully, but that error value
	// doesn't indicate anything about the status of the delivery. Even after
	// delivery, some mail may need to be reviewed and approved out of band
	// (e.g., a Google Groups mailing list configured with moderated posting).
	SendMail func(*workflow.TaskContext, MailHeader, MailContent) error

	// AnnounceMailHeader is the header to use for the release (pre-)announcement email.
	AnnounceMailHeader MailHeader

	// testHookNow is optionally set by tests to override time.Now.
	testHookNow func() time.Time
}

// SentMail represents an email that was sent to a mailing list.
//
// If the subject alone is guaranteed to uniquely identify the email,
// then keywords aren't needed. If the subject isn't unique enough,
// then keywords from the email body need to be specified as well.
type SentMail struct {
	Subject  string   // Subject of the email.
	Keywords []string // Keywords in the email. Optional, needed only if subject isn't unique.
}

// AnnounceRelease sends an email to Google Groups
// announcing that a Go release has been published.
func (t AnnounceMailTasks) AnnounceRelease(ctx *workflow.TaskContext, kind ReleaseKind, published []Published, security []string, users []string) (SentMail, error) {
	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
		return SentMail{}, fmt.Errorf("insufficient time for announce release task; a minimum of a minute left on context is required")
	}
	if len(published) < 1 || len(published) > 2 {
		return SentMail{}, fmt.Errorf("got %d published Go releases, AnnounceRelease supports only 1 or 2 at once", len(published))
	}
	names, err := coordinatorFirstNames(users)
	if err != nil {
		return SentMail{}, err
	}

	r := releaseAnnouncement{
		Kind:     kind,
		Version:  published[0].Version,
		Security: security,
		Names:    names,
	}
	if len(published) == 2 {
		r.SecondaryVersion = published[1].Version
	}

	return t.generateAndSendAnnouncementMail(ctx, r)
}

// PreAnnounceRelease sends an email pre-announcing a Go release
// containing PRIVATE track security fixes planned for the target date.
func (t AnnounceMailTasks) PreAnnounceRelease(ctx *workflow.TaskContext, versions []string, target Date, security string, cves []string, users []string) (SentMail, error) {
	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
		return SentMail{}, fmt.Errorf("insufficient time for pre-announce release task; a minimum of a minute left on context is required")
	}
	if len(versions) < 1 || len(versions) > 2 {
		return SentMail{}, fmt.Errorf("got %d planned Go releases, PreAnnounceRelease supports only 1 or 2 at once", len(versions))
	}
	now := time.Now().UTC()
	if t.testHookNow != nil {
		now = t.testHookNow()
	}
	if !target.After(now.Year(), now.Month(), now.Day()) { // A very simple check. Improve as needed.
		return SentMail{}, fmt.Errorf("target release date is not in the future")
	}
	if security == "" {
		return SentMail{}, fmt.Errorf("security content is not specified")
	}
	if len(cves) == 0 {
		return SentMail{}, errors.New("CVEs are not specified")
	}
	names, err := coordinatorFirstNames(users)
	if err != nil {
		return SentMail{}, err
	}

	r := releasePreAnnouncement{
		Target:   target,
		Version:  versions[0],
		Security: security,
		CVEs:     cves,
		Names:    names,
	}
	if len(versions) == 2 {
		r.SecondaryVersion = versions[1]
	}

	return t.generateAndSendAnnouncementMail(ctx, r)
}

func coordinatorFirstNames(users []string) ([]string, error) {
	return mapCoordinators(users, func(p *gophers.Person) string {
		name, _, _ := strings.Cut(p.Name, " ")
		return name
	})
}

func (t AnnounceMailTasks) PreAnnounceXFix(ctx *workflow.TaskContext, module string, pkgs []string, target Date, cves []string, users []string) (SentMail, error) {
	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
		return SentMail{}, fmt.Errorf("insufficient time for pre-announce release task; a minimum of a minute left on context is required")
	}
	now := time.Now().UTC()
	if t.testHookNow != nil {
		now = t.testHookNow()
	}
	for _, p := range pkgs {
		if !strings.HasPrefix(p, module) {
			return SentMail{}, fmt.Errorf("package %q is not inside module %q", p, module)
		}
	}
	if !target.After(now.Year(), now.Month(), now.Day()) { // A very simple check. Improve as needed.
		return SentMail{}, fmt.Errorf("target release date is not in the future")
	}
	if len(cves) == 0 {
		return SentMail{}, errors.New("CVEs are not specified")
	}
	names, err := coordinatorFirstNames(users)
	if err != nil {
		return SentMail{}, err
	}
	r := golangOrgXPreAnnouncement{
		Target:   target,
		Module:   module,
		Packages: pkgs,
		CVEs:     cves,
		Names:    names,
	}

	return t.generateAndSendAnnouncementMail(ctx, r)
}

func CoordinatorEmails(users []string) ([]string, error) { return coordinatorEmails(users) }
func coordinatorEmails(users []string) ([]string, error) {
	return mapCoordinators(users, func(p *gophers.Person) string {
		return p.Gerrit
	})
}

// mapCoordinators maps users to f(user). It's a precondition that users have
// already been validated with [CheckCoordinators] in advance.
//
// [SecurityReviewersParameter] is an exception; it passes @google.com emails
// through as-is without calling f.
func mapCoordinators(users []string, f func(*gophers.Person) string) ([]string, error) {
	var outs []string
	for _, user := range users {
		// To avoid more complexity and changing
		// a lot of call-sites, we carve out an
		// exception for suffixed @google.com
		// emails used in security contexts.
		if strings.HasSuffix(user, "@google.com") {
			outs = append(outs, user)
			continue
		}
		person, err := lookupCoordinator(user)
		if err != nil {
			return nil, err
		}
		outs = append(outs, f(person))
	}
	return outs, nil
}

// CheckCoordinators checks that all users are known
// and have required information (name, Gerrit email, GitHub user name).
func CheckCoordinators(users []string) error {
	var report strings.Builder
	for _, user := range users {
		if _, err := lookupCoordinator(user); err != nil {
			fmt.Fprintln(&report, err)
		}
	}
	if report.Len() != 0 {
		return errors.New(report.String())
	}
	return nil
}

func lookupCoordinator(user string) (*gophers.Person, error) {
	person := gophers.GetPerson(user + "@golang.org")
	if person == nil {
		person = gophers.GetPerson(user + "@google.com")
	}
	if person == nil {
		return nil, fmt.Errorf("unknown username %q: no @golang or @google account", user)
	}
	if person.Name == "" {
		return nil, fmt.Errorf("release coordinator %q is missing a name", person.Gerrit)
	}
	if person.GitHub == "" {
		return nil, fmt.Errorf("release coordinator %q is missing github user name", person.Gerrit)
	}
	return person, nil
}

// A MailHeader is an email header.
type MailHeader struct {
	From mail.Address // An RFC 5322 address. For example, "Barry Gibbs <bg@example.com>".
	To   mail.Address
	BCC  []mail.Address
}

// A MailContent holds the content of an email.
type MailContent struct {
	Subject  string
	BodyHTML string
	BodyText string
}

// announcementMail generates the (pre-)announcement email using data,
// which must be one of these types:
//   - releaseAnnouncement for a release announcement
//   - releasePreAnnouncement for a release pre-announcement
//   - golangOrgXPreAnnouncement for a golang.org/x (or other Go module) security
//     release pre-announcement
//   - goplsPrereleaseAnnouncement for a gopls pre-announcement
//   - vscodeGo…Announcement for vscode-go announcements
//
// It also returns keywords intended to help uniquely identify the mail
// after it is sent to a mailing list. For example, if the keywords are
// "CVE-2025-1234" and "CVE-2025-1235", then it's expected that a query
// like https://groups.google.com/g/golang-announce/search?q=CVE-2025-1234%20CVE-2025-1235
// should find the email. If the email subject alone will uniquely find
// the email, there's no need to specify any keywords.
func announcementMail(data any) (_ MailContent, sentMailKeywords []string, _ error) {
	// Select the appropriate template name.
	var name string
	switch r := data.(type) {
	case releaseAnnouncement:
		switch r.Kind {
		case KindBeta:
			name = "announce-beta.md"
		case KindRC:
			name = "announce-rc.md"
		case KindMajor:
			name = "announce-major.md"
		case KindMinor:
			name = "announce-minor.md"
		default:
			return MailContent{}, nil, fmt.Errorf("unknown release kind: %v", r.Kind)
		}
		if len(r.Security) > 0 && (name != "announce-minor.md" && name != "announce-rc.md") {
			// The Security field isn't supported in templates other than minor and RC,
			// so report an error instead of silently dropping it.
			return MailContent{}, nil, fmt.Errorf("email template %q doesn't support the Security field; this field can only be used in minor releases", name)
		} else if r.SecondaryVersion != "" && name != "announce-minor.md" {
			return MailContent{}, nil, fmt.Errorf("email template %q doesn't support more than one release; the SecondaryVersion field can only be used in minor releases", name)
		}
	case releasePreAnnouncement:
		name = "pre-announce-minor.md"
	case golangOrgXPreAnnouncement:
		name = "pre-announce-x.md"
		sentMailKeywords = r.CVEs // Rely on CVEs in body to differentiate "[security] {{.Module}} fix pre-announcement" emails.
	case goplsReleaseAnnouncement:
		name = "gopls-announce.md"
	case goplsPrereleaseAnnouncement:
		name = "gopls-pre-announce.md"
	case vscodeGoReleaseAnnouncement:
		name = "vscode-go-announce.md"
	case vscodeGoPrereleaseAnnouncement:
		name = "vscode-go-pre-announce.md"
	case vscodeGoInsiderAnnouncement:
		name = "vscode-go-insider-announce.md"
	default:
		return MailContent{}, nil, fmt.Errorf("unknown template data type %T", data)
	}

	// Render the (pre-)announcement email template.
	//
	// It'll produce a valid message with a MIME header and a body, so parse it as such.
	var buf bytes.Buffer
	if err := announceTmpl.ExecuteTemplate(&buf, name, data); err != nil {
		return MailContent{}, nil, err
	}
	m, err := mail.ReadMessage(&buf)
	if err != nil {
		return MailContent{}, nil, fmt.Errorf(`email template must be formatted like a mail message, but reading it failed: %v`, err)
	}

	// Get the email subject (it's a plain string, no further processing needed).
	if _, ok := m.Header["Subject"]; !ok {
		return MailContent{}, nil, fmt.Errorf(`email template must have a "Subject" key in its MIME header, but it's not found`)
	} else if n := len(m.Header["Subject"]); n != 1 {
		return MailContent{}, nil, fmt.Errorf(`email template must have a single "Subject" value in its MIME header, but have %d values`, n)
	}
	subject := m.Header.Get("Subject")

	// Render the email body, in Markdown format at this point, to HTML and plain text.
	html, text, err := renderMarkdown(m.Body)
	if err != nil {
		return MailContent{}, nil, err
	}

	return MailContent{subject, html, text}, sentMailKeywords, nil
}

// announceTmpl holds templates for Go release announcement emails.
//
// Each email template starts with a MIME-style header with a Subject key,
// and the rest of it is Markdown for the email body.
var announceTmpl = template.Must(template.New("").Funcs(template.FuncMap{
	"join": func(s []string) string {
		switch len(s) {
		case 0:
			return ""
		case 1:
			return s[0]
		case 2:
			return s[0] + " and " + s[1]
		default:
			return strings.Join(s[:len(s)-1], ", ") + ", and " + s[len(s)-1]
		}
	},
	"indent": func(s string) string { return "\t" + strings.ReplaceAll(s, "\n", "\n\t") },

	// subjectPrefix returns the email subject prefix for release r, if any.
	"subjectPrefix": func(r releaseAnnouncement) string {
		switch {
		case len(r.Security) > 0:
			// Include a security prefix as documented at https://go.dev/doc/security/policy#receiving-security-updates:
			//
			//	> The best way to receive security announcements is to subscribe to the golang-announce mailing list.
			//	> Any messages pertaining to a security issue will be prefixed with [security].
			//
			return "[security]"
		default:
			return ""
		}
	},

	// shortcommit is used to shorten git commit hashes.
	"shortcommit": func(v string) string {
		if len(v) > 7 {
			return v[:7]
		}
		return v
	},

	// short and helpers below manipulate valid Go version strings
	// for the current needs of the announcement templates.
	"short": func(v string) string { return strings.TrimPrefix(v, "go") },
	// major extracts the major prefix of a valid Go version.
	// For example, major("go1.18.4") == "1.18".
	"major": func(v string) (string, error) {
		x, ok := version.Go1PointX(v)
		if !ok {
			return "", fmt.Errorf("internal error: version.Go1PointX(%q) is not ok", v)
		}
		return fmt.Sprintf("1.%d", x), nil
	},
	// atLeast reports whether v1 >= v2.
	"atLeast": func(v1, v2 string) bool {
		return goversion.Compare(v1, v2) >= 0
	},
	// build extracts the pre-release build number of a valid Go version.
	// For example, build("go1.19beta2") == "2".
	"build": func(v string) (string, error) {
		if _, number, ok := strings.Cut(v, "beta"); ok {
			return number, nil
		} else if _, number, ok := strings.Cut(v, "rc"); ok {
			return number, nil
		}
		return "", fmt.Errorf("internal error: unhandled pre-release Go version %q", v)
	},
}).ParseFS(tmplDir,
	"template/announce-*.md",
	"template/pre-announce-minor.md",
	"template/pre-announce-x.md",
	// Gopls release announcements.
	"template/gopls-announce.md",
	"template/gopls-pre-announce.md",
	// VSCode Go release announcements.
	"template/vscode-go-pre-announce.md",
	"template/vscode-go-insider-announce.md",
	"template/vscode-go-announce.md",
))

//go:embed template
var tmplDir embed.FS

// ReleaseCoordinatorAsTheMailSender implements the manual strategy
// of asking the release coordinator to handle sending the email.
// It's used as a fallback when a more automated solution can't be used.
type ReleaseCoordinatorAsTheMailSender struct {
	ApproveAction func(*workflow.TaskContext) error // non-nil
}

// SendMail prompts the release coordinator to send an email
// and awaits them to approve the task after they've done so.
func (r ReleaseCoordinatorAsTheMailSender) SendMail(ctx *workflow.TaskContext, h MailHeader, m MailContent) error {
	addressListString := func(as []mail.Address) string {
		if len(as) == 0 {
			return "(empty address list)"
		}
		var b strings.Builder
		for i, a := range as {
			if i != 0 {
				b.WriteString(", ")
			}
			b.WriteString(a.String())
		}
		return b.String()
	}

	ctx.Printf(`ReleaseCoordinatorAsTheMailSender.SendMail running...

* * * * *

Please send the following email using your @golang.org or @google.com email,
then approve this task as completed.

To: %s
BCC: %s
Subject: %s
Body Text:

%s`, &h.To, addressListString(h.BCC), m.Subject, m.BodyText)
	return r.ApproveAction(ctx)
}

type realSendGridMailClient struct {
	sg *sendgrid.Client
}

// NewSendGridMailClient creates a SendGrid mail client authenticated with the given API key.
func NewSendGridMailClient(sendgridAPIKey string) realSendGridMailClient {
	return realSendGridMailClient{sg: sendgrid.NewSendClient(sendgridAPIKey)}
}

// SendMail sends an email by making an authenticated request to the SendGrid API.
func (c realSendGridMailClient) SendMail(_ *workflow.TaskContext, h MailHeader, m MailContent) error {
	from, to := sendgridmail.Email(h.From), sendgridmail.Email(h.To)
	req := sendgridmail.NewSingleEmail(&from, m.Subject, &to, m.BodyText, m.BodyHTML)
	if len(req.Personalizations) != 1 {
		return fmt.Errorf("internal error: len(req.Personalizations) is %d, want 1", len(req.Personalizations))
	}
	for _, bcc := range h.BCC {
		bcc := sendgridmail.Email(bcc)
		req.Personalizations[0].AddBCCs(&bcc)
	}
	no := false
	req.TrackingSettings = &sendgridmail.TrackingSettings{
		ClickTracking:        &sendgridmail.ClickTrackingSetting{Enable: &no},
		OpenTracking:         &sendgridmail.OpenTrackingSetting{Enable: &no},
		SubscriptionTracking: &sendgridmail.SubscriptionTrackingSetting{Enable: &no},
	}
	resp, err := c.sg.Send(req)
	if err != nil {
		return err
	} else if resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("unexpected status %d %s, want 202 Accepted; body = %s", resp.StatusCode, http.StatusText(resp.StatusCode), resp.Body)
	}
	return nil
}

type realMailjetMailClient struct {
	cl *mailjet.Client
}

// NewMailjetMailClient creates a Mailjet mail client authenticated with the given API key.
func NewMailjetMailClient(key secret.MailjetCredentials) realMailjetMailClient {
	return realMailjetMailClient{cl: mailjet.NewMailjetClient(key.APIKeyPublic, key.APIKeyPrivate)}
}

// SendMail sends an email by making an authenticated request to the Mailjet API.
func (c realMailjetMailClient) SendMail(_ *workflow.TaskContext, h MailHeader, m MailContent) error {
	recipient := func(a mail.Address) *mailjet.RecipientV31 {
		return &mailjet.RecipientV31{Email: a.Address, Name: a.Name}
	}
	recipients := func(as []mail.Address) *mailjet.RecipientsV31 {
		var r mailjet.RecipientsV31
		for _, a := range as {
			r = append(r, *recipient(a))
		}
		return &r
	}
	resp, err := c.cl.SendMailV31(&mailjet.MessagesV31{Info: []mailjet.InfoMessagesV31{{
		From:     recipient(h.From),
		To:       recipients([]mail.Address{h.To}),
		Bcc:      recipients(h.BCC),
		Subject:  m.Subject,
		TextPart: m.BodyText,
		HTMLPart: m.BodyHTML,
	}}})
	if err != nil {
		return err
	} else if len(resp.ResultsV31) == 0 {
		return fmt.Errorf("no results; resp = %#v", resp)
	}
	for i, r := range resp.ResultsV31 {
		if r.Status != "success" {
			return fmt.Errorf("unexpected %dth result status %q, want 'success'; resp = %#v", i+1, r.Status, resp)
		}
	}
	return nil
}

// AwaitAnnounceMail waits for an announcement email with the specified subject
// to show up on Google Groups, and returns its canonical URL.
func (t AnnounceMailTasks) AwaitAnnounceMail(ctx *workflow.TaskContext, m SentMail) (announcementURL string, _ error) {
	// Find the URL for the announcement while giving the email a chance to be received and moderated.
	check := func() (string, bool, error) {
		// See if our email is available by now.
		threadURL, err := findGoogleGroupsThread(ctx, m.Subject, m.Keywords)
		if err != nil {
			ctx.Printf("findGoogleGroupsThread: %v", err)
			return "", false, nil
		}
		return threadURL, threadURL != "", nil
	}
	return AwaitCondition(ctx, 10*time.Second, check)
}

// findGoogleGroupsThread fetches a search query of threads from the golang-announce
// Google Groups mailing list, and looks for a thread with the matching subject line.
// The search query includes the email subject and/or any keywords that are provided.
// It returns its URL if found or the empty string if not found.
//
// findGoogleGroupsThread returns an error
// satisfying [errors.As](err, [fetchError]) with PossiblyRetryable set to true
// when it has signal that repeating the same call after some time may succeed.
func findGoogleGroupsThread(ctx *workflow.TaskContext, subject string, keywords []string) (threadURL string, _ error) {
	searchQuery := fmt.Sprintf(`subject:"%s"`, subject)
	if len(keywords) > 0 {
		searchQuery += " " + strings.Join(keywords, " ")
	}
	u := "https://groups.google.com/g/golang-announce/search?" + url.Values{"q": []string{searchQuery}}.Encode()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
	if err != nil {
		return "", err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", fetchError{Err: err, PossiblyRetryable: true}
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		possiblyRetryable := resp.StatusCode/100 == 5 // Consider a 5xx server response to possibly succeed later.
		body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
		return "", fetchError{fmt.Errorf("did not get acceptable status code: %v body: %q", resp.Status, body), possiblyRetryable}
	}
	if ct, want := resp.Header.Get("Content-Type"), "text/html; charset=utf-8"; ct != want {
		ctx.Printf("findGoogleGroupsThread: got response with non-'text/html; charset=utf-8' Content-Type header %q\n", ct)
		if mediaType, _, err := mime.ParseMediaType(ct); err != nil {
			return "", fmt.Errorf("bad Content-Type header %q: %v", ct, err)
		} else if mediaType != "text/html" {
			return "", fmt.Errorf("got media type %q, want %q", mediaType, "text/html")
		}
	}
	doc, err := html.Parse(retryableReader{io.LimitReader(resp.Body, 5<<20)})
	if err != nil {
		return "", err
	}
	var baseHref string
	var linkHref string
	var found bool
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "base" {
			baseHref = href(n)
		} else if n.Type == html.ElementNode && n.Data == "a" {
			linkHref = href(n)
		} else if n.Type == html.TextNode && n.Data == subject {
			// Found our link. Break out.
			found = true
			return
		}
		for c := n.FirstChild; c != nil && !found; c = c.NextSibling {
			f(c)
		}
	}
	f(doc)
	if !found {
		// Thread not found within the search query.
		return "", nil
	}
	base, err := url.Parse(baseHref)
	if err != nil {
		return "", err
	}
	link, err := url.Parse(linkHref)
	if err != nil {
		return "", err
	}
	threadURL = base.ResolveReference(link).String()
	const announcementPrefix = "https://groups.google.com/g/golang-announce/c/"
	if !strings.HasPrefix(threadURL, announcementPrefix) {
		return "", fmt.Errorf("found URL %q, but it doesn't have the expected prefix %q", threadURL, announcementPrefix)
	}
	return threadURL, nil
}

func href(n *html.Node) string {
	for _, a := range n.Attr {
		if a.Key == "href" {
			return a.Val
		}
	}
	return ""
}

// retryableReader annotates errors from
// RetryableReader as possibly retryable.
type retryableReader struct{ RetryableReader io.Reader }

func (r retryableReader) Read(p []byte) (n int, err error) {
	n, err = r.RetryableReader.Read(p)
	if err != nil && err != io.EOF {
		err = fetchError{Err: err, PossiblyRetryable: true}
	}
	return n, err
}

// fetchError records an error during a fetch operation over an unreliable network.
type fetchError struct {
	Err error // Non-nil.

	// PossiblyRetryable indicates whether Err is believed to be possibly caused by a
	// non-terminal network error, such that the caller can expect it may not happen
	// again if it simply tries the same fetch operation again after waiting a bit.
	PossiblyRetryable bool
}

func (e fetchError) Error() string { return e.Err.Error() }
func (e fetchError) Unwrap() error { return e.Err }

// renderMarkdown parses Markdown source
// and renders it to HTML and plain text.
//
// The Markdown specification and its various extensions are vast.
// Here we support a small, simple set of Markdown features
// that satisfies the needs of the announcement mail tasks.
func renderMarkdown(r io.Reader) (html, text string, _ error) {
	source, err := io.ReadAll(r)
	if err != nil {
		return "", "", err
	}
	// Configure a Markdown parser and HTML renderer fairly closely
	// to how it's done in x/website, just without raw HTML support
	// and other extensions we don't need for announcement emails.
	md := goldmark.New(
		goldmark.WithRendererOptions(goldmarkhtml.WithHardWraps()),
		goldmark.WithExtensions(
			extension.NewLinkify(extension.WithLinkifyAllowedProtocols([][]byte{[]byte("https:")})),
		),
	)
	doc := md.Parser().Parse(goldmarktext.NewReader(source))
	var htmlBuf, textBuf bytes.Buffer
	err = md.Renderer().Render(&htmlBuf, source, doc)
	if err != nil {
		return "", "", err
	}
	err = (markdownToTextRenderer{}).Render(&textBuf, source, doc)
	if err != nil {
		return "", "", err
	}
	return htmlBuf.String(), textBuf.String(), nil
}

// markdownToTextRenderer is a simple goldmark/renderer.Renderer implementation
// that renders Markdown to plain text for the needs of announcement mail tasks.
//
// It produces an output suitable for email clients that cannot (or choose not to)
// display the HTML version of the email. (It also helps a bit with the readability
// of our test data, since without a browser plain text is more readable than HTML.)
//
// The output is mostly plain text that doesn't preserve Markdown syntax (for example,
// `code` is rendered without backticks, code blocks aren't indented, and so on),
// though there is very lightweight formatting applied (links are written as "text <URL>").
//
// We can in theory choose to delete this renderer at any time if its maintenance costs
// start to outweigh its benefits, since Markdown by definition is designed to be human
// readable and can be used as plain text without any processing.
type markdownToTextRenderer struct{}

func (markdownToTextRenderer) Render(w io.Writer, source []byte, n ast.Node) error {
	const debug = false
	if debug {
		n.Dump(source, 0)
	}

	var markers []byte // Stack of list markers, from outermost to innermost.

	walk := func(n ast.Node, entering bool) (ast.WalkStatus, error) {
		if entering {
			if n.Type() == ast.TypeBlock && n.PreviousSibling() != nil {
				// Print a blank line between block nodes.
				switch n.PreviousSibling().Kind() {
				default:
					fmt.Fprint(w, "\n\n")
				case ast.KindCodeBlock, ast.KindFencedCodeBlock:
					// A code block always ends with a newline, so only need one more.
					fmt.Fprintln(w)
				}

				// If we're in a list, indent accordingly.
				if n.Kind() != ast.KindListItem {
					fmt.Fprint(w, strings.Repeat("\t", len(markers)))
				}
			}

			switch n := n.(type) {
			case *ast.Text:
				fmt.Fprintf(w, "%s", n.Text(source))

				// Print a line break.
				if n.SoftLineBreak() || n.HardLineBreak() {
					fmt.Fprintln(w)

					// If we're in a list, indent accordingly.
					fmt.Fprint(w, strings.Repeat("\t", len(markers)))
				}
			case *ast.CodeBlock, *ast.FencedCodeBlock:
				// Code blocks are printed as is in plain text.
				for i := 0; i < n.Lines().Len(); i++ {
					s := n.Lines().At(i)
					if i != 0 {
						// If we're in a list, indent inner lines accordingly.
						fmt.Fprint(w, strings.Repeat("\t", len(markers)))
					}
					fmt.Fprint(w, string(source[s.Start:s.Stop]))
				}
			case *ast.AutoLink:
				// Auto-links are printed as is in plain text.
				//
				// For example, the Markdown "https://go.dev/issue/123"
				// is rendered as plain text "https://go.dev/issue/123".
				fmt.Fprint(w, string(n.Label(source)))
			case *ast.List:
				// Push list marker on the stack.
				markers = append(markers, n.Marker)
			case *ast.ListItem:
				fmt.Fprintf(w, "%s%c\t", strings.Repeat("\t", len(markers)-1), markers[len(markers)-1])
			}
		} else {
			switch n := n.(type) {
			case *ast.Link:
				// Append the link's URL after its text.
				//
				// For example, the Markdown "[CL 456](https://go.dev/cl/456)"
				// is rendered as plain text "CL 456 <https://go.dev/cl/456>".
				fmt.Fprintf(w, " <%s>", n.Destination)
			case *ast.List:
				// Pop list marker off the stack.
				markers = markers[:len(markers)-1]
			}

			if n.Type() == ast.TypeDocument && n.ChildCount() != 0 {
				// Print a newline at the end of the document, if it's not empty.
				fmt.Fprintln(w)
			}
		}
		return ast.WalkContinue, nil
	}
	return ast.Walk(n, walk)
}
func (markdownToTextRenderer) AddOptions(...renderer.Option) {}

func (t AnnounceMailTasks) generateAndSendAnnouncementMail(ctx *workflow.TaskContext, data any) (SentMail, error) {
	// Generate the announcement email.
	m, sentMailKeywords, err := announcementMail(data)
	if err != nil {
		return SentMail{}, err
	}
	ctx.Printf("announcement subject: %s\n\n", m.Subject)
	ctx.Printf("announcement body HTML:\n%s\n", m.BodyHTML)
	ctx.Printf("announcement body text:\n%s", m.BodyText)

	// Before sending, check to see if this announcement already exists.
	if threadURL, err := findGoogleGroupsThread(ctx, m.Subject, sentMailKeywords); err != nil {
		return SentMail{}, fmt.Errorf("stopping early due to error checking for an existing Google Groups thread: %w", err)
	} else if threadURL != "" {
		// This should never happen since this task runs once per release.
		// It can happen under unusual circumstances, for example if the task crashes after
		// mailing but before completion, or if parts of the release workflow are restarted,
		// or if a human mails the announcement email manually out of band.
		//
		// So if we see that the email exists, consider it as "task completed successfully"
		// and pretend we were the ones that sent it, so the high level workflow can keep going.
		ctx.Printf("a Google Groups thread with matching subject %q and keywords %q already exists at %q, so we'll consider that as it being sent successfully", m.Subject, sentMailKeywords, threadURL)
		return SentMail{m.Subject, sentMailKeywords}, nil
	}

	// Send the announcement email to the destination mailing lists.
	if t.SendMail == nil {
		return SentMail{"[dry-run] " + m.Subject, sentMailKeywords}, nil
	}
	ctx.DisableRetries()
	err = t.SendMail(ctx, t.AnnounceMailHeader, m)
	if err != nil {
		return SentMail{}, err
	}

	return SentMail{m.Subject, sentMailKeywords}, nil
}

// SecurityCommunicationTasks contains communication tasks
// relevant to Go releases containing security fixes.
type SecurityCommunicationTasks struct {
	PrivateGerrit GerritClient
}

// GetSecuritySummary returns a short security content summary following the template:
//
//	Includes {N} security fixes to {the standard library|the toolchain|the standard library and the toolchain}.
//
// It returns an empty string when there are 0 security fixes.
func (t SecurityCommunicationTasks) GetSecuritySummary(ctx *workflow.TaskContext, milestoneNum string) (string, error) {
	rm, err := FetchReleaseMilestone(ctx, t.PrivateGerrit, milestoneNum)
	if err != nil {
		return "", err
	}
	ctx.Printf("fetched security release milestone containing %d security fixes", len(rm.Patches))
	if len(rm.Patches) == 0 {
		// No security fixes.
		return "", nil
	}
	var std, toolchain bool
	for _, p := range rm.Patches {
		if p.Toolchain {
			toolchain = true
		} else {
			std = true
		}
	}
	var components string
	switch {
	case std && !toolchain:
		components = "the standard library"
	case !std && toolchain:
		components = "the toolchain"
	case std && toolchain:
		components = "the standard library and the toolchain"
	}
	return fmt.Sprintf("Includes %d security fixes to %s.", len(rm.Patches), components), nil
}

// GetSecurityReleaseNotes fetches a list of descriptions, one for each distinct security fix
// included in the release identified by milestoneNum, in Markdown format.
func (t SecurityCommunicationTasks) GetSecurityReleaseNotes(ctx *workflow.TaskContext, milestoneNum string) (releaseNotes []string, _ error) {
	rm, err := FetchReleaseMilestone(ctx, t.PrivateGerrit, milestoneNum)
	if err != nil {
		return nil, err
	}
	ctx.Printf("fetched security release milestone containing %d security fixes", len(rm.Patches))
	for _, p := range rm.Patches {
		releaseNotes = append(releaseNotes, p.ReleaseNote)
	}
	return releaseNotes, nil
}

var (
	SecurityMilestoneParameter = workflow.ParamDef[string]{
		Name:      "Release Milestone",
		ParamType: workflow.BasicString,
		Doc: `Release Milestone owns the canonical source for all security patches.

You can check with the security release coordinator to confirm this input.`,
		Example: "123456",
		Check: func(num string) error {
			if !numOnlyRE.MatchString(num) {
				return errors.New("milestone number must contain only numbers")
			}
			return nil
		},
	}
	OptionalSecurityMilestoneParameter = workflow.ParamDef[string]{
		Name:      "Release Milestone (optional)",
		ParamType: workflow.BasicString,
		Doc: `Release Milestone owns the canonical source for all security patches.

Empty means no security patches are intended for this release.
You can check with the security release coordinator to confirm this input.`,
		Example: "123456",
		Check: func(num string) error {
			if num == "" {
				return nil
			}
			if !numOnlyRE.MatchString(num) {
				return errors.New("milestone number must contain only numbers")
			}
			return nil
		},
	}
	numOnlyRE = regexp.MustCompile(`^\d+$`)

	SecurityReviewersParameter = workflow.ParamDef[[]string]{
		Name:      "Security Reviewer Emails",
		ParamType: workflow.SliceShort,
		Doc:       `Valid @google.com email strings for privileged reviewers involved in the release.`,
		Example:   "nealpatel@google.com",
		Check: func(users []string) error {
			for _, u := range users {
				if !strings.HasSuffix(u, "@google.com") {
					return fmt.Errorf("security reviewer %q must be an @google.com address", u)
				}
			}
			return nil
		},
	}
)

func FetchReleaseMilestone(ctx context.Context, private GerritClient, milestoneNum string) (relmeta.ReleaseMilestone, error) {
	const project = "security-metadata"
	head, err := private.ReadBranchHead(ctx, project, "main")
	if err != nil {
		return relmeta.ReleaseMilestone{}, err
	}
	b, err := private.ReadFile(ctx, project, head, path.Join("data", "milestones", milestoneNum+".yaml"))
	if err != nil {
		return relmeta.ReleaseMilestone{}, err
	}
	var rm relmeta.ReleaseMilestone
	if err := yaml.Unmarshal(b, &rm); err != nil {
		return relmeta.ReleaseMilestone{}, fmt.Errorf("cannot YAML unmarshal the milestone: %v", err)
	}
	return rm, nil
}