acme: add Pebble integration testing

This commit adds integration test coverage for a complete TLS-ALPN-01
and HTTP-01 based issuance flow.

For each tested challenge type we:

* Spin up a pebble/pebble-challtestsrv environment
* Spin up a small challenge response server
* Create an ACME account
* Create an order for multiple DNS type identifiers
* Provision challenge responses based on the challenge type under test
* Wait for the order to become ready for issuance
* Finalize the order, issuing a certificate
* Check the newly issued certificate chain validates with the Pebble
  trust anchor, and that the certificate is valid for each of the names
  from our initial order

These tests are skipped in short mode (Pebble has variable delays for
validation requests).

The Pebble source is fetched through the Go module proxy (unless
a local directory is specified to aid development), similar to how the
stdlib crypto packages fetch BoGo tooling.

More test coverage for various other parts of the protocol (key
rollover, account/authz deactivation, revocation, etc) can be added as
follow-up work now that the groundwork for integration testing is laid.

Fixes golang/go#73914

Cq-Include-Trybots: luci.golang.try:x_crypto-gotip-linux-amd64-longtest
Change-Id: I4e79f4858f31ef290a0c91d345e15fbdc510e9ab
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/677575
Reviewed-by: Roland Shoemaker <roland@golang.org>
Auto-Submit: Daniel McCarney <daniel@binaryparadox.net>
Reviewed-by: Ian Stapleton Cordasco <graffatcolmingov@gmail.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/acme/pebble_test.go b/acme/pebble_test.go
new file mode 100644
index 0000000..625e20b
--- /dev/null
+++ b/acme/pebble_test.go
@@ -0,0 +1,793 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package acme_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"golang.org/x/crypto/acme"
+)
+
+const (
+	// pebbleModVersion is the module version used for Pebble and Pebble's
+	// challenge test server. It is ignored if `-pebble-local-dir` is provided.
+	pebbleModVersion = "v2.7.0"
+	// startingPort is the first port number used for binding interface
+	// addresses. Each call to takeNextPort() will increment a port number
+	// starting at this value.
+	startingPort = 5555
+)
+
+var (
+	pebbleLocalDir = flag.String(
+		"pebble-local-dir",
+		"",
+		"Local Pebble to use, instead of fetching from source",
+	)
+	nextPort atomic.Uint32
+)
+
+func init() {
+	nextPort.Store(startingPort)
+}
+
+func TestWithPebble(t *testing.T) {
+	// We want to use process groups w/ syscall.Kill, and the acme package
+	// is very platform-agnostic, so skip on non-Linux.
+	if runtime.GOOS != "linux" {
+		t.Skip("skipping pebble tests on non-linux OS")
+	}
+
+	if testing.Short() {
+		t.Skip("skipping pebble tests in short mode")
+	}
+
+	tests := []struct {
+		name     string
+		challSrv func(*environment) (challengeServer, string)
+	}{
+		{
+			name: "TLSALPN01-Issuance",
+			challSrv: func(env *environment) (challengeServer, string) {
+				bindAddr := fmt.Sprintf(":%d", env.config.TLSPort)
+				return newChallTLSServer(bindAddr), bindAddr
+			},
+		},
+
+		{
+			name: "HTTP01-Issuance",
+			challSrv: func(env *environment) (challengeServer, string) {
+				bindAddr := fmt.Sprintf(":%d", env.config.HTTPPort)
+				return newChallHTTPServer(bindAddr), bindAddr
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			env := startPebbleEnvironment(t, nil)
+			challSrv, challSrvAddr := tt.challSrv(&env)
+			challSrv.Run()
+
+			t.Cleanup(func() {
+				challSrv.Shutdown()
+			})
+
+			waitForServer(t, challSrvAddr)
+			testIssuance(t, &env, challSrv)
+		})
+	}
+}
+
+// challengeServer abstracts over the details of running a challenge response
+// server for some supported acme.Challenge type. Responses are provisioned
+// during the test issuance process to be presented to the ACME server's
+// validation authority.
+type challengeServer interface {
+	Run()
+	Shutdown() error
+	Supported(chal *acme.Challenge) bool
+	Provision(client *acme.Client, ident acme.AuthzID, chal *acme.Challenge) error
+}
+
+// challTLSServer is a simple challenge response server that listens for TLS
+// connections on a specific port and if they are TLS-ALPN-01 challenge
+// requests, completes the handshake using the configured challenge response
+// certificate for the SNI value provided.
+type challTLSServer struct {
+	*http.Server
+	// mu protects challCerts.
+	mu sync.RWMutex
+	// challCerts is a map from SNI domain name to challenge response certificate.
+	challCerts map[string]*tls.Certificate
+}
+
+// https://datatracker.ietf.org/doc/html/rfc8737#section-4
+const acmeTLSAlpnProtocol = "acme-tls/1"
+
+func newChallTLSServer(address string) *challTLSServer {
+	challServer := &challTLSServer{Server: &http.Server{
+		Addr:         address,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 5 * time.Second,
+	}, challCerts: make(map[string]*tls.Certificate)}
+
+	// Configure the server to support the TLS-ALPN-01 challenge protocol
+	// and to use a callback for selecting the handshake certificate.
+	challServer.Server.TLSConfig = &tls.Config{
+		NextProtos:     []string{acmeTLSAlpnProtocol},
+		GetCertificate: challServer.getCertificate,
+	}
+
+	return challServer
+}
+
+func (c *challTLSServer) Shutdown() error {
+	log.Printf("challTLSServer: shutting down")
+	ctx, cancel := context.WithTimeout(context.Background(), 10)
+	defer cancel()
+	return c.Server.Shutdown(ctx)
+}
+
+func (c *challTLSServer) Run() {
+	go func() {
+		// Note: certFile and keyFile are empty because our config uses a
+		// GetCertificate callback.
+		if err := c.Server.ListenAndServeTLS("", ""); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				log.Printf("challTLSServer error: %v", err)
+			}
+		}
+	}()
+}
+
+func (c *challTLSServer) Supported(chal *acme.Challenge) bool {
+	return chal.Type == "tls-alpn-01"
+}
+
+func (c *challTLSServer) Provision(client *acme.Client, ident acme.AuthzID, chal *acme.Challenge) error {
+	respCert, err := client.TLSALPN01ChallengeCert(chal.Token, ident.Value)
+	if err != nil {
+		return fmt.Errorf("challTLSServer: failed to generate challlenge response cert for %s: %w",
+			ident.Value, err)
+	}
+
+	log.Printf("challTLSServer: setting challenge response certificate for %s", ident.Value)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.challCerts[ident.Value] = &respCert
+
+	return nil
+}
+
+func (c *challTLSServer) getCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	// Verify the request looks like a TLS-ALPN-01 challenge request.
+	if len(clientHello.SupportedProtos) != 1 || clientHello.SupportedProtos[0] != acmeTLSAlpnProtocol {
+		return nil, fmt.Errorf(
+			"challTLSServer: non-TLS-ALPN-01 challenge request received with SupportedProtos: %s",
+			clientHello.SupportedProtos)
+	}
+
+	serverName := clientHello.ServerName
+
+	// TLS-ALPN-01 challenge requests for IP addresses are encoded in the SNI
+	// using the reverse-DNS notation. See RFC 8738 Section 6:
+	//   https://www.rfc-editor.org/rfc/rfc8738.html#section-6
+	if strings.HasSuffix(serverName, ".in-addr.arpa") {
+		serverName = strings.TrimSuffix(serverName, ".in-addr.arpa")
+		parts := strings.Split(serverName, ".")
+		for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
+			parts[i], parts[j] = parts[j], parts[i]
+		}
+		serverName = strings.Join(parts, ".")
+	}
+
+	log.Printf("challTLSServer: selecting certificate for request from %s for %s",
+		clientHello.Conn.RemoteAddr(), serverName)
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	cert := c.challCerts[serverName]
+	if cert == nil {
+		return nil, fmt.Errorf("challTLSServer: no challenge response certificate configured for %s", serverName)
+	}
+
+	return cert, nil
+}
+
+// challHTTPServer is a simple challenge response server that listens for HTTP
+// connections on a specific port and if they are HTTP-01 challenge requests,
+// serves the challenge response key authorization.
+type challHTTPServer struct {
+	*http.Server
+	// mu protects challMap
+	mu sync.RWMutex
+	// challMap is a mapping from request path to response body.
+	challMap map[string]string
+}
+
+func newChallHTTPServer(address string) *challHTTPServer {
+	challServer := &challHTTPServer{
+		Server: &http.Server{
+			Addr:         address,
+			ReadTimeout:  5 * time.Second,
+			WriteTimeout: 5 * time.Second,
+		},
+		challMap: make(map[string]string),
+	}
+
+	challServer.Server.Handler = challServer
+
+	return challServer
+}
+
+func (c *challHTTPServer) Supported(chal *acme.Challenge) bool {
+	return chal.Type == "http-01"
+}
+
+func (c *challHTTPServer) Provision(client *acme.Client, ident acme.AuthzID, chall *acme.Challenge) error {
+	path := client.HTTP01ChallengePath(chall.Token)
+	body, err := client.HTTP01ChallengeResponse(chall.Token)
+	if err != nil {
+		return fmt.Errorf("failed to generate HTTP-01 challenge response for %v challenge %s token %s: %w",
+			ident, chall.URI, chall.Token, err)
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	log.Printf("challHTTPServer: setting challenge response for %s", path)
+	c.challMap[path] = body
+
+	return nil
+}
+
+func (c *challHTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log.Printf("challHTTPServer: handling %s to %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	response, exists := c.challMap[r.URL.Path]
+
+	if !exists {
+		http.NotFound(w, r)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.Write([]byte(response))
+}
+
+func (c *challHTTPServer) Shutdown() error {
+	log.Printf("challHTTPServer: shutting down")
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	return c.Server.Shutdown(ctx)
+}
+
+func (c *challHTTPServer) Run() {
+	go func() {
+		if err := c.Server.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				log.Printf("challHTTPServer error: %v", err)
+			}
+		}
+	}()
+}
+
+func testIssuance(t *testing.T, env *environment, challSrv challengeServer) {
+	t.Helper()
+
+	// Bound the total issuance process by a timeout of 60 seconds.
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Create a new ACME account.
+	client := env.client
+	acct, err := client.Register(ctx, &acme.Account{}, acme.AcceptTOS)
+	if err != nil {
+		t.Fatalf("failed to register account: %v", err)
+	}
+	if acct.Status != acme.StatusValid {
+		t.Fatalf("expected new account status to be valid, got %v", acct.Status)
+	}
+	log.Printf("registered account: %s", acct.URI)
+
+	// Create a new order for some example identifiers
+	identifiers := []acme.AuthzID{
+		{
+			Type:  "dns",
+			Value: "example.com",
+		},
+		{
+			Type:  "dns",
+			Value: "www.example.com",
+		},
+		// TODO(@cpu): enable this identifier once IP addresses are handled correctly
+		//   by acme.TLSALPN01ChallengeCert
+		/*
+			{
+				Type:  "ip",
+				Value: "127.0.0.1",
+			},
+		*/
+	}
+	order, err := client.AuthorizeOrder(ctx, identifiers)
+	if err != nil {
+		t.Fatalf("failed to create order for %v: %v", identifiers, err)
+	}
+	if order.Status != acme.StatusPending {
+		t.Fatalf("expected new order status to be pending, got %v", order.Status)
+	}
+	orderURL := order.URI
+	log.Printf("created order: %v", orderURL)
+
+	// For each pending authz provision a supported challenge type's response
+	// with the test challenge server, and tell the ACME server to verify it.
+	for _, authzURL := range order.AuthzURLs {
+		authz, err := client.GetAuthorization(ctx, authzURL)
+		if err != nil {
+			t.Fatalf("failed to get order %s authorization %s: %v",
+				orderURL, authzURL, err)
+		}
+
+		if authz.Status != acme.StatusPending {
+			continue
+		}
+
+		for _, challenge := range authz.Challenges {
+			if challenge.Status != acme.StatusPending || !challSrv.Supported(challenge) {
+				continue
+			}
+
+			if err := challSrv.Provision(client, authz.Identifier, challenge); err != nil {
+				t.Fatalf("failed to provision challenge %s: %v", challenge.URI, err)
+			}
+
+			_, err = client.Accept(ctx, challenge)
+			if err != nil {
+				t.Fatalf("failed to accept order %s challenge %s: %v",
+					orderURL, challenge.URI, err)
+			}
+		}
+	}
+
+	// Wait for the order to become ready for finalization.
+	order, err = client.WaitOrder(ctx, order.URI)
+	if err != nil {
+		t.Fatalf("failed to wait for order %s: %s", orderURL, err)
+	}
+	if order.Status != acme.StatusReady {
+		t.Fatalf("expected order %s status to be ready, got %v",
+			orderURL, order.Status)
+	}
+
+	// Generate a certificate keypair and a CSR for the order identifiers.
+	certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("failed to generate certificate key: %v", err)
+	}
+	var dnsNames []string
+	var ipAddresses []net.IP
+	for _, ident := range identifiers {
+		switch ident.Type {
+		case "dns":
+			dnsNames = append(dnsNames, ident.Value)
+		case "ip":
+			ipAddresses = append(ipAddresses, net.ParseIP(ident.Value))
+		default:
+			t.Fatalf("unsupported identifier type: %s", ident.Type)
+		}
+	}
+	csrDer, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{
+		DNSNames:    dnsNames,
+		IPAddresses: ipAddresses,
+	}, certKey)
+	if err != nil {
+		t.Fatalf("failed to create CSR: %v", err)
+	}
+
+	// Finalize the order by creating a certificate with our CSR.
+	chain, _, err := client.CreateOrderCert(ctx, order.FinalizeURL, csrDer, true)
+	if err != nil {
+		t.Fatalf("failed to finalize order %s with finalize URL %s: %v",
+			orderURL, order.FinalizeURL, err)
+	}
+
+	// Split the chain into the leaf and any intermediates.
+	leaf := chain[0]
+	intermediatesDER := chain[1:]
+	leafCert, err := x509.ParseCertificate(leaf)
+	if err != nil {
+		t.Fatalf("failed to parse order %s leaf certificate: %v", orderURL, err)
+	}
+	intermediates := x509.NewCertPool()
+	for i, intermediateDER := range intermediatesDER {
+		intermediate, err := x509.ParseCertificate(intermediateDER)
+		if err != nil {
+			t.Fatalf("failed to parse intermediate %d: %v", i, err)
+		}
+		intermediates.AddCert(intermediate)
+	}
+
+	// Verify there is a valid path from the leaf certificate to Pebble's
+	// issuing root using the provided intermediate certificates.
+	roots, err := env.RootCert()
+	if err != nil {
+		t.Fatalf("failed to get Pebble issuer root certs: %v", err)
+	}
+	paths, err := leafCert.Verify(x509.VerifyOptions{
+		Intermediates: intermediates,
+		Roots:         roots,
+	})
+	if err != nil {
+		t.Fatalf("failed to verify order %s leaf certificate: %v", orderURL, err)
+	}
+	log.Printf("verified %d path(s) from issued leaf certificate to Pebble root CA", len(paths))
+
+	// Also verify that the leaf cert is valid for each of the DNS names
+	// and IP addresses from our order's identifiers.
+	for _, name := range dnsNames {
+		if err := leafCert.VerifyHostname(name); err != nil {
+			t.Fatalf("failed to verify order %s leaf certificate for order DNS name %s: %v",
+				orderURL, name, err)
+		}
+	}
+	for _, ip := range ipAddresses {
+		if err := leafCert.VerifyHostname(ip.String()); err != nil {
+			t.Fatalf("failed to verify order %s leaf certificate for order IP address %s: %v",
+				orderURL, ip, err)
+		}
+	}
+}
+
+type environment struct {
+	config *environmentConfig
+	client *acme.Client
+}
+
+// RootCert returns the Pebble CA's primary issuing hierarchy root certificate.
+// This is generated randomly at each startup and can be used to verify
+// certificate chains issued by Pebble's ACME interface. Note that this
+// is separate from the static root certificate used by the Pebble ACME
+// HTTPS interface.
+func (e *environment) RootCert() (*x509.CertPool, error) {
+	// NOTE: in the future we may want to consider the alternative chains
+	// 		 returned as Link alternative headers.
+	rootURL := fmt.Sprintf("https://%s/roots/0", e.config.pebbleConfig.ManagementListenAddress)
+	resp, err := e.client.HTTPClient.Get(rootURL)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to GET Pebble root CA from %s: %v", rootURL, err)
+	}
+
+	roots := x509.NewCertPool()
+	rootPEM, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse Pebble root CA PEM: %v", err)
+	}
+	rootDERBlock, _ := pem.Decode(rootPEM)
+	rootCA, err := x509.ParseCertificate(rootDERBlock.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse Pebble root CA DER: %v", err)
+	}
+	roots.AddCert(rootCA)
+
+	return roots, nil
+}
+
+// environmentConfig describes the Pebble configuration, and configuration
+// shared between pebble and pebble-challtestsrv.
+type environmentConfig struct {
+	pebbleConfig
+	dnsPort uint32
+}
+
+// defaultConfig returns an environmentConfig populated with default values.
+// The provided pebbleDir is used to specify certificate/private key paths
+// for the HTTPS ACME interface.
+func defaultConfig(pebbleDir string) environmentConfig {
+	return environmentConfig{
+		pebbleConfig: pebbleConfig{
+			ListenAddress:                  fmt.Sprintf("127.0.0.1:%d", takeNextPort()),
+			ManagementListenAddress:        fmt.Sprintf("127.0.0.1:%d", takeNextPort()),
+			HTTPPort:                       takeNextPort(),
+			TLSPort:                        takeNextPort(),
+			Certificate:                    fmt.Sprintf("%s/test/certs/localhost/cert.pem", pebbleDir),
+			PrivateKey:                     fmt.Sprintf("%s/test/certs/localhost/key.pem", pebbleDir),
+			OCSPResponderURL:               "",
+			ExternalAccountBindingRequired: false,
+			ExternalAccountMACKeys:         make(map[string]string),
+			DomainBlocklist:                []string{"blocked-domain.example"},
+			Profiles: map[string]struct {
+				Description    string
+				ValidityPeriod uint64
+			}{
+				"default": {
+					Description:    "default profile",
+					ValidityPeriod: 3600,
+				},
+			},
+			RetryAfter: struct {
+				Authz int
+				Order int
+			}{
+				3,
+				5,
+			},
+		},
+		dnsPort: takeNextPort(),
+	}
+}
+
+// pebbleConfig matches the JSON structure of the Pebble configuration file.
+type pebbleConfig struct {
+	ListenAddress                  string
+	ManagementListenAddress        string
+	HTTPPort                       uint32
+	TLSPort                        uint32
+	Certificate                    string
+	PrivateKey                     string
+	OCSPResponderURL               string
+	ExternalAccountBindingRequired bool
+	ExternalAccountMACKeys         map[string]string
+	DomainBlocklist                []string
+	Profiles                       map[string]struct {
+		Description    string
+		ValidityPeriod uint64
+	}
+	RetryAfter struct {
+		Authz int
+		Order int
+	}
+}
+
+func takeNextPort() uint32 {
+	return nextPort.Add(1) - 1
+}
+
+// startPebbleEnvironment is a test helper that spawns Pebble and Pebble
+// challenge test server processes based on the provided environmentConfig. The
+// processes will be torn down when the test ends.
+func startPebbleEnvironment(t *testing.T, config *environmentConfig) environment {
+	t.Helper()
+
+	var pebbleDir string
+	if *pebbleLocalDir != "" {
+		pebbleDir = *pebbleLocalDir
+	} else {
+		pebbleDir = fetchModule(t, "github.com/letsencrypt/pebble/v2", pebbleModVersion)
+	}
+
+	binDir := prepareBinaries(t, pebbleDir)
+
+	if config == nil {
+		cfg := defaultConfig(pebbleDir)
+		config = &cfg
+	}
+
+	marshalConfig := struct {
+		Pebble pebbleConfig
+	}{
+		Pebble: config.pebbleConfig,
+	}
+
+	configData, err := json.Marshal(marshalConfig)
+	if err != nil {
+		t.Fatalf("failed to marshal config: %v", err)
+	}
+
+	configFile, err := os.CreateTemp("", "pebble-config-*.json")
+	if err != nil {
+		t.Fatalf("failed to create temp config file: %v", err)
+	}
+	t.Cleanup(func() { os.Remove(configFile.Name()) })
+
+	if _, err := configFile.Write(configData); err != nil {
+		t.Fatalf("failed to write config file: %v", err)
+	}
+	configFile.Close()
+
+	log.Printf("pebble dir: %s", pebbleDir)
+	log.Printf("config file: %s", configFile.Name())
+
+	// Spawn the Pebble CA server. It answers ACME requests and performs
+	// outbound validations. We configure it to use a mock DNS server that
+	// always answers 127.0.0.1 for all A queries so that validation
+	// requests for any domain name will resolve to our local challenge
+	// server instances.
+	spawnServerProcess(t, binDir, "pebble", "-config", configFile.Name(),
+		"-dnsserver", fmt.Sprintf("127.0.0.1:%d", config.dnsPort),
+		"-strict")
+
+	// Spawn the Pebble challenge test server. We'll use it to mock DNS
+	// responses but disable all the other interfaces. We want to stand
+	// up our own challenge response servers for TLS-ALPN-01,
+	// etc.
+	// Note: we specify -defaultIPv6 "" so that no AAAA records are served.
+	// The LUCI CI runners have issues with IPv6 connectivity on localhost.
+	spawnServerProcess(t, binDir, "pebble-challtestsrv",
+		"-dns01", fmt.Sprintf(":%d", config.dnsPort),
+		"-defaultIPv6", "",
+		"-management", fmt.Sprintf(":%d", takeNextPort()),
+		"-doh", "",
+		"-http01", "",
+		"-tlsalpn01", "",
+		"-https01", "")
+
+	waitForServer(t, config.pebbleConfig.ListenAddress)
+	waitForServer(t, fmt.Sprintf("127.0.0.1:%d", config.dnsPort))
+
+	log.Printf("pebble environment ready")
+
+	// Construct a cert pool that contains the CA certificate used by the ACME
+	// interface's certificate chain. This is separate from the issuing
+	// hierarchy and is used for the ACME client to interact with the ACME
+	// interface without cert verification error.
+	caCertPath := filepath.Join(pebbleDir, "test/certs/pebble.minica.pem")
+	caCert, err := os.ReadFile(caCertPath)
+	if err != nil {
+		t.Fatalf("failed to read CA certificate %s: %v", caCertPath, err)
+	}
+	caCertPool := x509.NewCertPool()
+	if !caCertPool.AppendCertsFromPEM(caCert) {
+		t.Fatalf("failed to parse CA certificate %s", caCertPath)
+	}
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: caCertPool,
+			},
+		},
+	}
+
+	// Create an ACME account keypair/client and verify it can discover
+	// the Pebble server's ACME directory without error.
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("failed to generate account key: %v", err)
+	}
+	client := &acme.Client{
+		Key:          key,
+		HTTPClient:   httpClient,
+		DirectoryURL: fmt.Sprintf("https://%s/dir", config.ListenAddress),
+	}
+	_, err = client.Discover(context.TODO())
+	if err != nil {
+		t.Fatalf("failed to discover ACME directory: %v", err)
+	}
+
+	return environment{
+		config: config,
+		client: client,
+	}
+}
+
+func waitForServer(t *testing.T, addr string) {
+	t.Helper()
+
+	for i := 0; i < 10; i++ {
+		if conn, err := net.Dial("tcp", addr); err == nil {
+			conn.Close()
+			return
+		}
+		time.Sleep(time.Duration(i*100) * time.Millisecond)
+	}
+	t.Fatalf("failed to connect to %q after 10 tries", addr)
+}
+
+// fetchModule fetches the module at the given version and returns the directory
+// containing its source tree. It skips the test if fetching modules is not
+// possible in this environment.
+//
+// Copied from the stdlib cryptotest.FetchModule and adapted to not rely on the
+// stdlib internal testenv package.
+func fetchModule(t *testing.T, module, version string) string {
+	// If the default GOMODCACHE doesn't exist, use a temporary directory
+	// instead. (For example, run.bash sets GOPATH=/nonexist-gopath.)
+	out, err := exec.Command("go", "env", "GOMODCACHE").Output()
+	if err != nil {
+		t.Errorf("go env GOMODCACHE: %v\n%s", err, out)
+		if ee, ok := err.(*exec.ExitError); ok {
+			t.Logf("%s", ee.Stderr)
+		}
+		t.FailNow()
+	}
+	modcacheOk := false
+	if gomodcache := string(bytes.TrimSpace(out)); gomodcache != "" {
+		if _, err := os.Stat(gomodcache); err == nil {
+			modcacheOk = true
+		}
+	}
+	if !modcacheOk {
+		t.Setenv("GOMODCACHE", t.TempDir())
+		// Allow t.TempDir() to clean up subdirectories.
+		t.Setenv("GOFLAGS", os.Getenv("GOFLAGS")+" -modcacherw")
+	}
+
+	t.Logf("fetching %s@%s\n", module, version)
+
+	output, err := exec.Command("go", "mod", "download", "-json", module+"@"+version).CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to download %s@%s: %s\n%s\n", module, version, err, output)
+	}
+	var j struct {
+		Dir string
+	}
+	if err := json.Unmarshal(output, &j); err != nil {
+		t.Fatalf("failed to parse 'go mod download': %s\n%s\n", err, output)
+	}
+
+	return j.Dir
+}
+
+func prepareBinaries(t *testing.T, pebbleDir string) string {
+	t.Helper()
+
+	// We don't want to build in the module cache dir, which might not be
+	// writable or to pollute the user's clone with binaries if pebbleLocalDir
+	//is used.
+	binDir := t.TempDir()
+
+	build := func(cmd string) {
+		log.Printf("building %s", cmd)
+		buildCmd := exec.Command(
+			"go",
+			"build", "-o", filepath.Join(binDir, cmd), "-mod", "mod", "./cmd/"+cmd)
+		buildCmd.Dir = pebbleDir
+		output, err := buildCmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("failed to build %s: %s\n%s\n", cmd, err, output)
+		}
+	}
+
+	build("pebble")
+	build("pebble-challtestsrv")
+
+	return binDir
+}
+
+func spawnServerProcess(t *testing.T, dir string, cmd string, args ...string) {
+	t.Helper()
+
+	cmdInstance := exec.Command("./"+cmd, args...)
+	cmdInstance.Dir = dir
+	cmdInstance.Stdout = os.Stdout
+	cmdInstance.Stderr = os.Stderr
+	if err := cmdInstance.Start(); err != nil {
+		t.Fatalf("failed to start %s: %v", cmd, err)
+	}
+	t.Cleanup(func() {
+		cmdInstance.Process.Kill()
+	})
+}