internal/proxy/client.go

// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package proxy provides a client for interacting with a proxy.
package proxy

import (
	"archive/zip"
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"go.opencensus.io/plugin/ochttp"
	"golang.org/x/mod/module"
	"golang.org/x/net/context/ctxhttp"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/version"
)

// A Client is used by the fetch service to communicate with a module
// proxy. It handles all methods defined by go help goproxy.
type Client struct {
	// URL of the module proxy web server
	url string

	// Client used for HTTP requests. It is mutable for testing purposes.
	HTTPClient *http.Client

	// Whether fetch should be disabled.
	disableFetch bool

	cache *cache
}

// A VersionInfo contains metadata about a given version of a module.
type VersionInfo struct {
	Version string
	Time    time.Time
}

// Setting this header to true prevents the proxy from fetching uncached
// modules.
const DisableFetchHeader = "Disable-Module-Fetch"

// New constructs a *Client using the provided url, which is expected to
// be an absolute URI that can be directly passed to http.Get.
func New(u string) (_ *Client, err error) {
	defer derrors.WrapStack(&err, "proxy.New(%q)", u)
	return &Client{
		url:          strings.TrimRight(u, "/"),
		HTTPClient:   &http.Client{Transport: &ochttp.Transport{}},
		disableFetch: false,
	}, nil
}

// WithFetchDisabled returns a new client that sets the Disable-Module-Fetch
// header so that the proxy does not fetch a module it doesn't already know
// about.
func (c *Client) WithFetchDisabled() *Client {
	c2 := *c
	c2.disableFetch = true
	return &c2
}

// FetchDisabled reports whether proxy fetch is disabled.
func (c *Client) FetchDisabled() bool {
	return c.disableFetch
}

// WithCache returns a new client that caches some RPCs.
func (c *Client) WithCache() *Client {
	c2 := *c
	c2.cache = &cache{}
	return &c2
}

// Info makes a request to $GOPROXY/<module>/@v/<requestedVersion>.info and
// transforms that data into a *VersionInfo.
// If requestedVersion is internal.LatestVersion, it uses the proxy's @latest
// endpoint instead.
func (c *Client) Info(ctx context.Context, modulePath, requestedVersion string) (_ *VersionInfo, err error) {
	defer func() {
		// Don't report NotFetched, because it is the normal result of fetching
		// an uncached module when fetch is disabled.
		// Don't report timeouts, because they are relatively frequent and not actionable.
		wrap := derrors.Wrap
		if !errors.Is(err, derrors.NotFetched) && !errors.Is(err, derrors.ProxyTimedOut) && !errors.Is(err, derrors.NotFound) {
			wrap = derrors.WrapAndReport
		}
		wrap(&err, "proxy.Client.Info(%q, %q)", modulePath, requestedVersion)
	}()

	if v := c.cache.getInfo(modulePath, requestedVersion); v != nil {
		return v, nil
	}
	data, err := c.readBody(ctx, modulePath, requestedVersion, "info")
	if err != nil {
		return nil, err
	}
	var v VersionInfo
	if err := json.Unmarshal(data, &v); err != nil {
		return nil, err
	}
	c.cache.putInfo(modulePath, requestedVersion, &v)
	return &v, nil
}

// Mod makes a request to $GOPROXY/<module>/@v/<resolvedVersion>.mod and returns the raw data.
func (c *Client) Mod(ctx context.Context, modulePath, resolvedVersion string) (_ []byte, err error) {
	defer derrors.WrapStack(&err, "proxy.Client.Mod(%q, %q)", modulePath, resolvedVersion)

	if b := c.cache.getMod(modulePath, resolvedVersion); b != nil {
		return b, nil
	}
	b, err := c.readBody(ctx, modulePath, resolvedVersion, "mod")
	if err != nil {
		return nil, err
	}
	c.cache.putMod(modulePath, resolvedVersion, b)
	return b, nil
}

// Zip makes a request to $GOPROXY/<modulePath>/@v/<resolvedVersion>.zip and
// transforms that data into a *zip.Reader. <resolvedVersion> must have already
// been resolved by first making a request to
// $GOPROXY/<modulePath>/@v/<requestedVersion>.info to obtained the valid
// semantic version.
func (c *Client) Zip(ctx context.Context, modulePath, resolvedVersion string) (_ *zip.Reader, err error) {
	defer derrors.WrapStack(&err, "proxy.Client.Zip(ctx, %q, %q)", modulePath, resolvedVersion)

	if r := c.cache.getZip(modulePath, resolvedVersion); r != nil {
		return r, nil
	}
	bodyBytes, err := c.readBody(ctx, modulePath, resolvedVersion, "zip")
	if err != nil {
		return nil, err
	}
	zipReader, err := zip.NewReader(bytes.NewReader(bodyBytes), int64(len(bodyBytes)))
	if err != nil {
		return nil, fmt.Errorf("zip.NewReader: %v: %w", err, derrors.BadModule)
	}
	c.cache.putZip(modulePath, resolvedVersion, zipReader)
	return zipReader, nil
}

// ZipSize gets the size in bytes of the zip from the proxy, without downloading it.
// The version must be resolved, as by a call to Client.Info.
func (c *Client) ZipSize(ctx context.Context, modulePath, resolvedVersion string) (_ int64, err error) {
	defer derrors.WrapStack(&err, "proxy.Client.ZipSize(ctx, %q, %q)", modulePath, resolvedVersion)

	url, err := c.EscapedURL(modulePath, resolvedVersion, "zip")
	if err != nil {
		return 0, err
	}
	res, err := ctxhttp.Head(ctx, c.HTTPClient, url)
	if err != nil {
		return 0, fmt.Errorf("ctxhttp.Head(ctx, client, %q): %v", url, err)
	}
	defer res.Body.Close()
	if err := responseError(res, false); err != nil {
		return 0, err
	}
	if res.ContentLength < 0 {
		return 0, errors.New("unknown content length")
	}
	return res.ContentLength, nil
}

func (c *Client) EscapedURL(modulePath, requestedVersion, suffix string) (_ string, err error) {
	defer derrors.WrapStack(&err, "Client.escapedURL(%q, %q, %q)", modulePath, requestedVersion, suffix)

	if suffix != "info" && suffix != "mod" && suffix != "zip" {
		return "", errors.New(`suffix must be "info", "mod" or "zip"`)
	}
	escapedPath, err := module.EscapePath(modulePath)
	if err != nil {
		return "", fmt.Errorf("path: %v: %w", err, derrors.InvalidArgument)
	}
	if requestedVersion == version.Latest {
		if suffix != "info" {
			return "", fmt.Errorf("cannot ask for latest with suffix %q", suffix)
		}
		return fmt.Sprintf("%s/%s/@latest", c.url, escapedPath), nil
	}
	escapedVersion, err := module.EscapeVersion(requestedVersion)
	if err != nil {
		return "", fmt.Errorf("version: %v: %w", err, derrors.InvalidArgument)
	}
	return fmt.Sprintf("%s/%s/@v/%s.%s", c.url, escapedPath, escapedVersion, suffix), nil
}

func (c *Client) readBody(ctx context.Context, modulePath, requestedVersion, suffix string) (_ []byte, err error) {
	defer derrors.WrapStack(&err, "Client.readBody(%q, %q, %q)", modulePath, requestedVersion, suffix)

	u, err := c.EscapedURL(modulePath, requestedVersion, suffix)
	if err != nil {
		return nil, err
	}
	var data []byte
	err = c.executeRequest(ctx, u, func(body io.Reader) error {
		var err error
		data, err = io.ReadAll(body)
		return err
	})
	if err != nil {
		return nil, err
	}
	return data, nil
}

// Versions makes a request to $GOPROXY/<path>/@v/list and returns the
// resulting version strings.
func (c *Client) Versions(ctx context.Context, modulePath string) (_ []string, err error) {
	defer derrors.Wrap(&err, "Versions(ctx, %q)", modulePath)
	escapedPath, err := module.EscapePath(modulePath)
	if err != nil {
		return nil, fmt.Errorf("module.EscapePath(%q): %w", modulePath, derrors.InvalidArgument)
	}
	u := fmt.Sprintf("%s/%s/@v/list", c.url, escapedPath)
	var versions []string
	collect := func(body io.Reader) error {
		scanner := bufio.NewScanner(body)
		for scanner.Scan() {
			versions = append(versions, scanner.Text())
		}
		return scanner.Err()
	}
	if err := c.executeRequest(ctx, u, collect); err != nil {
		return nil, err
	}
	return versions, nil
}

// executeRequest executes an HTTP GET request for u, then calls the bodyFunc
// on the response body, if no error occurred.
func (c *Client) executeRequest(ctx context.Context, u string, bodyFunc func(body io.Reader) error) (err error) {
	defer func() {
		if ctx.Err() != nil {
			err = fmt.Errorf("%v: %w", err, derrors.ProxyTimedOut)
		}
		derrors.WrapStack(&err, "executeRequest(ctx, %q)", u)
	}()

	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return err
	}
	if c.disableFetch {
		req.Header.Set(DisableFetchHeader, "true")
	}
	r, err := ctxhttp.Do(ctx, c.HTTPClient, req)
	if err != nil {
		return fmt.Errorf("ctxhttp.Do(ctx, client, %q): %v", u, err)
	}
	defer r.Body.Close()
	if err := responseError(r, c.disableFetch); err != nil {
		return err
	}
	return bodyFunc(r.Body)
}

// responseError translates the response status code to an appropriate error.
func responseError(r *http.Response, fetchDisabled bool) error {
	switch {
	case 200 <= r.StatusCode && r.StatusCode < 300:
		return nil
	case 500 <= r.StatusCode:
		return derrors.ProxyError
	case r.StatusCode == http.StatusNotFound,
		r.StatusCode == http.StatusGone:
		// Treat both 404 Not Found and 410 Gone responses
		// from the proxy as a "not found" error category.
		// If the response body contains "fetch timed out", treat this
		// as a 504 response so that we retry fetching the module version again
		// later.
		//
		// If the Disable-Module-Fetch header was set, use a different
		// error code so we can tell the difference.
		data, err := io.ReadAll(r.Body)
		if err != nil {
			return fmt.Errorf("io.ReadAll: %v", err)
		}
		d := string(data)
		switch {
		case strings.Contains(d, "fetch timed out"):
			err = derrors.ProxyTimedOut
		case fetchDisabled:
			err = derrors.NotFetched
		default:
			err = derrors.NotFound
		}
		return fmt.Errorf("%q: %w", d, err)
	default:
		return fmt.Errorf("unexpected status %d %s", r.StatusCode, r.Status)
	}
}