internal/stdlib/stdlib.go

// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package stdlib supports special handling of the Go standard library.
// Regardless of the how the standard library has been split into modules for
// development and testing, the discovery site treats it as a single module
// named "std".
package stdlib

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/mod/semver"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/testing/testhelper"

	"github.com/go-git/go-billy/v5/osfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/filemode"
	"github.com/go-git/go-git/v5/plumbing/object"
	"github.com/go-git/go-git/v5/storage/memory"
)

// ModulePath is the name of the module for the standard library.
const ModulePath = "std"

var (
	// Regexp for matching go tags. The groups are:
	// 1  the major.minor version
	// 2  the patch version, or empty if none
	// 3  the entire prerelease, if present
	// 4  the prerelease type ("beta" or "rc")
	// 5  the prerelease number
	tagRegexp = regexp.MustCompile(`^go(\d+\.\d+)(\.\d+|)((beta|rc)(\d+))?$`)
)

// VersionForTag returns the semantic version for the Go tag, or "" if
// tag doesn't correspond to a Go release or beta tag.
// Examples:
//   "go1" => "v1.0.0"
//   "go1.2" => "v1.2.0"
//   "go1.13beta1" => "v1.13.0-beta.1"
//   "go1.9rc2" => "v1.9.0-rc.2"
func VersionForTag(tag string) string {
	// Special cases for go1.
	if tag == "go1" {
		return "v1.0.0"
	}
	if tag == "go1.0" {
		return ""
	}
	m := tagRegexp.FindStringSubmatch(tag)
	if m == nil {
		return ""
	}
	version := "v" + m[1]
	if m[2] != "" {
		version += m[2]
	} else {
		version += ".0"
	}
	if m[3] != "" {
		version += "-" + m[4] + "." + m[5]
	}
	return version
}

// TagForVersion returns the Go standard library repository tag corresponding
// to semver. The Go tags differ from standard semantic versions in a few ways,
// such as beginning with "go" instead of "v".
func TagForVersion(version string) (_ string, err error) {
	defer derrors.Wrap(&err, "TagForVersion(%q)", version)

	// Special case: v1.0.0 => go1.
	if version == "v1.0.0" {
		return "go1", nil
	}
	if !semver.IsValid(version) {
		return "", fmt.Errorf("%w: requested version is not a valid semantic version: %q ", derrors.InvalidArgument, version)
	}
	goVersion := semver.Canonical(version)
	prerelease := semver.Prerelease(goVersion)
	versionWithoutPrerelease := strings.TrimSuffix(goVersion, prerelease)
	patch := strings.TrimPrefix(versionWithoutPrerelease, semver.MajorMinor(goVersion)+".")
	if patch == "0" {
		versionWithoutPrerelease = strings.TrimSuffix(versionWithoutPrerelease, ".0")
	}

	goVersion = fmt.Sprintf("go%s", strings.TrimPrefix(versionWithoutPrerelease, "v"))
	if prerelease != "" {
		// Go prereleases look like  "beta1" instead of "beta.1".
		// "beta1" is bad for sorting (since beta10 comes before beta9), so
		// require the dot form.
		i := finalDigitsIndex(prerelease)
		if i >= 1 {
			if prerelease[i-1] != '.' {
				return "", fmt.Errorf("%w: final digits in a prerelease must follow a period", derrors.InvalidArgument)
			}
			// Remove the dot.
			prerelease = prerelease[:i-1] + prerelease[i:]
		}
		goVersion += strings.TrimPrefix(prerelease, "-")
	}
	return goVersion, nil
}

// MajorVersionForVersion returns the Go major version for version.
// E.g. "v1.13.3" => "go1".
func MajorVersionForVersion(version string) (_ string, err error) {
	defer derrors.Wrap(&err, "MajorVersionForVersion(%q)", version)

	tag, err := TagForVersion(version)
	if err != nil {
		return "", err
	}
	if tag == "go1" {
		return tag, nil
	}
	i := strings.IndexRune(tag, '.')
	if i < 0 {
		return "", fmt.Errorf("no '.' in go tag %q", tag)
	}
	return tag[:i], nil
}

// finalDigitsIndex returns the index of the first digit in the sequence of digits ending s.
// If s doesn't end in digits, it returns -1.
func finalDigitsIndex(s string) int {
	// Assume ASCII (since the semver package does anyway).
	var i int
	for i = len(s) - 1; i >= 0; i-- {
		if s[i] < '0' || s[i] > '9' {
			break
		}
	}
	if i == len(s)-1 {
		return -1
	}
	return i + 1
}

const (
	GoRepoURL       = "https://go.googlesource.com/go"
	GoSourceRepoURL = "https://github.com/golang/go"
)

// UseTestData determines whether to really clone the Go repo, or use
// stripped-down versions of the repo from the testdata directory.
var UseTestData = false

// TestCommitTime is the time used for all commits when UseTestData is true.
var TestCommitTime = time.Date(2019, 9, 4, 1, 2, 3, 0, time.UTC)

// getGoRepo returns a repo object for the Go repo at version.
func getGoRepo(version string) (_ *git.Repository, err error) {
	tag, err := TagForVersion(version)
	if err != nil {
		return nil, err
	}
	return git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
		URL:           GoRepoURL,
		ReferenceName: plumbing.NewTagReferenceName(tag),
		SingleBranch:  true,
		Depth:         1,
		Tags:          git.NoTags,
	})
}

// getTestGoRepo gets a Go repo for testing.
func getTestGoRepo(version string) (_ *git.Repository, err error) {
	fs := osfs.New(filepath.Join(testhelper.TestDataPath("testdata"), version))
	repo, err := git.Init(memory.NewStorage(), fs)
	if err != nil {
		return nil, err
	}
	wt, err := repo.Worktree()
	if err != nil {
		return nil, err
	}
	// Add all files in the directory.
	if _, err := wt.Add(""); err != nil {
		return nil, err
	}
	_, err = wt.Commit("", &git.CommitOptions{All: true, Author: &object.Signature{
		Name:  "Joe Random",
		Email: "joe@example.com",
		When:  TestCommitTime,
	}})
	if err != nil {
		return nil, err
	}
	return repo, nil
}

// Versions returns all the versions of Go that are relevant to the discovery
// site. These are all release versions (tags of the forms "goN.N" and
// "goN.N.N", where N is a number) and beta or rc versions (tags of the forms
// "goN.NbetaN" and "goN.N.NbetaN", and similarly for "rc" replacing "beta").
func Versions() (_ []string, err error) {
	defer derrors.Wrap(&err, "Versions()")

	var refNames []plumbing.ReferenceName
	if UseTestData {
		refNames = testRefs
	} else {
		re := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
			URLs: []string{GoRepoURL},
		})
		refs, err := re.List(&git.ListOptions{})
		if err != nil {
			return nil, err
		}
		for _, r := range refs {
			refNames = append(refNames, r.Name())
		}
	}

	var versions []string
	for _, name := range refNames {
		if !name.IsTag() {
			continue
		}
		v := VersionForTag(name.Short())
		if v != "" {
			versions = append(versions, v)
		}
	}
	return versions, nil
}

// Directory returns the directory of the standard library relative to the repo root.
func Directory(version string) string {
	// For versions older than v1.4.0-beta.1, the stdlib is in src/pkg.
	if semver.Compare(version, "v1.4.0-beta.1") == -1 {
		return "src/pkg"
	}
	return "src"
}

// Zip creates a module zip representing the entire Go standard library at the
// given version and returns a reader to it. It also returns the time of the
// commit for that version. The zip file is in module form, with each path
// prefixed by ModuleName + "@" + version.
//
// Zip reads the standard library at the Go repository tag corresponding to to
// the given semantic version.
//
// Zip ignores go.mod files in the standard library, treating it as if it were a
// single module named "std" at the given version.
func Zip(version string) (_ *zip.Reader, commitTime time.Time, err error) {
	// This code taken, with modifications, from
	// https://github.com/shurcooL/play/blob/master/256/moduleproxy/std/std.go.
	defer derrors.Wrap(&err, "stdlib.Zip(%q)", version)

	knownVersions, err := Versions()
	if err != nil {
		return nil, time.Time{}, err
	}
	found := false
	for _, v := range knownVersions {
		if v == version {
			found = true
			break
		}
	}
	if !found {
		return nil, time.Time{}, fmt.Errorf("%w: requested version unknown: %q", derrors.InvalidArgument, version)
	}

	var repo *git.Repository
	if UseTestData {
		repo, err = getTestGoRepo(version)
	} else {
		repo, err = getGoRepo(version)
	}
	if err != nil {
		return nil, time.Time{}, err
	}
	var buf bytes.Buffer
	z := zip.NewWriter(&buf)
	head, err := repo.Head()
	if err != nil {
		return nil, time.Time{}, err
	}
	commit, err := repo.CommitObject(head.Hash())
	if err != nil {
		return nil, time.Time{}, err
	}
	root, err := repo.TreeObject(commit.TreeHash)
	if err != nil {
		return nil, time.Time{}, err
	}
	prefixPath := ModulePath + "@" + version
	// Add top-level files.
	if err := addFiles(z, repo, root, prefixPath, false); err != nil {
		return nil, time.Time{}, err
	}
	// Add files from the stdlib directory.
	libdir := root
	for _, d := range strings.Split(Directory(version), "/") {
		libdir, err = subTree(repo, libdir, d)
		if err != nil {
			return nil, time.Time{}, err
		}
	}
	if err := addFiles(z, repo, libdir, prefixPath, true); err != nil {
		return nil, time.Time{}, err
	}
	if err := z.Close(); err != nil {
		return nil, time.Time{}, err
	}
	br := bytes.NewReader(buf.Bytes())
	zr, err := zip.NewReader(br, int64(br.Len()))
	if err != nil {
		return nil, time.Time{}, err
	}
	return zr, commit.Committer.When, nil
}

// addFiles adds the files in t to z, using dirpath as the path prefix.
// If recursive is true, it also adds the files in all subdirectories.
func addFiles(z *zip.Writer, r *git.Repository, t *object.Tree, dirpath string, recursive bool) error {
	for _, e := range t.Entries {
		if strings.HasPrefix(e.Name, ".") || strings.HasPrefix(e.Name, "_") {
			continue
		}
		if e.Name == "go.mod" {
			// ignore; we'll synthesize our own
			continue
		}
		switch e.Mode {
		case filemode.Regular, filemode.Executable:
			blob, err := r.BlobObject(e.Hash)
			if err != nil {
				return err
			}
			src, err := blob.Reader()
			if err != nil {
				return err
			}
			if err := writeZipFile(z, path.Join(dirpath, e.Name), src); err != nil {
				_ = src.Close()
				return err
			}
			if err := src.Close(); err != nil {
				return err
			}
		case filemode.Dir:
			if !recursive || e.Name == "testdata" {
				continue
			}
			t2, err := r.TreeObject(e.Hash)
			if err != nil {
				return err
			}
			if err := addFiles(z, r, t2, path.Join(dirpath, e.Name), recursive); err != nil {
				return err
			}
		}
	}
	return nil
}

func writeZipFile(z *zip.Writer, pathname string, src io.Reader) error {
	dst, err := z.Create(pathname)
	if err != nil {
		return err
	}
	_, err = io.Copy(dst, src)
	return err
}

// subTree looks non-recursively for a directory with the given name in t,
// and returns the corresponding tree.
// If a directory with such name doesn't exist in t, it returns os.ErrNotExist.
func subTree(r *git.Repository, t *object.Tree, name string) (*object.Tree, error) {
	for _, e := range t.Entries {
		if e.Name == name {
			return r.TreeObject(e.Hash)
		}
	}
	return nil, os.ErrNotExist
}

// Contains reports whether the given import path could be part of the Go standard library,
// by reporting whether the first component lacks a '.'.
func Contains(path string) bool {
	if i := strings.IndexByte(path, '/'); i != -1 {
		path = path[:i]
	}
	return !strings.Contains(path, ".")
}

// References used for Versions during testing.
var testRefs = []plumbing.ReferenceName{
	"refs/changes/56/93156/13",
	"refs/tags/weekly.2011-04-13",
	"refs/tags/go1.8rc2",
	"refs/tags/go1.8",
	"refs/tags/release.r59",
	"refs/tags/go1.9rc1",
	"refs/tags/go1.12.1",
	"refs/tags/go1.12.5",
	"refs/tags/go1.12.9",
	"refs/tags/go1.6beta1",
	"refs/tags/go1.6",
	"refs/tags/go1.13beta1",
	"refs/tags/go1.12",
	"refs/tags/go1.2.1",
	"refs/tags/go1.4.3",
	"refs/tags/go1.6.3",
	"refs/tags/go1.4.2",
	"refs/tags/go1.11",
	"refs/tags/go1.13",
	"refs/tags/go1.3.2",
}