vendor/cmd/go/internal/modfetch/coderepo.go

// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modfetch

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"cmd/go/internal/modconv"
	"cmd/go/internal/modfetch/codehost"
	"cmd/go/internal/modfile"
	"cmd/go/internal/module"
	"cmd/go/internal/semver"
)

// A codeRepo implements modfetch.Repo using an underlying codehost.Repo.
type codeRepo struct {
	modPath  string
	code     codehost.Repo
	codeRoot string
	codeDir  string

	path        string
	pathPrefix  string
	pathMajor   string
	pseudoMajor string
}

func newCodeRepo(code codehost.Repo, path string) (Repo, error) {
	codeRoot := code.Root()
	if !hasPathPrefix(path, codeRoot) {
		return nil, fmt.Errorf("mismatched repo: found %s for %s", codeRoot, path)
	}
	pathPrefix, pathMajor, ok := module.SplitPathVersion(path)
	if !ok {
		return nil, fmt.Errorf("invalid module path %q", path)
	}
	pseudoMajor := "v0"
	if pathMajor != "" {
		pseudoMajor = pathMajor[1:]
	}

	// At this point we might have:
	//	codeRoot = github.com/rsc/foo
	//	path = github.com/rsc/foo/bar/v2
	//	pathPrefix = github.com/rsc/foo/bar
	//	pathMajor = /v2
	//	pseudoMajor = v2
	//
	// Compute codeDir = bar, the subdirectory within the repo
	// corresponding to the module root.
	codeDir := strings.Trim(strings.TrimPrefix(pathPrefix, codeRoot), "/")
	if strings.HasPrefix(path, "gopkg.in/") {
		// But gopkg.in is a special legacy case, in which pathPrefix does not start with codeRoot.
		// For example we might have:
		//	codeRoot = gopkg.in/yaml.v2
		//	pathPrefix = gopkg.in/yaml
		//	pathMajor = .v2
		//	pseudoMajor = v2
		//	codeDir = pathPrefix (because codeRoot is not a prefix of pathPrefix)
		// Clear codeDir - the module root is the repo root for gopkg.in repos.
		codeDir = ""
	}

	r := &codeRepo{
		modPath:     path,
		code:        code,
		codeRoot:    codeRoot,
		codeDir:     codeDir,
		pathPrefix:  pathPrefix,
		pathMajor:   pathMajor,
		pseudoMajor: pseudoMajor,
	}

	return r, nil
}

func (r *codeRepo) ModulePath() string {
	return r.modPath
}

func (r *codeRepo) Versions(prefix string) ([]string, error) {
	p := prefix
	if r.codeDir != "" {
		p = r.codeDir + "/" + p
	}
	tags, err := r.code.Tags(p)
	if err != nil {
		return nil, err
	}
	list := []string{}
	for _, tag := range tags {
		if !strings.HasPrefix(tag, p) {
			continue
		}
		v := tag
		if r.codeDir != "" {
			v = v[len(r.codeDir)+1:]
		}
		if !semver.IsValid(v) || v != semver.Canonical(v) || isPseudoVersion(v) || !module.MatchPathMajor(v, r.pathMajor) {
			continue
		}
		list = append(list, v)
	}
	SortVersions(list)
	return list, nil
}

func (r *codeRepo) Stat(rev string) (*RevInfo, error) {
	if rev == "latest" {
		return r.Latest()
	}
	codeRev := r.revToRev(rev)
	if semver.IsValid(codeRev) && r.codeDir != "" {
		codeRev = r.codeDir + "/" + codeRev
	}
	info, err := r.code.Stat(codeRev)
	if err != nil {
		return nil, err
	}
	return r.convert(info)
}

func (r *codeRepo) Latest() (*RevInfo, error) {
	info, err := r.code.Latest()
	if err != nil {
		return nil, err
	}
	return r.convert(info)
}

func (r *codeRepo) convert(info *codehost.RevInfo) (*RevInfo, error) {
	versionOK := func(v string) bool {
		return semver.IsValid(v) && v == semver.Canonical(v) && !isPseudoVersion(v) && module.MatchPathMajor(v, r.pathMajor)
	}
	v := info.Version
	if r.codeDir == "" {
		if !versionOK(v) {
			v = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
		}
	} else {
		p := r.codeDir + "/"
		if strings.HasPrefix(v, p) && versionOK(v[len(p):]) {
			v = v[len(p):]
		} else {
			v = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
		}
	}

	info2 := &RevInfo{
		Name:    info.Name,
		Short:   info.Short,
		Time:    info.Time,
		Version: v,
	}
	return info2, nil
}

func (r *codeRepo) revToRev(rev string) string {
	if semver.IsValid(rev) {
		if isPseudoVersion(rev) {
			i := strings.Index(rev, "-")
			j := strings.Index(rev[i+1:], "-")
			return rev[i+1+j+1:]
		}
		if r.codeDir == "" {
			return rev
		}
		return r.codeDir + "/" + rev
	}
	return rev
}

func (r *codeRepo) versionToRev(version string) (rev string, err error) {
	if !semver.IsValid(version) {
		return "", fmt.Errorf("malformed semantic version %q", version)
	}
	return r.revToRev(version), nil
}

func (r *codeRepo) findDir(version string) (rev, dir string, gomod []byte, err error) {
	rev, err = r.versionToRev(version)
	if err != nil {
		return "", "", nil, err
	}
	if r.pathMajor == "" || strings.HasPrefix(r.pathMajor, ".") {
		if r.codeDir == "" {
			return rev, "", nil, nil
		}
		file1 := path.Join(r.codeDir, "go.mod")
		gomod1, err1 := r.code.ReadFile(rev, file1, codehost.MaxGoMod)
		if err1 != nil {
			return "", "", nil, fmt.Errorf("missing go.mod")
		}
		return rev, r.codeDir, gomod1, nil
	}

	// Suppose pathMajor is "/v2".
	// Either go.mod should claim v2 and v2/go.mod should not exist,
	// or v2/go.mod should exist and claim v2. Not both.
	// Note that we don't check the full path, just the major suffix,
	// because of replacement modules. This might be a fork of
	// the real module, found at a different path, usable only in
	// a replace directive.
	file1 := path.Join(r.codeDir, "go.mod")
	file2 := path.Join(r.codeDir, r.pathMajor[1:], "go.mod")
	gomod1, err1 := r.code.ReadFile(rev, file1, codehost.MaxGoMod)
	gomod2, err2 := r.code.ReadFile(rev, file2, codehost.MaxGoMod)
	found1 := err1 == nil && isMajor(gomod1, r.pathMajor)
	found2 := err2 == nil && isMajor(gomod2, r.pathMajor)

	if err2 == nil && !found2 {
		return "", "", nil, fmt.Errorf("%s has non-...%s module path", file2, r.pathMajor)
	}
	if found1 && found2 {
		return "", "", nil, fmt.Errorf("both %s and %s claim ...%s module", file1, file2, r.pathMajor)
	}
	if found2 {
		return rev, filepath.Join(r.codeDir, r.pathMajor), gomod2, nil
	}
	if found1 {
		return rev, r.codeDir, gomod1, nil
	}
	return "", "", nil, fmt.Errorf("missing go.mod")
}

func isMajor(gomod []byte, pathMajor string) bool {
	return strings.HasSuffix(modPath(gomod), pathMajor)
}

var moduleStr = []byte("module")

func modPath(mod []byte) string {
	for len(mod) > 0 {
		line := mod
		mod = nil
		if i := bytes.IndexByte(line, '\n'); i >= 0 {
			line, mod = line[:i], line[i+1:]
		}
		line = bytes.TrimSpace(line)
		if !bytes.HasPrefix(line, moduleStr) {
			continue
		}
		line = line[len(moduleStr):]
		n := len(line)
		line = bytes.TrimSpace(line)
		if len(line) == n || len(line) == 0 || line[0] != '"' && line[0] != '`' {
			continue
		}
		q := line[0]
		i := 1
		for i < len(line) && line[i] != q {
			if q == '"' && line[i] == '\\' {
				i++
			}
			i++
		}
		if i >= len(line) {
			return "" // malformed quoted string or multiline module path
		}
		p, err := strconv.Unquote(string(line[:i+1]))
		if err != nil {
			return ""
		}
		return p
	}
	return "" // missing module path
}

func (r *codeRepo) GoMod(version string) (data []byte, err error) {
	rev, dir, gomod, err := r.findDir(version)
	if err != nil {
		return nil, err
	}
	if gomod != nil {
		return gomod, nil
	}
	data, err = r.code.ReadFile(rev, path.Join(dir, "go.mod"), codehost.MaxGoMod)
	if err != nil {
		if e := strings.ToLower(err.Error()); strings.Contains(e, "not found") || strings.Contains(e, "404") { // TODO
			return r.legacyGoMod(rev, dir), nil
		}
		return nil, err
	}
	return data, nil
}

var altConfigs = []string{
	"Gopkg.lock",

	"GLOCKFILE",
	"Godeps/Godeps.json",
	"dependencies.tsv",
	"glide.lock",
	"vendor.conf",
	"vendor.yml",
	"vendor/manifest",
	"vendor/vendor.json",
}

func (r *codeRepo) legacyGoMod(rev, dir string) []byte {
	mf := new(modfile.File)
	mf.AddModuleStmt(r.modPath)
	for _, file := range altConfigs {
		data, err := r.code.ReadFile(rev, path.Join(dir, file), codehost.MaxGoMod)
		if err != nil {
			continue
		}
		convert := modconv.Converters[file]
		if convert == nil {
			continue
		}
		if err := ConvertLegacyConfig(mf, file, data); err != nil {
			continue
		}
		break
	}
	data, err := mf.Format()
	if err != nil {
		return []byte(fmt.Sprintf("%s\nmodule %q\n", modconv.Prefix, r.modPath))
	}
	return append([]byte(modconv.Prefix+"\n"), data...)
}

func (r *codeRepo) modPrefix(rev string) string {
	return r.modPath + "@" + rev
}

func (r *codeRepo) Zip(version string, tmpdir string) (tmpfile string, err error) {
	rev, dir, _, err := r.findDir(version)
	if err != nil {
		return "", err
	}
	dl, actualDir, err := r.code.ReadZip(rev, dir, codehost.MaxZipFile)
	if err != nil {
		return "", err
	}
	if actualDir != "" && !hasPathPrefix(dir, actualDir) {
		return "", fmt.Errorf("internal error: downloading %v %v: dir=%q but actualDir=%q", r.path, rev, dir, actualDir)
	}
	subdir := strings.Trim(strings.TrimPrefix(dir, actualDir), "/")

	// Spool to local file.
	f, err := ioutil.TempFile(tmpdir, "vgo-codehost-")
	if err != nil {
		dl.Close()
		return "", err
	}
	defer os.Remove(f.Name())
	defer f.Close()
	maxSize := int64(codehost.MaxZipFile)
	lr := &io.LimitedReader{R: dl, N: maxSize + 1}
	if _, err := io.Copy(f, lr); err != nil {
		dl.Close()
		return "", err
	}
	dl.Close()
	if lr.N <= 0 {
		return "", fmt.Errorf("downloaded zip file too large")
	}
	size := (maxSize + 1) - lr.N
	if _, err := f.Seek(0, 0); err != nil {
		return "", err
	}

	// Translate from zip file we have to zip file we want.
	zr, err := zip.NewReader(f, size)
	if err != nil {
		return "", err
	}
	f2, err := ioutil.TempFile(tmpdir, "vgo-")
	if err != nil {
		return "", err
	}

	zw := zip.NewWriter(f2)
	newName := f2.Name()
	defer func() {
		f2.Close()
		if err != nil {
			os.Remove(newName)
		}
	}()
	if subdir != "" {
		subdir += "/"
	}
	haveLICENSE := false
	topPrefix := ""
	haveGoMod := make(map[string]bool)
	for _, zf := range zr.File {
		if topPrefix == "" {
			i := strings.Index(zf.Name, "/")
			if i < 0 {
				return "", fmt.Errorf("missing top-level directory prefix")
			}
			topPrefix = zf.Name[:i+1]
		}
		if !strings.HasPrefix(zf.Name, topPrefix) {
			return "", fmt.Errorf("zip file contains more than one top-level directory")
		}
		dir, file := path.Split(zf.Name)
		if file == "go.mod" {
			haveGoMod[dir] = true
		}
	}
	root := topPrefix + subdir
	inSubmodule := func(name string) bool {
		for {
			dir, _ := path.Split(name)
			if len(dir) <= len(root) {
				return false
			}
			if haveGoMod[dir] {
				return true
			}
			name = dir[:len(dir)-1]
		}
	}
	for _, zf := range zr.File {
		if topPrefix == "" {
			i := strings.Index(zf.Name, "/")
			if i < 0 {
				return "", fmt.Errorf("missing top-level directory prefix")
			}
			topPrefix = zf.Name[:i+1]
		}
		if strings.HasSuffix(zf.Name, "/") { // drop directory dummy entries
			continue
		}
		if !strings.HasPrefix(zf.Name, topPrefix) {
			return "", fmt.Errorf("zip file contains more than one top-level directory")
		}
		name := strings.TrimPrefix(zf.Name, topPrefix)
		if !strings.HasPrefix(name, subdir) {
			continue
		}
		name = strings.TrimPrefix(name, subdir)
		if isVendoredPackage(name) {
			continue
		}
		if inSubmodule(zf.Name) {
			continue
		}
		base := path.Base(name)
		if strings.ToLower(base) == "go.mod" && base != "go.mod" {
			return "", fmt.Errorf("zip file contains %s, want all lower-case go.mod", zf.Name)
		}
		if name == "LICENSE" {
			haveLICENSE = true
		}
		size := int64(zf.UncompressedSize)
		if size < 0 || maxSize < size {
			return "", fmt.Errorf("module source tree too big")
		}
		maxSize -= size

		rc, err := zf.Open()
		if err != nil {
			return "", err
		}
		w, err := zw.Create(r.modPrefix(version) + "/" + name)
		lr := &io.LimitedReader{R: rc, N: size + 1}
		if _, err := io.Copy(w, lr); err != nil {
			return "", err
		}
		if lr.N <= 0 {
			return "", fmt.Errorf("individual file too large")
		}
	}

	if !haveLICENSE && subdir != "" {
		if data, err := r.code.ReadFile(rev, "LICENSE", codehost.MaxLICENSE); err == nil {
			w, err := zw.Create(r.modPrefix(version) + "/LICENSE")
			if err != nil {
				return "", err
			}
			if _, err := w.Write(data); err != nil {
				return "", err
			}
		}
	}
	if err := zw.Close(); err != nil {
		return "", err
	}
	if err := f2.Close(); err != nil {
		return "", err
	}

	return f2.Name(), nil
}

// hasPathPrefix reports whether the path s begins with the
// elements in prefix.
func hasPathPrefix(s, prefix string) bool {
	switch {
	default:
		return false
	case len(s) == len(prefix):
		return s == prefix
	case len(s) > len(prefix):
		if prefix != "" && prefix[len(prefix)-1] == '/' {
			return strings.HasPrefix(s, prefix)
		}
		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
	}
}

func isVendoredPackage(name string) bool {
	var i int
	if strings.HasPrefix(name, "vendor/") {
		i += len("vendor/")
	} else if j := strings.Index(name, "/vendor/"); j >= 0 {
		i += len("/vendor/")
	} else {
		return false
	}
	return strings.Contains(name[i:], "/")
}

func PseudoVersion(major string, t time.Time, rev string) string {
	if major == "" {
		major = "v0"
	}
	return fmt.Sprintf("%s.0.0-%s-%s", major, t.UTC().Format("20060102150405"), rev)
}

var ErrNotPseudoVersion = errors.New("not a pseudo-version")

/*
func ParsePseudoVersion(repo Repo, version string) (rev string, err error) {
	major := semver.Major(version)
	if major == "" {
		return "", ErrNotPseudoVersion
	}
	majorPrefix := major + ".0.0-"
	if !strings.HasPrefix(version, majorPrefix) || !strings.Contains(version[len(majorPrefix):], "-") {
		return "", ErrNotPseudoVersion
	}
	versionSuffix := version[len(majorPrefix):]
	for i := 0; versionSuffix[i] != '-'; i++ {
		c := versionSuffix[i]
		if c < '0' || '9' < c {
			return "", ErrNotPseudoVersion
		}
	}
	rev = versionSuffix[strings.Index(versionSuffix, "-")+1:]
	if rev == "" {
		return "", ErrNotPseudoVersion
	}
	if proxyURL != "" {
		return version, nil
	}
	fullRev, t, err := repo.CommitInfo(rev)
	if err != nil {
		return "", fmt.Errorf("unknown pseudo-version %s: loading %v: %v", version, rev, err)
	}
	v := PseudoVersion(major, t, repo.ShortRev(fullRev))
	if v != version {
		return "", fmt.Errorf("unknown pseudo-version %s: %v is %v", version, rev, v)
	}
	return fullRev, nil
}
*/

var pseudoVersionRE = regexp.MustCompile(`^v[0-9]+\.0\.0-[0-9]{14}-[A-Za-z0-9]+$`)

func isPseudoVersion(v string) bool {
	return pseudoVersionRE.MatchString(v)
}