src/cmd/goinstall/download.go

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Download remote packages.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

const dashboardURL = "http://godashboard.appspot.com/package"

// maybeReportToDashboard reports path to dashboard unless
// -dashboard=false is on command line.  It ignores errors.
func maybeReportToDashboard(path string) {
	// if -dashboard=false was on command line, do nothing
	if !*reportToDashboard {
		return
	}

	// otherwise lob url to dashboard
	r, _ := http.Post(dashboardURL, "application/x-www-form-urlencoded", strings.NewReader("path="+path))
	if r != nil && r.Body != nil {
		r.Body.Close()
	}
}

// a vcs represents a version control system
// like Mercurial, Git, or Subversion.
type vcs struct {
	name          string
	cmd           string
	metadir       string
	checkout      string
	clone         string
	update        string
	updateRevFlag string
	pull          string
	pullForceFlag string
	tagList       string
	tagListRe     *regexp.Regexp
	check         string
	protocols     []string
	suffix        string
}

func (v *vcs) String() string {
	return v.name
}

var vcsMap = map[string]*vcs{
	"hg": {
		name:      "Mercurial",
		cmd:       "hg",
		metadir:   ".hg",
		checkout:  "checkout",
		clone:     "clone",
		update:    "update",
		pull:      "pull",
		tagList:   "tags",
		tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"),
		check:     "identify",
		protocols: []string{"https", "http"},
		suffix:    ".hg",
	},

	"git": {
		name:      "Git",
		cmd:       "git",
		metadir:   ".git",
		checkout:  "checkout",
		clone:     "clone",
		update:    "pull",
		pull:      "fetch",
		tagList:   "tag",
		tagListRe: regexp.MustCompile("([^\n]+)\n"),
		check:     "ls-remote",
		protocols: []string{"git", "https", "http"},
		suffix:    ".git",
	},

	"svn": {
		name:      "Subversion",
		cmd:       "svn",
		metadir:   ".svn",
		checkout:  "checkout",
		clone:     "checkout",
		update:    "update",
		check:     "info",
		protocols: []string{"https", "http", "svn"},
		suffix:    ".svn",
	},

	"bzr": {
		name:          "Bazaar",
		cmd:           "bzr",
		metadir:       ".bzr",
		checkout:      "update",
		clone:         "branch",
		update:        "update",
		updateRevFlag: "-r",
		pull:          "pull",
		pullForceFlag: "--overwrite",
		tagList:       "tags",
		tagListRe:     regexp.MustCompile("([^ ]+)[^\n]+\n"),
		check:         "info",
		protocols:     []string{"https", "http", "bzr"},
		suffix:        ".bzr",
	},
}

type RemoteRepo interface {
	// IsCheckedOut returns whether this repository is checked
	// out inside the given srcDir (eg, $GOPATH/src).
	IsCheckedOut(srcDir string) bool

	// Repo returns the information about this repository: its url,
	// the part of the import path that forms the repository root,
	// and the version control system it uses. It may discover this
	// information by using the supplied client to make HTTP requests.
	Repo(*http.Client) (url, root string, vcs *vcs, err error)
}

type host struct {
	pattern *regexp.Regexp
	repo    func(repo string) (RemoteRepo, error)
}

var knownHosts = []host{
	{
		regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`),
		matchGoogleRepo,
	},
	{
		regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$`),
		matchGithubRepo,
	},
	{
		regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$`),
		matchBitbucketRepo,
	},
	{
		regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`),
		matchLaunchpadRepo,
	},
}

// baseRepo is the base implementation of RemoteRepo.
type baseRepo struct {
	url, root string
	vcs       *vcs
}

func (r *baseRepo) Repo(*http.Client) (url, root string, vcs *vcs, err error) {
	return r.url, r.root, r.vcs, nil
}

// IsCheckedOut reports whether the repo root inside srcDir contains a
// repository metadir. It updates the baseRepo's vcs field if necessary.
func (r *baseRepo) IsCheckedOut(srcDir string) bool {
	pkgPath := filepath.Join(srcDir, r.root)
	if r.vcs == nil {
		for _, vcs := range vcsMap {
			if isDir(filepath.Join(pkgPath, vcs.metadir)) {
				r.vcs = vcs
				return true
			}
		}
		return false
	}
	return isDir(filepath.Join(pkgPath, r.vcs.metadir))
}

// matchGithubRepo handles matches for github.com repositories.
func matchGithubRepo(root string) (RemoteRepo, error) {
	if strings.HasSuffix(root, ".git") {
		return nil, errors.New("path must not include .git suffix")
	}
	return &baseRepo{"http://" + root + ".git", root, vcsMap["git"]}, nil
}

// matchLaunchpadRepo handles matches for launchpad.net repositories.
func matchLaunchpadRepo(root string) (RemoteRepo, error) {
	return &baseRepo{"https://" + root, root, vcsMap["bzr"]}, nil
}

// matchGoogleRepo matches repos like "code.google.com/p/repo.subrepo/path".
func matchGoogleRepo(id string) (RemoteRepo, error) {
	root := "code.google.com/p/" + id
	return &googleRepo{baseRepo{"https://" + root, root, nil}}, nil
}

// googleRepo implements a RemoteRepo that discovers a Google Code
// repository's VCS type by scraping the code.google.com source checkout page.
type googleRepo struct{ baseRepo }

var googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)

func (r *googleRepo) Repo(client *http.Client) (url, root string, vcs *vcs, err error) {
	if r.vcs != nil {
		return r.url, r.root, r.vcs, nil
	}

	// Use the code.google.com source checkout page to find the VCS type.
	const prefix = "code.google.com/p/"
	p := strings.SplitN(r.root[len(prefix):], ".", 2)
	u := fmt.Sprintf("https://%s%s/source/checkout", prefix, p[0])
	if len(p) == 2 {
		u += fmt.Sprintf("?repo=%s", p[1])
	}
	resp, err := client.Get(u)
	if err != nil {
		return "", "", nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", "", nil, fmt.Errorf("fetching %s: %v", u, resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", "", nil, fmt.Errorf("fetching %s: %v", u, err)
	}

	// Scrape result for vcs details.
	m := googleRepoRe.FindSubmatch(b)
	if len(m) == 2 {
		if v := vcsMap[string(m[1])]; v != nil {
			r.vcs = v
			return r.url, r.root, r.vcs, nil
		}
	}

	return "", "", nil, errors.New("could not detect googlecode vcs")
}

// matchBitbucketRepo handles matches for all bitbucket.org repositories.
func matchBitbucketRepo(root string) (RemoteRepo, error) {
	if strings.HasSuffix(root, ".git") {
		return nil, errors.New("path must not include .git suffix")
	}
	return &bitbucketRepo{baseRepo{root: root}}, nil
}

// bitbucketRepo implements a RemoteRepo that uses the BitBucket API to
// discover the repository's VCS type.
type bitbucketRepo struct{ baseRepo }

func (r *bitbucketRepo) Repo(client *http.Client) (url, root string, vcs *vcs, err error) {
	if r.vcs != nil && r.url != "" {
		return r.url, r.root, r.vcs, nil
	}

	// Use the BitBucket API to find which kind of repository this is.
	const apiUrl = "https://api.bitbucket.org/1.0/repositories/"
	resp, err := client.Get(apiUrl + strings.SplitN(r.root, "/", 2)[1])
	if err != nil {
		return "", "", nil, fmt.Errorf("BitBucket API: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", "", nil, fmt.Errorf("BitBucket API: %v", resp.Status)
	}
	var response struct {
		Vcs string `json:"scm"`
	}
	err = json.NewDecoder(resp.Body).Decode(&response)
	if err != nil {
		return "", "", nil, fmt.Errorf("BitBucket API: %v", err)
	}
	switch response.Vcs {
	case "git":
		r.url = "http://" + r.root + ".git"
	case "hg":
		r.url = "http://" + r.root
	default:
		return "", "", nil, errors.New("unsupported bitbucket vcs: " + response.Vcs)
	}
	if r.vcs = vcsMap[response.Vcs]; r.vcs == nil {
		panic("vcs is nil when it should not be")
	}
	return r.url, r.root, r.vcs, nil
}

// findPublicRepo checks whether importPath is a well-formed path for one of
// the supported code hosting sites and, if so, returns a RemoteRepo.
func findPublicRepo(importPath string) (RemoteRepo, error) {
	for _, host := range knownHosts {
		if hm := host.pattern.FindStringSubmatch(importPath); hm != nil {
			return host.repo(hm[1])
		}
	}
	return nil, nil
}

// findAnyRepo matches import paths with a repo suffix (.git, etc).
func findAnyRepo(importPath string) RemoteRepo {
	for _, v := range vcsMap {
		i := strings.Index(importPath+"/", v.suffix+"/")
		if i < 0 {
			continue
		}
		if !strings.Contains(importPath[:i], "/") {
			continue // don't match vcs suffix in the host name
		}
		return &anyRepo{
			baseRepo{
				root: importPath[:i] + v.suffix,
				vcs:  v,
			},
			importPath[:i],
		}
	}
	return nil
}

// anyRepo implements an discoverable remote repo with a suffix (.git, etc).
type anyRepo struct {
	baseRepo
	rootWithoutSuffix string
}

func (r *anyRepo) Repo(*http.Client) (url, root string, vcs *vcs, err error) {
	if r.url != "" {
		return r.url, r.root, r.vcs, nil
	}
	url, err = r.vcs.findURL(r.rootWithoutSuffix)
	if url == "" && err == nil {
		err = fmt.Errorf("couldn't find %s repository", r.vcs.name)
	}
	if err != nil {
		return "", "", nil, err
	}
	r.url = url
	return r.url, r.root, r.vcs, nil
}

// findURL finds the URL for a given repo root by trying each combination of
// protocol and suffix in series.
func (v *vcs) findURL(root string) (string, error) {
	for _, proto := range v.protocols {
		for _, suffix := range []string{"", v.suffix} {
			url := proto + "://" + root + suffix
			out, err := exec.Command(v.cmd, v.check, url).CombinedOutput()
			if err == nil {
				printf("find %s: found %s\n", root, url)
				return url, nil
			}
			printf("findURL(%s): %s %s %s: %v\n%s\n", root, v.cmd, v.check, url, err, out)
		}
	}
	return "", nil
}

var oldGoogleRepo = regexp.MustCompile(`^([a-z0-9\-]+)\.googlecode\.com/(svn|git|hg)(/[a-z0-9A-Z_.\-/]+)?$`)

type errOldGoogleRepo struct {
	fixedPath string
}

func (e *errOldGoogleRepo) Error() string {
	return fmt.Sprintf("unsupported import path; should be %q", e.fixedPath)
}

// download checks out or updates the specified package from the remote server.
func download(importPath, srcDir string) (public bool, err error) {
	if strings.Contains(importPath, "..") {
		err = errors.New("invalid path (contains ..)")
		return
	}

	if m := oldGoogleRepo.FindStringSubmatch(importPath); m != nil {
		fixedPath := "code.google.com/p/" + m[1] + m[3]
		err = &errOldGoogleRepo{fixedPath}
		return
	}

	repo, err := findPublicRepo(importPath)
	if err != nil {
		return false, err
	}
	if repo != nil {
		public = true
	} else {
		repo = findAnyRepo(importPath)
	}
	if repo == nil {
		err = errors.New("cannot download: " + importPath)
		return
	}
	err = checkoutRepo(srcDir, repo)
	return
}

// checkoutRepo checks out repo into srcDir (if it's not checked out already)
// and, if the -u flag is set, updates the repository.
func checkoutRepo(srcDir string, repo RemoteRepo) error {
	if !repo.IsCheckedOut(srcDir) {
		// do checkout
		url, root, vcs, err := repo.Repo(http.DefaultClient)
		if err != nil {
			return err
		}
		repoPath := filepath.Join(srcDir, root)
		parent, _ := filepath.Split(repoPath)
		if err = os.MkdirAll(parent, 0777); err != nil {
			return err
		}
		if err = run(string(filepath.Separator), nil, vcs.cmd, vcs.clone, url, repoPath); err != nil {
			return err
		}
		return vcs.updateRepo(repoPath)
	}
	if *update {
		// do update
		_, root, vcs, err := repo.Repo(http.DefaultClient)
		if err != nil {
			return err
		}
		repoPath := filepath.Join(srcDir, root)
		// Retrieve new revisions from the remote branch, if the VCS
		// supports this operation independently (e.g. svn doesn't)
		if vcs.pull != "" {
			if vcs.pullForceFlag != "" {
				if err = run(repoPath, nil, vcs.cmd, vcs.pull, vcs.pullForceFlag); err != nil {
					return err
				}
			} else if err = run(repoPath, nil, vcs.cmd, vcs.pull); err != nil {
				return err
			}
		}
		// Update to release or latest revision
		return vcs.updateRepo(repoPath)
	}
	return nil
}

// updateRepo gets a list of tags in the repository and
// checks out the tag closest to the current runtime.Version.
// If no matching tag is found, it just updates to tip.
func (v *vcs) updateRepo(repoPath string) error {
	if v.tagList == "" || v.tagListRe == nil {
		// TODO(adg): fix for svn
		return run(repoPath, nil, v.cmd, v.update)
	}

	// Get tag list.
	stderr := new(bytes.Buffer)
	cmd := exec.Command(v.cmd, v.tagList)
	cmd.Dir = repoPath
	cmd.Stderr = stderr
	out, err := cmd.Output()
	if err != nil {
		return &RunError{strings.Join(cmd.Args, " "), repoPath, out, err}
	}
	var tags []string
	for _, m := range v.tagListRe.FindAllStringSubmatch(string(out), -1) {
		tags = append(tags, m[1])
	}

	// Only use the tag component of runtime.Version.
	ver := strings.Split(runtime.Version(), " ")[0]

	// Select tag.
	if tag := selectTag(ver, tags); tag != "" {
		printf("selecting revision %q\n", tag)
		return run(repoPath, nil, v.cmd, v.checkout, v.updateRevFlag+tag)
	}

	// No matching tag found, make default selection.
	printf("selecting tip\n")
	return run(repoPath, nil, v.cmd, v.update)
}

// selectTag returns the closest matching tag for a given version.
// Closest means the latest one that is not after the current release.
// Version "release.rN" matches tags of the form "go.rN" (N being a decimal).
// Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD".
func selectTag(goVersion string, tags []string) (match string) {
	const rPrefix = "release.r"
	if strings.HasPrefix(goVersion, rPrefix) {
		p := "go.r"
		v, err := strconv.ParseFloat(goVersion[len(rPrefix):], 64)
		if err != nil {
			return ""
		}
		var matchf float64
		for _, t := range tags {
			if !strings.HasPrefix(t, p) {
				continue
			}
			tf, err := strconv.ParseFloat(t[len(p):], 64)
			if err != nil {
				continue
			}
			if matchf < tf && tf <= v {
				match, matchf = t, tf
			}
		}
	}
	const wPrefix = "weekly."
	if strings.HasPrefix(goVersion, wPrefix) {
		p := "go.weekly."
		v := goVersion[len(wPrefix):]
		for _, t := range tags {
			if !strings.HasPrefix(t, p) {
				continue
			}
			if match < t && t[len(p):] <= v {
				match = t
			}
		}
	}
	return match
}

func isDir(dir string) bool {
	fi, err := os.Stat(dir)
	return err == nil && fi.IsDir()
}