| // Copyright 2018 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package codehost |
| |
| import ( |
| "encoding/xml" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "os" |
| "path/filepath" |
| "regexp" |
| "sort" |
| "strconv" |
| "strings" |
| "sync" |
| "time" |
| |
| "cmd/go/internal/par" |
| "cmd/go/internal/str" |
| ) |
| |
| func NewRepo(vcs, remote string) (Repo, error) { |
| type key struct { |
| vcs string |
| remote string |
| } |
| type cached struct { |
| repo Repo |
| err error |
| } |
| c := vcsRepoCache.Do(key{vcs, remote}, func() interface{} { |
| repo, err := newVCSRepo(vcs, remote) |
| return cached{repo, err} |
| }).(cached) |
| |
| return c.repo, c.err |
| } |
| |
| var vcsRepoCache par.Cache |
| |
| type vcsRepo struct { |
| remote string |
| cmd *vcsCmd |
| dir string |
| |
| tagsOnce sync.Once |
| tags map[string]bool |
| |
| branchesOnce sync.Once |
| branches map[string]bool |
| |
| fetchOnce sync.Once |
| fetchErr error |
| } |
| |
| func newVCSRepo(vcs, remote string) (Repo, error) { |
| if vcs == "git" { |
| return newGitRepo(remote, false) |
| } |
| cmd := vcsCmds[vcs] |
| if cmd == nil { |
| return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote) |
| } |
| if !strings.Contains(remote, "://") { |
| return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote) |
| } |
| r := &vcsRepo{remote: remote, cmd: cmd} |
| if cmd.init == nil { |
| return r, nil |
| } |
| dir, err := WorkDir(vcsWorkDirType+vcs, r.remote) |
| if err != nil { |
| return nil, err |
| } |
| r.dir = dir |
| if _, err := os.Stat(filepath.Join(dir, "."+vcs)); err != nil { |
| if _, err := Run(dir, cmd.init(r.remote)); err != nil { |
| os.RemoveAll(dir) |
| return nil, err |
| } |
| } |
| return r, nil |
| } |
| |
| const vcsWorkDirType = "vcs1." |
| |
| type vcsCmd struct { |
| vcs string // vcs name "hg" |
| init func(remote string) []string // cmd to init repo to track remote |
| tags func(remote string) []string // cmd to list local tags |
| tagRE *regexp.Regexp // regexp to extract tag names from output of tags cmd |
| branches func(remote string) []string // cmd to list local branches |
| branchRE *regexp.Regexp // regexp to extract branch names from output of tags cmd |
| badLocalRevRE *regexp.Regexp // regexp of names that must not be served out of local cache without doing fetch first |
| statLocal func(rev, remote string) []string // cmd to stat local rev |
| parseStat func(rev, out string) (*RevInfo, error) // cmd to parse output of statLocal |
| fetch []string // cmd to fetch everything from remote |
| latest string // name of latest commit on remote (tip, HEAD, etc) |
| readFile func(rev, file, remote string) []string // cmd to read rev's file |
| readZip func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file |
| } |
| |
| var re = regexp.MustCompile |
| |
| var vcsCmds = map[string]*vcsCmd{ |
| "hg": { |
| vcs: "hg", |
| init: func(remote string) []string { |
| return []string{"hg", "clone", "-U", remote, "."} |
| }, |
| tags: func(remote string) []string { |
| return []string{"hg", "tags", "-q"} |
| }, |
| tagRE: re(`(?m)^[^\n]+$`), |
| branches: func(remote string) []string { |
| return []string{"hg", "branches", "-c", "-q"} |
| }, |
| branchRE: re(`(?m)^[^\n]+$`), |
| badLocalRevRE: re(`(?m)^(tip)$`), |
| statLocal: func(rev, remote string) []string { |
| return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"} |
| }, |
| parseStat: hgParseStat, |
| fetch: []string{"hg", "pull", "-f"}, |
| latest: "tip", |
| readFile: func(rev, file, remote string) []string { |
| return []string{"hg", "cat", "-r", rev, file} |
| }, |
| readZip: func(rev, subdir, remote, target string) []string { |
| pattern := []string{} |
| if subdir != "" { |
| pattern = []string{"-I", subdir + "/**"} |
| } |
| return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, target) |
| }, |
| }, |
| |
| "svn": { |
| vcs: "svn", |
| init: nil, // no local checkout |
| tags: func(remote string) []string { |
| return []string{"svn", "list", strings.TrimSuffix(remote, "/trunk") + "/tags"} |
| }, |
| tagRE: re(`(?m)^(.*?)/?$`), |
| statLocal: func(rev, remote string) []string { |
| suffix := "@" + rev |
| if rev == "latest" { |
| suffix = "" |
| } |
| return []string{"svn", "log", "-l1", "--xml", remote + suffix} |
| }, |
| parseStat: svnParseStat, |
| latest: "latest", |
| readFile: func(rev, file, remote string) []string { |
| return []string{"svn", "cat", remote + "/" + file + "@" + rev} |
| }, |
| // TODO: zip |
| }, |
| |
| "bzr": { |
| vcs: "bzr", |
| init: func(remote string) []string { |
| return []string{"bzr", "branch", "--use-existing-dir", remote, "."} |
| }, |
| fetch: []string{ |
| "bzr", "pull", "--overwrite-tags", |
| }, |
| tags: func(remote string) []string { |
| return []string{"bzr", "tags"} |
| }, |
| tagRE: re(`(?m)^\S+`), |
| badLocalRevRE: re(`^revno:-`), |
| statLocal: func(rev, remote string) []string { |
| return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev} |
| }, |
| parseStat: bzrParseStat, |
| latest: "revno:-1", |
| readFile: func(rev, file, remote string) []string { |
| return []string{"bzr", "cat", "-r", rev, file} |
| }, |
| readZip: func(rev, subdir, remote, target string) []string { |
| extra := []string{} |
| if subdir != "" { |
| extra = []string{"./" + subdir} |
| } |
| return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", target, extra) |
| }, |
| }, |
| |
| "fossil": { |
| vcs: "fossil", |
| init: func(remote string) []string { |
| return []string{"fossil", "clone", remote, ".fossil"} |
| }, |
| fetch: []string{"fossil", "pull", "-R", ".fossil"}, |
| tags: func(remote string) []string { |
| return []string{"fossil", "tag", "-R", ".fossil", "list"} |
| }, |
| tagRE: re(`XXXTODO`), |
| statLocal: func(rev, remote string) []string { |
| return []string{"fossil", "info", "-R", ".fossil", rev} |
| }, |
| parseStat: fossilParseStat, |
| latest: "trunk", |
| readFile: func(rev, file, remote string) []string { |
| return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file} |
| }, |
| readZip: func(rev, subdir, remote, target string) []string { |
| extra := []string{} |
| if subdir != "" && !strings.ContainsAny(subdir, "*?[],") { |
| extra = []string{"--include", subdir} |
| } |
| // Note that vcsRepo.ReadZip below rewrites this command |
| // to run in a different directory, to work around a fossil bug. |
| return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, rev, target) |
| }, |
| }, |
| } |
| |
| func (r *vcsRepo) loadTags() { |
| out, err := Run(r.dir, r.cmd.tags(r.remote)) |
| if err != nil { |
| return |
| } |
| |
| // Run tag-listing command and extract tags. |
| r.tags = make(map[string]bool) |
| for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) { |
| if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) { |
| continue |
| } |
| r.tags[tag] = true |
| } |
| } |
| |
| func (r *vcsRepo) loadBranches() { |
| if r.cmd.branches == nil { |
| return |
| } |
| |
| out, err := Run(r.dir, r.cmd.branches(r.remote)) |
| if err != nil { |
| return |
| } |
| |
| r.branches = make(map[string]bool) |
| for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) { |
| if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) { |
| continue |
| } |
| r.branches[branch] = true |
| } |
| } |
| |
| func (r *vcsRepo) Tags(prefix string) ([]string, error) { |
| r.tagsOnce.Do(r.loadTags) |
| |
| tags := []string{} |
| for tag := range r.tags { |
| if strings.HasPrefix(tag, prefix) { |
| tags = append(tags, tag) |
| } |
| } |
| sort.Strings(tags) |
| return tags, nil |
| } |
| |
| func (r *vcsRepo) Stat(rev string) (*RevInfo, error) { |
| if rev == "latest" { |
| rev = r.cmd.latest |
| } |
| r.branchesOnce.Do(r.loadBranches) |
| revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev] |
| if revOK { |
| if info, err := r.statLocal(rev); err == nil { |
| return info, nil |
| } |
| } |
| |
| r.fetchOnce.Do(r.fetch) |
| if r.fetchErr != nil { |
| return nil, r.fetchErr |
| } |
| info, err := r.statLocal(rev) |
| if err != nil { |
| return nil, err |
| } |
| if !revOK { |
| info.Version = info.Name |
| } |
| return info, nil |
| } |
| |
| func (r *vcsRepo) fetch() { |
| _, r.fetchErr = Run(r.dir, r.cmd.fetch) |
| } |
| |
| func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) { |
| out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote)) |
| if err != nil { |
| return nil, fmt.Errorf("unknown revision %s", rev) |
| } |
| return r.cmd.parseStat(rev, string(out)) |
| } |
| |
| func (r *vcsRepo) Latest() (*RevInfo, error) { |
| return r.Stat("latest") |
| } |
| |
| func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { |
| if rev == "latest" { |
| rev = r.cmd.latest |
| } |
| _, err := r.Stat(rev) // download rev into local repo |
| if err != nil { |
| return nil, err |
| } |
| out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote)) |
| if err != nil { |
| return nil, os.ErrNotExist |
| } |
| return out, nil |
| } |
| |
| func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) { |
| if rev == "latest" { |
| rev = r.cmd.latest |
| } |
| f, err := ioutil.TempFile("", "go-readzip-*.zip") |
| if err != nil { |
| return nil, "", err |
| } |
| if r.cmd.vcs == "fossil" { |
| // If you run |
| // fossil zip -R .fossil --name prefix trunk /tmp/x.zip |
| // fossil fails with "unable to create directory /tmp" [sic]. |
| // Change the command to run in /tmp instead, |
| // replacing the -R argument with an absolute path. |
| args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name())) |
| for i := range args { |
| if args[i] == ".fossil" { |
| args[i] = filepath.Join(r.dir, ".fossil") |
| } |
| } |
| _, err = Run(filepath.Dir(f.Name()), args) |
| } else { |
| _, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name())) |
| } |
| if err != nil { |
| f.Close() |
| os.Remove(f.Name()) |
| return nil, "", err |
| } |
| return &deleteCloser{f}, "", nil |
| } |
| |
| // deleteCloser is a file that gets deleted on Close. |
| type deleteCloser struct { |
| *os.File |
| } |
| |
| func (d *deleteCloser) Close() error { |
| defer os.Remove(d.File.Name()) |
| return d.File.Close() |
| } |
| |
| func hgParseStat(rev, out string) (*RevInfo, error) { |
| f := strings.Fields(string(out)) |
| if len(f) < 3 { |
| return nil, fmt.Errorf("unexpected response from hg log: %q", out) |
| } |
| hash := f[0] |
| version := rev |
| if strings.HasPrefix(hash, version) { |
| version = hash // extend to full hash |
| } |
| t, err := strconv.ParseInt(f[1], 10, 64) |
| if err != nil { |
| return nil, fmt.Errorf("invalid time from hg log: %q", out) |
| } |
| |
| var tags []string |
| for _, tag := range f[3:] { |
| if tag != "tip" { |
| tags = append(tags, tag) |
| } |
| } |
| sort.Strings(tags) |
| |
| info := &RevInfo{ |
| Name: hash, |
| Short: ShortenSHA1(hash), |
| Time: time.Unix(t, 0).UTC(), |
| Version: version, |
| Tags: tags, |
| } |
| return info, nil |
| } |
| |
| func svnParseStat(rev, out string) (*RevInfo, error) { |
| var log struct { |
| Logentry struct { |
| Revision int64 `xml:"revision,attr"` |
| Date string `xml:"date"` |
| } `xml:"logentry"` |
| } |
| if err := xml.Unmarshal([]byte(out), &log); err != nil { |
| return nil, fmt.Errorf("unexpected response from svn log --xml: %v\n%s", err, out) |
| } |
| |
| t, err := time.Parse(time.RFC3339, log.Logentry.Date) |
| if err != nil { |
| return nil, fmt.Errorf("unexpected response from svn log --xml: %v\n%s", err, out) |
| } |
| |
| info := &RevInfo{ |
| Name: fmt.Sprintf("%d", log.Logentry.Revision), |
| Short: fmt.Sprintf("%012d", log.Logentry.Revision), |
| Time: t.UTC(), |
| Version: rev, |
| } |
| return info, nil |
| } |
| |
| func bzrParseStat(rev, out string) (*RevInfo, error) { |
| var revno int64 |
| var tm time.Time |
| for _, line := range strings.Split(out, "\n") { |
| if line == "" || line[0] == ' ' || line[0] == '\t' { |
| // End of header, start of commit message. |
| break |
| } |
| if line[0] == '-' { |
| continue |
| } |
| i := strings.Index(line, ":") |
| if i < 0 { |
| // End of header, start of commit message. |
| break |
| } |
| key, val := line[:i], strings.TrimSpace(line[i+1:]) |
| switch key { |
| case "revno": |
| if j := strings.Index(val, " "); j >= 0 { |
| val = val[:j] |
| } |
| i, err := strconv.ParseInt(val, 10, 64) |
| if err != nil { |
| return nil, fmt.Errorf("unexpected revno from bzr log: %q", line) |
| } |
| revno = i |
| case "timestamp": |
| j := strings.Index(val, " ") |
| if j < 0 { |
| return nil, fmt.Errorf("unexpected timestamp from bzr log: %q", line) |
| } |
| t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:]) |
| if err != nil { |
| return nil, fmt.Errorf("unexpected timestamp from bzr log: %q", line) |
| } |
| tm = t.UTC() |
| } |
| } |
| if revno == 0 || tm.IsZero() { |
| return nil, fmt.Errorf("unexpected response from bzr log: %q", out) |
| } |
| |
| info := &RevInfo{ |
| Name: fmt.Sprintf("%d", revno), |
| Short: fmt.Sprintf("%012d", revno), |
| Time: tm, |
| Version: rev, |
| } |
| return info, nil |
| } |
| |
| func fossilParseStat(rev, out string) (*RevInfo, error) { |
| for _, line := range strings.Split(out, "\n") { |
| if strings.HasPrefix(line, "uuid:") { |
| f := strings.Fields(line) |
| if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" { |
| return nil, fmt.Errorf("unexpected response from fossil info: %q", line) |
| } |
| t, err := time.Parse("2006-01-02 15:04:05", f[2]+" "+f[3]) |
| if err != nil { |
| return nil, fmt.Errorf("unexpected response from fossil info: %q", line) |
| } |
| hash := f[1] |
| version := rev |
| if strings.HasPrefix(hash, version) { |
| version = hash // extend to full hash |
| } |
| info := &RevInfo{ |
| Name: hash, |
| Short: ShortenSHA1(hash), |
| Time: t, |
| Version: version, |
| } |
| return info, nil |
| } |
| } |
| return nil, fmt.Errorf("unexpected response from fossil info: %q", out) |
| } |