Add gosrc
Move github.com/garyburd/gosrc to this repo.
diff --git a/gosrc/LICENSE b/gosrc/LICENSE
new file mode 100644
index 0000000..65d761b
--- /dev/null
+++ b/gosrc/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2013 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/gosrc/README.markdown b/gosrc/README.markdown
new file mode 100644
index 0000000..7dc2a8a
--- /dev/null
+++ b/gosrc/README.markdown
@@ -0,0 +1,9 @@
+Package gosrc fetches Go package source code from version control services.
+
+Contributions
+-------------
+Contributions to this project are welcome, though please send mail before
+starting work on anything major. Contributors retain their copyright, so we
+need you to fill out a short form before we can accept your contribution:
+https://developers.google.com/open-source/cla/individual
+
diff --git a/gosrc/bitbucket.go b/gosrc/bitbucket.go
new file mode 100644
index 0000000..b1da770
--- /dev/null
+++ b/gosrc/bitbucket.go
@@ -0,0 +1,102 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "net/http"
+ "path"
+ "regexp"
+)
+
+func init() {
+ addService(&service{
+ pattern: regexp.MustCompile(`^bitbucket\.org/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`),
+ prefix: "bitbucket.org/",
+ get: getBitbucketDir,
+ })
+}
+
+var bitbucketEtagRe = regexp.MustCompile(`^(hg|git)-`)
+
+func getBitbucketDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+
+ c := &httpClient{client: client}
+
+ if m := bitbucketEtagRe.FindStringSubmatch(savedEtag); m != nil {
+ match["vcs"] = m[1]
+ } else {
+ var repo struct {
+ Scm string
+ }
+ if err := c.getJSON(expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil {
+ return nil, err
+ }
+ match["vcs"] = repo.Scm
+ }
+
+ tags := make(map[string]string)
+ for _, nodeType := range []string{"branches", "tags"} {
+ var nodes map[string]struct {
+ Node string
+ }
+ if err := c.getJSON(expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil {
+ return nil, err
+ }
+ for t, n := range nodes {
+ tags[t] = n.Node
+ }
+ }
+
+ var err error
+ match["tag"], match["commit"], err = bestTag(tags, defaultTags[match["vcs"]])
+ if err != nil {
+ return nil, err
+ }
+
+ etag := expand("{vcs}-{commit}", match)
+ if etag == savedEtag {
+ return nil, ErrNotModified
+ }
+
+ var contents struct {
+ Directories []string
+ Files []struct {
+ Path string
+ }
+ }
+
+ if err := c.getJSON(expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), &contents); err != nil {
+ return nil, err
+ }
+
+ var files []*File
+ var dataURLs []string
+
+ for _, f := range contents.Files {
+ _, name := path.Split(f.Path)
+ if isDocFile(name) {
+ files = append(files, &File{Name: name, BrowseURL: expand("https://bitbucket.org/{owner}/{repo}/src/{tag}/{0}", match, f.Path)})
+ dataURLs = append(dataURLs, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/raw/{tag}/{0}", match, f.Path))
+ }
+ }
+
+ if err := c.getFiles(dataURLs, files); err != nil {
+ return nil, err
+ }
+
+ return &Directory{
+ BrowseURL: expand("https://bitbucket.org/{owner}/{repo}/src/{tag}{dir}", match),
+ Etag: etag,
+ Files: files,
+ LineFmt: "%s#cl-%d",
+ ProjectName: match["repo"],
+ ProjectRoot: expand("bitbucket.org/{owner}/{repo}", match),
+ ProjectURL: expand("https://bitbucket.org/{owner}/{repo}/", match),
+ Subdirectories: contents.Directories,
+ VCS: match["vcs"],
+ }, nil
+}
diff --git a/gosrc/build.go b/gosrc/build.go
new file mode 100644
index 0000000..5ffc969
--- /dev/null
+++ b/gosrc/build.go
@@ -0,0 +1,62 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "bytes"
+ "go/build"
+ "io"
+ "io/ioutil"
+ "os"
+ "path"
+ "strings"
+ "time"
+)
+
+// Import returns details about the package in the directory.
+func (dir *Directory) Import(ctx *build.Context, mode build.ImportMode) (*build.Package, error) {
+ safeCopy := *ctx
+ ctx = &safeCopy
+ ctx.JoinPath = path.Join
+ ctx.IsAbsPath = path.IsAbs
+ ctx.SplitPathList = func(list string) []string { return strings.Split(list, ":") }
+ ctx.IsDir = func(path string) bool { return false }
+ ctx.HasSubdir = func(root, dir string) (rel string, ok bool) { return "", false }
+ ctx.ReadDir = dir.readDir
+ ctx.OpenFile = dir.openFile
+ return ctx.ImportDir(".", mode)
+}
+
+type fileInfo struct{ f *File }
+
+func (fi fileInfo) Name() string { return fi.f.Name }
+func (fi fileInfo) Size() int64 { return int64(len(fi.f.Data)) }
+func (fi fileInfo) Mode() os.FileMode { return 0 }
+func (fi fileInfo) ModTime() time.Time { return time.Time{} }
+func (fi fileInfo) IsDir() bool { return false }
+func (fi fileInfo) Sys() interface{} { return nil }
+
+func (dir *Directory) readDir(name string) ([]os.FileInfo, error) {
+ if name != "." {
+ return nil, os.ErrNotExist
+ }
+ fis := make([]os.FileInfo, len(dir.Files))
+ for i, f := range dir.Files {
+ fis[i] = fileInfo{f}
+ }
+ return fis, nil
+}
+
+func (dir *Directory) openFile(path string) (io.ReadCloser, error) {
+ name := strings.TrimPrefix(path, "./")
+ for _, f := range dir.Files {
+ if f.Name == name {
+ return ioutil.NopCloser(bytes.NewReader(f.Data)), nil
+ }
+ }
+ return nil, os.ErrNotExist
+}
diff --git a/gosrc/client.go b/gosrc/client.go
new file mode 100644
index 0000000..c025e4d
--- /dev/null
+++ b/gosrc/client.go
@@ -0,0 +1,124 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+)
+
+type httpClient struct {
+ errFn func(*http.Response) error
+ header http.Header
+ client *http.Client
+}
+
+func (c *httpClient) err(resp *http.Response) error {
+ if resp.StatusCode == 404 {
+ return NotFoundError{"Resource not found: " + resp.Request.URL.String()}
+ }
+ if c.errFn != nil {
+ return c.errFn(resp)
+ }
+ return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: (%s)", resp.StatusCode, resp.Request.URL.String())}
+}
+
+func (c *httpClient) get(url string) (*http.Response, error) {
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, err
+ }
+ for k, vs := range c.header {
+ req.Header[k] = vs
+ }
+ resp, err := c.client.Do(req)
+ if err != nil {
+ return nil, &RemoteError{req.URL.Host, err}
+ }
+ return resp, err
+}
+
+func (c *httpClient) getBytes(url string) ([]byte, error) {
+ resp, err := c.get(url)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, c.err(resp)
+ }
+ p, err := ioutil.ReadAll(resp.Body)
+ resp.Body.Close()
+ return p, err
+}
+
+func (c *httpClient) getReader(url string) (io.ReadCloser, error) {
+ resp, err := c.get(url)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, c.err(resp)
+ }
+ return resp.Body, nil
+}
+
+func (c *httpClient) getJSON(url string, v interface{}) error {
+ resp, err := c.get(url)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return c.err(resp)
+ }
+ err = json.NewDecoder(resp.Body).Decode(v)
+ if _, ok := err.(*json.SyntaxError); ok {
+ err = NotFoundError{"JSON syntax error at " + url}
+ }
+ return err
+}
+
+func (c *httpClient) getFiles(urls []string, files []*File) error {
+ ch := make(chan error, len(files))
+ for i := range files {
+ go func(i int) {
+ resp, err := c.get(urls[i])
+ if err != nil {
+ ch <- err
+ return
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ var err error
+ if c.errFn != nil {
+ err = c.errFn(resp)
+ } else {
+ err = &RemoteError{resp.Request.URL.Host, fmt.Errorf("get %s -> %d", urls[i], resp.StatusCode)}
+ }
+ ch <- err
+ return
+ }
+ files[i].Data, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ ch <- &RemoteError{resp.Request.URL.Host, err}
+ return
+ }
+ ch <- nil
+ }(i)
+ }
+ for _ = range files {
+ if err := <-ch; err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/gosrc/github.go b/gosrc/github.go
new file mode 100644
index 0000000..347a608
--- /dev/null
+++ b/gosrc/github.go
@@ -0,0 +1,296 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+ "time"
+)
+
+func init() {
+ addService(&service{
+ pattern: regexp.MustCompile(`^github\.com/(?P<owner>[a-z0-9A-Z_.\-]+)/(?P<repo>[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]*)?$`),
+ prefix: "github.com/",
+ get: getGitHubDir,
+ getPresentation: getGitHubPresentation,
+ getProject: getGitHubProject,
+ })
+
+ addService(&service{
+ pattern: regexp.MustCompile(`^gist\.github\.com/(?P<gist>[a-z0-9A-Z_.\-]+)\.git$`),
+ prefix: "gist.github.com/",
+ get: getGistDir,
+ })
+}
+
+var (
+ gitHubRawHeader = http.Header{"Accept": {"application/vnd.github-blob.raw"}}
+ gitHubPreviewHeader = http.Header{"Accept": {"application/vnd.github.preview"}}
+)
+
+func gitHubError(resp *http.Response) error {
+ var e struct {
+ Message string `json:"message"`
+ }
+ if err := json.NewDecoder(resp.Body).Decode(&e); err == nil {
+ return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: %s (%s)", resp.StatusCode, e.Message, resp.Request.URL.String())}
+ }
+ return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: (%s)", resp.StatusCode, resp.Request.URL.String())}
+}
+
+func getGitHubDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+
+ c := &httpClient{client: client, errFn: gitHubError}
+
+ var refs []*struct {
+ Object struct {
+ Type string
+ Sha string
+ URL string
+ }
+ Ref string
+ URL string
+ }
+
+ if err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}/git/refs", match), &refs); err != nil {
+ return nil, err
+ }
+
+ tags := make(map[string]string)
+ for _, ref := range refs {
+ switch {
+ case strings.HasPrefix(ref.Ref, "refs/heads/"):
+ tags[ref.Ref[len("refs/heads/"):]] = ref.Object.Sha
+ case strings.HasPrefix(ref.Ref, "refs/tags/"):
+ tags[ref.Ref[len("refs/tags/"):]] = ref.Object.Sha
+ }
+ }
+
+ var commit string
+ var err error
+ match["tag"], commit, err = bestTag(tags, "master")
+ if err != nil {
+ return nil, err
+ }
+
+ if commit == savedEtag {
+ return nil, ErrNotModified
+ }
+
+ var contents []*struct {
+ Type string
+ Name string
+ GitURL string `json:"git_url"`
+ HTMLURL string `json:"html_url"`
+ }
+
+ if err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}?ref={tag}", match), &contents); err != nil {
+ return nil, err
+ }
+
+ if len(contents) == 0 {
+ return nil, NotFoundError{"No files in directory."}
+ }
+
+ // Because Github API URLs are case-insensitive, we check that the owner
+ // and repo returned from Github matches the one that we are requesting.
+ if !strings.HasPrefix(contents[0].GitURL, expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
+ return nil, NotFoundError{"Github import path has incorrect case."}
+ }
+
+ var files []*File
+ var dataURLs []string
+ var subdirs []string
+
+ for _, item := range contents {
+ switch {
+ case item.Type == "dir":
+ if isValidPathElement(item.Name) {
+ subdirs = append(subdirs, item.Name)
+ }
+ case isDocFile(item.Name):
+ files = append(files, &File{Name: item.Name, BrowseURL: item.HTMLURL})
+ dataURLs = append(dataURLs, item.GitURL)
+ }
+ }
+
+ c.header = gitHubRawHeader
+ if err := c.getFiles(dataURLs, files); err != nil {
+ return nil, err
+ }
+
+ browseURL := expand("https://github.com/{owner}/{repo}", match)
+ if match["dir"] != "" {
+ browseURL = expand("https://github.com/{owner}/{repo}/tree/{tag}{dir}", match)
+ }
+
+ return &Directory{
+ BrowseURL: browseURL,
+ Etag: commit,
+ Files: files,
+ LineFmt: "%s#L%d",
+ ProjectName: match["repo"],
+ ProjectRoot: expand("github.com/{owner}/{repo}", match),
+ ProjectURL: expand("https://github.com/{owner}/{repo}", match),
+ Subdirectories: subdirs,
+ VCS: "git",
+ }, nil
+}
+
+func getGitHubPresentation(client *http.Client, match map[string]string) (*Presentation, error) {
+ c := &httpClient{client: client, header: gitHubRawHeader}
+
+ p, err := c.getBytes(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}/{file}", match))
+ if err != nil {
+ return nil, err
+ }
+
+ apiBase, err := url.Parse(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}/", match))
+ if err != nil {
+ return nil, err
+ }
+ rawBase, err := url.Parse(expand("https://raw.github.com/{owner}/{repo}/master{dir}/", match))
+ if err != nil {
+ return nil, err
+ }
+
+ c.header = gitHubRawHeader
+
+ b := &presBuilder{
+ data: p,
+ filename: match["file"],
+ fetch: func(fnames []string) ([]*File, error) {
+ var files []*File
+ var dataURLs []string
+ for _, fname := range fnames {
+ u, err := apiBase.Parse(fname)
+ if err != nil {
+ return nil, err
+ }
+ u.RawQuery = apiBase.RawQuery
+ files = append(files, &File{Name: fname})
+ dataURLs = append(dataURLs, u.String())
+ }
+ err := c.getFiles(dataURLs, files)
+ return files, err
+ },
+ resolveURL: func(fname string) string {
+ u, err := rawBase.Parse(fname)
+ if err != nil {
+ return "/notfound"
+ }
+ if strings.HasSuffix(fname, ".svg") {
+ u.Host = "rawgithub.com"
+ }
+ return u.String()
+ },
+ }
+
+ return b.build()
+}
+
+// GetGitHubUpdates returns the full names ("owner/repo") of recently pushed GitHub repositories.
+// by pushedAfter.
+func GetGitHubUpdates(client *http.Client, pushedAfter string) (maxPushedAt string, names []string, err error) {
+ c := httpClient{client: client, header: gitHubPreviewHeader}
+
+ if pushedAfter == "" {
+ pushedAfter = time.Now().Add(-24 * time.Hour).UTC().Format("2006-01-02T15:04:05Z")
+ }
+ u := "https://api.github.com/search/repositories?order=asc&sort=updated&q=fork:true+language:Go+pushed:>" + pushedAfter
+ var updates struct {
+ Items []struct {
+ FullName string `json:"full_name"`
+ PushedAt string `json:"pushed_at"`
+ }
+ }
+ err = c.getJSON(u, &updates)
+ if err != nil {
+ return pushedAfter, nil, err
+ }
+
+ maxPushedAt = pushedAfter
+ for _, item := range updates.Items {
+ names = append(names, item.FullName)
+ if item.PushedAt > maxPushedAt {
+ maxPushedAt = item.PushedAt
+ }
+ }
+ return maxPushedAt, names, nil
+}
+
+func getGitHubProject(client *http.Client, match map[string]string) (*Project, error) {
+ c := &httpClient{client: client, errFn: gitHubError}
+
+ var repo struct {
+ Description string
+ }
+
+ if err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
+ return nil, err
+ }
+
+ return &Project{
+ Description: repo.Description,
+ }, nil
+}
+
+func getGistDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+ c := &httpClient{client: client, errFn: gitHubError}
+
+ var gist struct {
+ Files map[string]struct {
+ Content string
+ }
+ HtmlUrl string `json:"html_url"`
+ History []struct {
+ Version string
+ }
+ }
+
+ if err := c.getJSON(expand("https://api.github.com/gists/{gist}", match), &gist); err != nil {
+ return nil, err
+ }
+
+ if len(gist.History) == 0 {
+ return nil, NotFoundError{"History not found."}
+ }
+ commit := gist.History[0].Version
+
+ if commit == savedEtag {
+ return nil, ErrNotModified
+ }
+
+ var files []*File
+
+ for name, file := range gist.Files {
+ if isDocFile(name) {
+ files = append(files, &File{
+ Name: name,
+ Data: []byte(file.Content),
+ BrowseURL: gist.HtmlUrl + "#file-" + strings.Replace(name, ".", "-", -1),
+ })
+ }
+ }
+
+ return &Directory{
+ BrowseURL: gist.HtmlUrl,
+ Etag: commit,
+ Files: files,
+ LineFmt: "%s-L%d",
+ ProjectName: match["gist"],
+ ProjectRoot: expand("gist.github.com/{gist}.git", match),
+ ProjectURL: gist.HtmlUrl,
+ Subdirectories: nil,
+ VCS: "git",
+ }, nil
+}
diff --git a/gosrc/google.go b/gosrc/google.go
new file mode 100644
index 0000000..8bd7cf3
--- /dev/null
+++ b/gosrc/google.go
@@ -0,0 +1,214 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "errors"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+)
+
+func init() {
+ addService(&service{
+ pattern: regexp.MustCompile(`^code\.google\.com/(?P<pr>[pr])/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`),
+ prefix: "code.google.com/",
+ get: getGoogleDir,
+ getPresentation: getGooglePresentation,
+ })
+}
+
+var (
+ googleRepoRe = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)
+ googleRevisionRe = regexp.MustCompile(`<h2>(?:[^ ]+ - )?Revision *([^:]+):`)
+ googleEtagRe = regexp.MustCompile(`^(hg|git|svn)-`)
+ googleFileRe = regexp.MustCompile(`<li><a href="([^"]+)"`)
+)
+
+func getGoogleDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+ c := &httpClient{client: client}
+
+ setupGoogleMatch(match)
+ if m := googleEtagRe.FindStringSubmatch(savedEtag); m != nil {
+ match["vcs"] = m[1]
+ } else if err := getGoogleVCS(c, match); err != nil {
+ return nil, err
+ }
+
+ // Scrape the repo browser to find the project revision and individual Go files.
+ p, err := c.getBytes(expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match))
+ if err != nil {
+ return nil, err
+ }
+
+ var etag string
+ m := googleRevisionRe.FindSubmatch(p)
+ if m == nil {
+ return nil, errors.New("Could not find revision for " + match["importPath"])
+ }
+ etag = expand("{vcs}-{0}", match, string(m[1]))
+ if etag == savedEtag {
+ return nil, ErrNotModified
+ }
+
+ var subdirs []string
+ var files []*File
+ var dataURLs []string
+ for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+ fname := string(m[1])
+ switch {
+ case strings.HasSuffix(fname, "/"):
+ fname = fname[:len(fname)-1]
+ if isValidPathElement(fname) {
+ subdirs = append(subdirs, fname)
+ }
+ case isDocFile(fname):
+ files = append(files, &File{Name: fname, BrowseURL: expand("http://code.google.com/{pr}/{repo}/source/browse{dir}/{0}{query}", match, fname)})
+ dataURLs = append(dataURLs, expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname))
+ }
+ }
+
+ if err := c.getFiles(dataURLs, files); err != nil {
+ return nil, err
+ }
+
+ var projectURL string
+ if match["subrepo"] == "" {
+ projectURL = expand("https://code.google.com/{pr}/{repo}/", match)
+ } else {
+ projectURL = expand("https://code.google.com/{pr}/{repo}/source/browse?repo={subrepo}", match)
+ }
+
+ return &Directory{
+ BrowseURL: expand("http://code.google.com/{pr}/{repo}/source/browse{dir}/{query}", match),
+ Etag: etag,
+ Files: files,
+ LineFmt: "%s#%d",
+ ProjectName: expand("{repo}{dot}{subrepo}", match),
+ ProjectRoot: expand("code.google.com/{pr}/{repo}{dot}{subrepo}", match),
+ ProjectURL: projectURL,
+ VCS: match["vcs"],
+ }, nil
+}
+
+func setupGoogleMatch(match map[string]string) {
+ if s := match["subrepo"]; s != "" {
+ match["dot"] = "."
+ match["query"] = "?repo=" + s
+ } else {
+ match["dot"] = ""
+ match["query"] = ""
+ }
+}
+
+func getGoogleVCS(c *httpClient, match map[string]string) error {
+ // Scrape the HTML project page to find the VCS.
+ p, err := c.getBytes(expand("http://code.google.com/{pr}/{repo}/source/checkout", match))
+ if err != nil {
+ return err
+ }
+ m := googleRepoRe.FindSubmatch(p)
+ if m == nil {
+ return NotFoundError{"Could not VCS on Google Code project page."}
+ }
+ match["vcs"] = string(m[1])
+ return nil
+}
+
+func getStandardDir(client *http.Client, importPath string, savedEtag string) (*Directory, error) {
+ c := &httpClient{client: client}
+
+ p, err := c.getBytes("http://go.googlecode.com/hg-history/release/src/pkg/" + importPath + "/")
+ if err != nil {
+ return nil, err
+ }
+
+ var etag string
+ m := googleRevisionRe.FindSubmatch(p)
+ if m == nil {
+ return nil, errors.New("Could not find revision for " + importPath)
+ }
+ etag = string(m[1])
+ if etag == savedEtag {
+ return nil, ErrNotModified
+ }
+
+ var files []*File
+ var dataURLs []string
+ for _, m := range googleFileRe.FindAllSubmatch(p, -1) {
+ fname := strings.Split(string(m[1]), "?")[0]
+ if isDocFile(fname) {
+ files = append(files, &File{Name: fname, BrowseURL: "http://code.google.com/p/go/source/browse/src/pkg/" + importPath + "/" + fname + "?name=release"})
+ dataURLs = append(dataURLs, "http://go.googlecode.com/hg-history/release/src/pkg/"+importPath+"/"+fname)
+ }
+ }
+
+ if err := c.getFiles(dataURLs, files); err != nil {
+ return nil, err
+ }
+
+ return &Directory{
+ BrowseURL: "http://code.google.com/p/go/source/browse/src/pkg/" + importPath + "?name=release",
+ Etag: etag,
+ Files: files,
+ ImportPath: importPath,
+ LineFmt: "%s#%d",
+ ProjectName: "Go",
+ ProjectRoot: "",
+ ProjectURL: "https://code.google.com/p/go/",
+ ResolvedPath: importPath,
+ VCS: "hg",
+ }, nil
+}
+
+func getGooglePresentation(client *http.Client, match map[string]string) (*Presentation, error) {
+ c := &httpClient{client: client}
+
+ setupGoogleMatch(match)
+ if err := getGoogleVCS(c, match); err != nil {
+ return nil, err
+ }
+
+ rawBase, err := url.Parse(expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match))
+ if err != nil {
+ return nil, err
+ }
+
+ p, err := c.getBytes(expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{file}", match))
+ if err != nil {
+ return nil, err
+ }
+
+ b := &presBuilder{
+ data: p,
+ filename: match["file"],
+ fetch: func(fnames []string) ([]*File, error) {
+ var files []*File
+ var dataURLs []string
+ for _, fname := range fnames {
+ u, err := rawBase.Parse(fname)
+ if err != nil {
+ return nil, err
+ }
+ files = append(files, &File{Name: fname})
+ dataURLs = append(dataURLs, u.String())
+ }
+ err := c.getFiles(dataURLs, files)
+ return files, err
+ },
+ resolveURL: func(fname string) string {
+ u, err := rawBase.Parse(fname)
+ if err != nil {
+ return "/notfound"
+ }
+ return u.String()
+ },
+ }
+
+ return b.build()
+}
diff --git a/gosrc/gosrc.go b/gosrc/gosrc.go
new file mode 100644
index 0000000..c1f5aab
--- /dev/null
+++ b/gosrc/gosrc.go
@@ -0,0 +1,371 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+// Package gosrc fetches Go package source code from version control services.
+package gosrc
+
+import (
+ "encoding/xml"
+ "errors"
+ "io"
+ "net/http"
+ "path"
+ "regexp"
+ "strings"
+)
+
+// File represents a file.
+type File struct {
+ // File name with no directory.
+ Name string
+
+ // Contents of the file.
+ Data []byte
+
+ // Location of file on version control service website.
+ BrowseURL string
+}
+
+// Directory describes a directory on a version control service.
+type Directory struct {
+ // The import path for this package.
+ ImportPath string
+
+ // Import path of package after resolving go-import meta tags, if any.
+ ResolvedPath string
+
+ // Import path prefix for all packages in the project.
+ ProjectRoot string
+
+ // Name of the project.
+ ProjectName string
+
+ // Project home page.
+ ProjectURL string
+
+ // Version control system: git, hg, bzr, ...
+ VCS string
+
+ // Cache validation tag. This tag is not necessarily an HTTP entity tag.
+ // The tag is "" if there is no meaningful cache validation for the VCS.
+ Etag string
+
+ // Files.
+ Files []*File
+
+ // Subdirectories, not guaranteed to contain Go code.
+ Subdirectories []string
+
+ // Location of directory on version control service website.
+ BrowseURL string
+
+ // Format specifier for link to source line. Example: "%s#L%d"
+ LineFmt string
+}
+
+// Project represents a repository.
+type Project struct {
+ Description string
+}
+
+// NotFoundError indicates that the directory or presentation was not found.
+type NotFoundError struct {
+ // Diagnostic message describing why the directory was not found.
+ Message string
+}
+
+func (e NotFoundError) Error() string {
+ return e.Message
+}
+
+// IsNotFound returns true if err is of type NotFoundError.
+func IsNotFound(err error) bool {
+ _, ok := err.(NotFoundError)
+ return ok
+}
+
+type RemoteError struct {
+ Host string
+ err error
+}
+
+func (e *RemoteError) Error() string {
+ return e.err.Error()
+}
+
+// ErrNotModified indicates that the directory matches the specified etag.
+var ErrNotModified = errors.New("package not modified")
+
+var errNoMatch = errors.New("no match")
+
+// service represents a source code control service.
+type service struct {
+ pattern *regexp.Regexp
+ prefix string
+ get func(*http.Client, map[string]string, string) (*Directory, error)
+ getPresentation func(*http.Client, map[string]string) (*Presentation, error)
+ getProject func(*http.Client, map[string]string) (*Project, error)
+}
+
+var services []*service
+
+func addService(s *service) {
+ if s.prefix == "" {
+ services = append(services, s)
+ } else {
+ services = append([]*service{s}, services...)
+ }
+}
+
+func (s *service) match(importPath string) (map[string]string, error) {
+ if !strings.HasPrefix(importPath, s.prefix) {
+ return nil, nil
+ }
+ m := s.pattern.FindStringSubmatch(importPath)
+ if m == nil {
+ if s.prefix != "" {
+ return nil, NotFoundError{"Import path prefix matches known service, but regexp does not."}
+ }
+ return nil, nil
+ }
+ match := map[string]string{"importPath": importPath}
+ for i, n := range s.pattern.SubexpNames() {
+ if n != "" {
+ match[n] = m[i]
+ }
+ }
+ return match, nil
+}
+
+func attrValue(attrs []xml.Attr, name string) string {
+ for _, a := range attrs {
+ if strings.EqualFold(a.Name.Local, name) {
+ return a.Value
+ }
+ }
+ return ""
+}
+
+func fetchMeta(client *http.Client, importPath string) (map[string]string, error) {
+ uri := importPath
+ if !strings.Contains(uri, "/") {
+ // Add slash for root of domain.
+ uri = uri + "/"
+ }
+ uri = uri + "?go-get=1"
+
+ c := httpClient{client: client}
+ scheme := "https"
+ resp, err := c.get(scheme + "://" + uri)
+ if err != nil || resp.StatusCode != 200 {
+ if err == nil {
+ resp.Body.Close()
+ }
+ scheme = "http"
+ resp, err = c.get(scheme + "://" + uri)
+ if err != nil {
+ return nil, err
+ }
+ }
+ defer resp.Body.Close()
+ return parseMeta(scheme, importPath, resp.Body)
+}
+
+func parseMeta(scheme, importPath string, r io.Reader) (map[string]string, error) {
+ var match map[string]string
+
+ d := xml.NewDecoder(r)
+ d.Strict = false
+metaScan:
+ for {
+ t, tokenErr := d.Token()
+ if tokenErr != nil {
+ break metaScan
+ }
+ switch t := t.(type) {
+ case xml.EndElement:
+ if strings.EqualFold(t.Name.Local, "head") {
+ break metaScan
+ }
+ case xml.StartElement:
+ if strings.EqualFold(t.Name.Local, "body") {
+ break metaScan
+ }
+ if !strings.EqualFold(t.Name.Local, "meta") ||
+ attrValue(t.Attr, "name") != "go-import" {
+ continue metaScan
+ }
+ f := strings.Fields(attrValue(t.Attr, "content"))
+ if len(f) != 3 ||
+ !strings.HasPrefix(importPath, f[0]) ||
+ !(len(importPath) == len(f[0]) || importPath[len(f[0])] == '/') {
+ continue metaScan
+ }
+ if match != nil {
+ return nil, NotFoundError{"More than one <meta> found at " + scheme + "://" + importPath}
+ }
+
+ projectRoot, vcs, repo := f[0], f[1], f[2]
+
+ repo = strings.TrimSuffix(repo, "."+vcs)
+ i := strings.Index(repo, "://")
+ if i < 0 {
+ return nil, NotFoundError{"Bad repo URL in <meta>."}
+ }
+ proto := repo[:i]
+ repo = repo[i+len("://"):]
+
+ match = map[string]string{
+ // Used in getVCSDoc, same as vcsPattern matches.
+ "importPath": importPath,
+ "repo": repo,
+ "vcs": vcs,
+ "dir": importPath[len(projectRoot):],
+
+ // Used in getVCSDoc
+ "scheme": proto,
+
+ // Used in getDynamic.
+ "projectRoot": projectRoot,
+ "projectName": path.Base(projectRoot),
+ "projectURL": scheme + "://" + projectRoot,
+ }
+ }
+ }
+ if match == nil {
+ return nil, NotFoundError{"<meta> not found."}
+ }
+ return match, nil
+}
+
+var getVCSDirFn = func(client *http.Client, m map[string]string, etag string) (*Directory, error) {
+ return nil, errNoMatch
+}
+
+// getDynamic gets a directory from a service that is not statically known.
+func getDynamic(client *http.Client, importPath, etag string) (*Directory, error) {
+ match, err := fetchMeta(client, importPath)
+ if err != nil {
+ return nil, err
+ }
+
+ if match["projectRoot"] != importPath {
+ rootMatch, err := fetchMeta(client, match["projectRoot"])
+ if err != nil {
+ return nil, err
+ }
+ if rootMatch["projectRoot"] != match["projectRoot"] {
+ return nil, NotFoundError{"Project root mismatch."}
+ }
+ }
+
+ dir, err := getStatic(client, expand("{repo}{dir}", match), etag)
+ if err == errNoMatch {
+ dir, err = getVCSDirFn(client, match, etag)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if dir != nil {
+ dir.ImportPath = importPath
+ dir.ProjectRoot = match["projectRoot"]
+ dir.ProjectName = match["projectName"]
+ dir.ProjectURL = match["projectURL"]
+ if dir.ResolvedPath == "" {
+ dir.ResolvedPath = dir.ImportPath
+ }
+ }
+
+ return dir, err
+}
+
+// getStatic gets a diretory from a statically known service. getStatic
+// returns errNoMatch if the import path is not recognized.
+func getStatic(client *http.Client, importPath, etag string) (*Directory, error) {
+ for _, s := range services {
+ if s.get == nil {
+ continue
+ }
+ match, err := s.match(importPath)
+ if err != nil {
+ return nil, err
+ }
+ if match != nil {
+ dir, err := s.get(client, match, etag)
+ if dir != nil {
+ dir.ImportPath = importPath
+ dir.ResolvedPath = importPath
+ }
+ return dir, err
+ }
+ }
+ return nil, errNoMatch
+}
+
+func Get(client *http.Client, importPath string, etag string) (dir *Directory, err error) {
+ switch {
+ case localPath != "":
+ dir, err = getLocal(importPath)
+ case IsGoRepoPath(importPath):
+ dir, err = getStandardDir(client, importPath, etag)
+ case IsValidRemotePath(importPath):
+ dir, err = getStatic(client, importPath, etag)
+ if err == errNoMatch {
+ dir, err = getDynamic(client, importPath, etag)
+ }
+ default:
+ err = errNoMatch
+ }
+
+ if err == errNoMatch {
+ err = NotFoundError{"Import path not valid:"}
+ }
+
+ return dir, err
+}
+
+// GetPresentation gets a presentation from the the given path.
+func GetPresentation(client *http.Client, importPath string) (*Presentation, error) {
+ ext := path.Ext(importPath)
+ if ext != ".slide" && ext != ".article" {
+ return nil, NotFoundError{"unknown file extension."}
+ }
+
+ importPath, file := path.Split(importPath)
+ importPath = strings.TrimSuffix(importPath, "/")
+ for _, s := range services {
+ if s.getPresentation == nil {
+ continue
+ }
+ match, err := s.match(importPath)
+ if err != nil {
+ return nil, err
+ }
+ if match != nil {
+ match["file"] = file
+ return s.getPresentation(client, match)
+ }
+ }
+ return nil, NotFoundError{"path does not match registered service"}
+}
+
+// GetProject gets information about a repository.
+func GetProject(client *http.Client, importPath string) (*Project, error) {
+ for _, s := range services {
+ if s.getProject == nil {
+ continue
+ }
+ match, err := s.match(importPath)
+ if err != nil {
+ return nil, err
+ }
+ if match != nil {
+ return s.getProject(client, match)
+ }
+ }
+ return nil, NotFoundError{"path does not match registered service"}
+}
diff --git a/gosrc/launchpad.go b/gosrc/launchpad.go
new file mode 100644
index 0000000..d63f00d
--- /dev/null
+++ b/gosrc/launchpad.go
@@ -0,0 +1,135 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "crypto/md5"
+ "encoding/hex"
+ "io"
+ "net/http"
+ "path"
+ "regexp"
+ "sort"
+ "strings"
+)
+
+func init() {
+ addService(&service{
+ pattern: regexp.MustCompile(`^launchpad\.net/(?P<repo>(?P<project>[a-z0-9A-Z_.\-]+)(?P<series>/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]+)*$`),
+ prefix: "launchpad.net/",
+ get: getLaunchpadDir,
+ })
+}
+
+type byHash []byte
+
+func (p byHash) Len() int { return len(p) / md5.Size }
+func (p byHash) Less(i, j int) bool {
+ return -1 == bytes.Compare(p[i*md5.Size:(i+1)*md5.Size], p[j*md5.Size:(j+1)*md5.Size])
+}
+func (p byHash) Swap(i, j int) {
+ var temp [md5.Size]byte
+ copy(temp[:], p[i*md5.Size:])
+ copy(p[i*md5.Size:(i+1)*md5.Size], p[j*md5.Size:])
+ copy(p[j*md5.Size:], temp[:])
+}
+
+func getLaunchpadDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+ c := &httpClient{client: client}
+
+ if match["project"] != "" && match["series"] != "" {
+ rc, err := c.getReader(expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match))
+ switch {
+ case err == nil:
+ rc.Close()
+ // The structure of the import path is launchpad.net/{root}/{dir}.
+ case IsNotFound(err):
+ // The structure of the import path is is launchpad.net/{project}/{dir}.
+ match["repo"] = match["project"]
+ match["dir"] = expand("{series}{dir}", match)
+ default:
+ return nil, err
+ }
+ }
+
+ p, err := c.getBytes(expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match))
+ if err != nil {
+ return nil, err
+ }
+
+ gzr, err := gzip.NewReader(bytes.NewReader(p))
+ if err != nil {
+ return nil, err
+ }
+ defer gzr.Close()
+
+ tr := tar.NewReader(gzr)
+
+ var hash []byte
+ inTree := false
+ dirPrefix := expand("+branch/{repo}{dir}/", match)
+ var files []*File
+ for {
+ h, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ d, f := path.Split(h.Name)
+ if !isDocFile(f) {
+ continue
+ }
+ b := make([]byte, h.Size)
+ if _, err := io.ReadFull(tr, b); err != nil {
+ return nil, err
+ }
+
+ m := md5.New()
+ m.Write(b)
+ hash = m.Sum(hash)
+
+ if !strings.HasPrefix(h.Name, dirPrefix) {
+ continue
+ }
+ inTree = true
+ if d == dirPrefix {
+ files = append(files, &File{
+ Name: f,
+ BrowseURL: expand("http://bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/{0}", match, f),
+ Data: b})
+ }
+ }
+
+ if !inTree {
+ return nil, NotFoundError{"Directory tree does not contain Go files."}
+ }
+
+ sort.Sort(byHash(hash))
+ m := md5.New()
+ m.Write(hash)
+ hash = m.Sum(hash[:0])
+ etag := hex.EncodeToString(hash)
+ if etag == savedEtag {
+ return nil, ErrNotModified
+ }
+
+ return &Directory{
+ BrowseURL: expand("http://bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/", match),
+ Etag: etag,
+ Files: files,
+ LineFmt: "%s#L%d",
+ ProjectName: match["repo"],
+ ProjectRoot: expand("launchpad.net/{repo}", match),
+ ProjectURL: expand("https://launchpad.net/{repo}/", match),
+ VCS: "bzr",
+ }, nil
+}
diff --git a/gosrc/local.go b/gosrc/local.go
new file mode 100644
index 0000000..d9ae236
--- /dev/null
+++ b/gosrc/local.go
@@ -0,0 +1,63 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "go/build"
+ "io/ioutil"
+ "path/filepath"
+ "strconv"
+ "time"
+)
+
+var localPath string
+
+// SetLocalDevMode sets the package to local development mode. In this mode,
+// the GOPATH specified by path is used to find directories instead of version
+// control services.
+func SetLocalDevMode(path string) {
+ localPath = path
+}
+
+func getLocal(importPath string) (*Directory, error) {
+ ctx := build.Default
+ if localPath != "" {
+ ctx.GOPATH = localPath
+ }
+ bpkg, err := ctx.Import(importPath, ".", build.FindOnly)
+ if err != nil {
+ return nil, err
+ }
+ dir := filepath.Join(bpkg.SrcRoot, filepath.FromSlash(importPath))
+ fis, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+ var modTime time.Time
+ var files []*File
+ for _, fi := range fis {
+ if fi.IsDir() || !isDocFile(fi.Name()) {
+ continue
+ }
+ if fi.ModTime().After(modTime) {
+ modTime = fi.ModTime()
+ }
+ b, err := ioutil.ReadFile(filepath.Join(dir, fi.Name()))
+ if err != nil {
+ return nil, err
+ }
+ files = append(files, &File{
+ Name: fi.Name(),
+ Data: b,
+ })
+ }
+ return &Directory{
+ ImportPath: importPath,
+ Etag: strconv.FormatInt(modTime.Unix(), 16),
+ Files: files,
+ }, nil
+}
diff --git a/gosrc/path.go b/gosrc/path.go
new file mode 100644
index 0000000..2b14adf
--- /dev/null
+++ b/gosrc/path.go
@@ -0,0 +1,529 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "path"
+ "regexp"
+ "strings"
+)
+
+var standardPath = map[string]bool{
+ "builtin": true,
+
+ // go list -f '"{{.ImportPath}}": true,' std
+ "archive/tar": true,
+ "archive/zip": true,
+ "bufio": true,
+ "bytes": true,
+ "compress/bzip2": true,
+ "compress/flate": true,
+ "compress/gzip": true,
+ "compress/lzw": true,
+ "compress/zlib": true,
+ "container/heap": true,
+ "container/list": true,
+ "container/ring": true,
+ "crypto": true,
+ "crypto/aes": true,
+ "crypto/cipher": true,
+ "crypto/des": true,
+ "crypto/dsa": true,
+ "crypto/ecdsa": true,
+ "crypto/elliptic": true,
+ "crypto/hmac": true,
+ "crypto/md5": true,
+ "crypto/rand": true,
+ "crypto/rc4": true,
+ "crypto/rsa": true,
+ "crypto/sha1": true,
+ "crypto/sha256": true,
+ "crypto/sha512": true,
+ "crypto/subtle": true,
+ "crypto/tls": true,
+ "crypto/x509": true,
+ "crypto/x509/pkix": true,
+ "database/sql": true,
+ "database/sql/driver": true,
+ "debug/dwarf": true,
+ "debug/elf": true,
+ "debug/gosym": true,
+ "debug/macho": true,
+ "debug/pe": true,
+ "encoding": true,
+ "encoding/ascii85": true,
+ "encoding/asn1": true,
+ "encoding/base32": true,
+ "encoding/base64": true,
+ "encoding/binary": true,
+ "encoding/csv": true,
+ "encoding/gob": true,
+ "encoding/hex": true,
+ "encoding/json": true,
+ "encoding/pem": true,
+ "encoding/xml": true,
+ "errors": true,
+ "expvar": true,
+ "flag": true,
+ "fmt": true,
+ "go/ast": true,
+ "go/build": true,
+ "go/doc": true,
+ "go/format": true,
+ "go/parser": true,
+ "go/printer": true,
+ "go/scanner": true,
+ "go/token": true,
+ "hash": true,
+ "hash/adler32": true,
+ "hash/crc32": true,
+ "hash/crc64": true,
+ "hash/fnv": true,
+ "html": true,
+ "html/template": true,
+ "image": true,
+ "image/color": true,
+ "image/color/palette": true,
+ "image/draw": true,
+ "image/gif": true,
+ "image/jpeg": true,
+ "image/png": true,
+ "index/suffixarray": true,
+ "io": true,
+ "io/ioutil": true,
+ "log": true,
+ "log/syslog": true,
+ "math": true,
+ "math/big": true,
+ "math/cmplx": true,
+ "math/rand": true,
+ "mime": true,
+ "mime/multipart": true,
+ "net": true,
+ "net/http": true,
+ "net/http/cgi": true,
+ "net/http/cookiejar": true,
+ "net/http/fcgi": true,
+ "net/http/httptest": true,
+ "net/http/httputil": true,
+ "net/http/pprof": true,
+ "net/mail": true,
+ "net/rpc": true,
+ "net/rpc/jsonrpc": true,
+ "net/smtp": true,
+ "net/textproto": true,
+ "net/url": true,
+ "os": true,
+ "os/exec": true,
+ "os/signal": true,
+ "os/user": true,
+ "path": true,
+ "path/filepath": true,
+ "reflect": true,
+ "regexp": true,
+ "regexp/syntax": true,
+ "runtime": true,
+ "runtime/cgo": true,
+ "runtime/debug": true,
+ "runtime/pprof": true,
+ "runtime/race": true,
+ "sort": true,
+ "strconv": true,
+ "strings": true,
+ "sync": true,
+ "sync/atomic": true,
+ "syscall": true,
+ "testing": true,
+ "testing/iotest": true,
+ "testing/quick": true,
+ "text/scanner": true,
+ "text/tabwriter": true,
+ "text/template": true,
+ "text/template/parse": true,
+ "time": true,
+ "unicode": true,
+ "unicode/utf16": true,
+ "unicode/utf8": true,
+ "unsafe": true,
+}
+
+var validTLD = map[string]bool{
+ // curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | sed -e '/#/ d' -e 's/.*/"&": true,/' | tr [:upper:] [:lower:]
+ ".ac": true,
+ ".ad": true,
+ ".ae": true,
+ ".aero": true,
+ ".af": true,
+ ".ag": true,
+ ".ai": true,
+ ".al": true,
+ ".am": true,
+ ".an": true,
+ ".ao": true,
+ ".aq": true,
+ ".ar": true,
+ ".arpa": true,
+ ".as": true,
+ ".asia": true,
+ ".at": true,
+ ".au": true,
+ ".aw": true,
+ ".ax": true,
+ ".az": true,
+ ".ba": true,
+ ".bb": true,
+ ".bd": true,
+ ".be": true,
+ ".bf": true,
+ ".bg": true,
+ ".bh": true,
+ ".bi": true,
+ ".biz": true,
+ ".bj": true,
+ ".bm": true,
+ ".bn": true,
+ ".bo": true,
+ ".br": true,
+ ".bs": true,
+ ".bt": true,
+ ".bv": true,
+ ".bw": true,
+ ".by": true,
+ ".bz": true,
+ ".ca": true,
+ ".cat": true,
+ ".cc": true,
+ ".cd": true,
+ ".cf": true,
+ ".cg": true,
+ ".ch": true,
+ ".ci": true,
+ ".ck": true,
+ ".cl": true,
+ ".cm": true,
+ ".cn": true,
+ ".co": true,
+ ".com": true,
+ ".coop": true,
+ ".cr": true,
+ ".cu": true,
+ ".cv": true,
+ ".cw": true,
+ ".cx": true,
+ ".cy": true,
+ ".cz": true,
+ ".de": true,
+ ".dj": true,
+ ".dk": true,
+ ".dm": true,
+ ".do": true,
+ ".dz": true,
+ ".ec": true,
+ ".edu": true,
+ ".ee": true,
+ ".eg": true,
+ ".er": true,
+ ".es": true,
+ ".et": true,
+ ".eu": true,
+ ".fi": true,
+ ".fj": true,
+ ".fk": true,
+ ".fm": true,
+ ".fo": true,
+ ".fr": true,
+ ".ga": true,
+ ".gb": true,
+ ".gd": true,
+ ".ge": true,
+ ".gf": true,
+ ".gg": true,
+ ".gh": true,
+ ".gi": true,
+ ".gl": true,
+ ".gm": true,
+ ".gn": true,
+ ".gov": true,
+ ".gp": true,
+ ".gq": true,
+ ".gr": true,
+ ".gs": true,
+ ".gt": true,
+ ".gu": true,
+ ".gw": true,
+ ".gy": true,
+ ".hk": true,
+ ".hm": true,
+ ".hn": true,
+ ".hr": true,
+ ".ht": true,
+ ".hu": true,
+ ".id": true,
+ ".ie": true,
+ ".il": true,
+ ".im": true,
+ ".in": true,
+ ".info": true,
+ ".int": true,
+ ".io": true,
+ ".iq": true,
+ ".ir": true,
+ ".is": true,
+ ".it": true,
+ ".je": true,
+ ".jm": true,
+ ".jo": true,
+ ".jobs": true,
+ ".jp": true,
+ ".ke": true,
+ ".kg": true,
+ ".kh": true,
+ ".ki": true,
+ ".km": true,
+ ".kn": true,
+ ".kp": true,
+ ".kr": true,
+ ".kw": true,
+ ".ky": true,
+ ".kz": true,
+ ".la": true,
+ ".lb": true,
+ ".lc": true,
+ ".li": true,
+ ".lk": true,
+ ".lr": true,
+ ".ls": true,
+ ".lt": true,
+ ".lu": true,
+ ".lv": true,
+ ".ly": true,
+ ".ma": true,
+ ".mc": true,
+ ".md": true,
+ ".me": true,
+ ".mg": true,
+ ".mh": true,
+ ".mil": true,
+ ".mk": true,
+ ".ml": true,
+ ".mm": true,
+ ".mn": true,
+ ".mo": true,
+ ".mobi": true,
+ ".mp": true,
+ ".mq": true,
+ ".mr": true,
+ ".ms": true,
+ ".mt": true,
+ ".mu": true,
+ ".museum": true,
+ ".mv": true,
+ ".mw": true,
+ ".mx": true,
+ ".my": true,
+ ".mz": true,
+ ".na": true,
+ ".name": true,
+ ".nc": true,
+ ".ne": true,
+ ".net": true,
+ ".nf": true,
+ ".ng": true,
+ ".ni": true,
+ ".nl": true,
+ ".no": true,
+ ".np": true,
+ ".nr": true,
+ ".nu": true,
+ ".nz": true,
+ ".om": true,
+ ".org": true,
+ ".pa": true,
+ ".pe": true,
+ ".pf": true,
+ ".pg": true,
+ ".ph": true,
+ ".pk": true,
+ ".pl": true,
+ ".pm": true,
+ ".pn": true,
+ ".post": true,
+ ".pr": true,
+ ".pro": true,
+ ".ps": true,
+ ".pt": true,
+ ".pw": true,
+ ".py": true,
+ ".qa": true,
+ ".re": true,
+ ".ro": true,
+ ".rs": true,
+ ".ru": true,
+ ".rw": true,
+ ".sa": true,
+ ".sb": true,
+ ".sc": true,
+ ".sd": true,
+ ".se": true,
+ ".sg": true,
+ ".sh": true,
+ ".si": true,
+ ".sj": true,
+ ".sk": true,
+ ".sl": true,
+ ".sm": true,
+ ".sn": true,
+ ".so": true,
+ ".sr": true,
+ ".st": true,
+ ".su": true,
+ ".sv": true,
+ ".sx": true,
+ ".sy": true,
+ ".sz": true,
+ ".tc": true,
+ ".td": true,
+ ".tel": true,
+ ".tf": true,
+ ".tg": true,
+ ".th": true,
+ ".tj": true,
+ ".tk": true,
+ ".tl": true,
+ ".tm": true,
+ ".tn": true,
+ ".to": true,
+ ".tp": true,
+ ".tr": true,
+ ".travel": true,
+ ".tt": true,
+ ".tv": true,
+ ".tw": true,
+ ".tz": true,
+ ".ua": true,
+ ".ug": true,
+ ".uk": true,
+ ".us": true,
+ ".uy": true,
+ ".uz": true,
+ ".va": true,
+ ".vc": true,
+ ".ve": true,
+ ".vg": true,
+ ".vi": true,
+ ".vn": true,
+ ".vu": true,
+ ".wf": true,
+ ".ws": true,
+ ".xn--0zwm56d": true,
+ ".xn--11b5bs3a9aj6g": true,
+ ".xn--3e0b707e": true,
+ ".xn--45brj9c": true,
+ ".xn--80akhbyknj4f": true,
+ ".xn--80ao21a": true,
+ ".xn--90a3ac": true,
+ ".xn--9t4b11yi5a": true,
+ ".xn--clchc0ea0b2g2a9gcd": true,
+ ".xn--deba0ad": true,
+ ".xn--fiqs8s": true,
+ ".xn--fiqz9s": true,
+ ".xn--fpcrj9c3d": true,
+ ".xn--fzc2c9e2c": true,
+ ".xn--g6w251d": true,
+ ".xn--gecrj9c": true,
+ ".xn--h2brj9c": true,
+ ".xn--hgbk6aj7f53bba": true,
+ ".xn--hlcj6aya9esc7a": true,
+ ".xn--j6w193g": true,
+ ".xn--jxalpdlp": true,
+ ".xn--kgbechtv": true,
+ ".xn--kprw13d": true,
+ ".xn--kpry57d": true,
+ ".xn--lgbbat1ad8j": true,
+ ".xn--mgb9awbf": true,
+ ".xn--mgbaam7a8h": true,
+ ".xn--mgbayh7gpa": true,
+ ".xn--mgbbh1a71e": true,
+ ".xn--mgbc0a9azcg": true,
+ ".xn--mgberp4a5d4ar": true,
+ ".xn--mgbx4cd0ab": true,
+ ".xn--o3cw4h": true,
+ ".xn--ogbpf8fl": true,
+ ".xn--p1ai": true,
+ ".xn--pgbs0dh": true,
+ ".xn--s9brj9c": true,
+ ".xn--wgbh1c": true,
+ ".xn--wgbl6a": true,
+ ".xn--xkc2al3hye2a": true,
+ ".xn--xkc2dl3a5ee0h": true,
+ ".xn--yfro4i67o": true,
+ ".xn--ygbi2ammx": true,
+ ".xn--zckzah": true,
+ ".xxx": true,
+ ".ye": true,
+ ".yt": true,
+ ".za": true,
+ ".zm": true,
+ ".zw": true,
+}
+
+var validHost = regexp.MustCompile(`^[-a-z0-9]+(?:\.[-a-z0-9]+)+$`)
+var validPathElement = regexp.MustCompile(`^[-A-Za-z0-9~+][-A-Za-z0-9_.]*$`)
+
+func isValidPathElement(s string) bool {
+ return validPathElement.MatchString(s) && s != "testdata"
+}
+
+// IsValidRemotePath returns true if importPath is structurally valid for "go get".
+func IsValidRemotePath(importPath string) bool {
+
+ parts := strings.Split(importPath, "/")
+
+ if len(parts) <= 1 {
+ // Import path must contain at least one "/".
+ return false
+ }
+
+ if !validTLD[path.Ext(parts[0])] {
+ return false
+ }
+
+ if !validHost.MatchString(parts[0]) {
+ return false
+ }
+
+ for _, part := range parts[1:] {
+ if !isValidPathElement(part) {
+ return false
+ }
+ }
+
+ return true
+}
+
+var goRepoPath = map[string]bool{}
+
+func init() {
+ for p := range standardPath {
+ for {
+ goRepoPath[p] = true
+ i := strings.LastIndex(p, "/")
+ if i < 0 {
+ break
+ }
+ p = p[:i]
+ }
+ }
+}
+
+func IsGoRepoPath(importPath string) bool {
+ return goRepoPath[importPath]
+}
+
+func IsValidPath(importPath string) bool {
+ return importPath == "C" || standardPath[importPath] || IsValidRemotePath(importPath)
+}
diff --git a/gosrc/path_test.go b/gosrc/path_test.go
new file mode 100644
index 0000000..024c186
--- /dev/null
+++ b/gosrc/path_test.go
@@ -0,0 +1,46 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "testing"
+)
+
+var goodImportPaths = []string{
+ "github.com/user/repo",
+ "github.com/user/repo/src/pkg/compress/somethingelse",
+ "github.com/user/repo/src/compress/gzip",
+ "github.com/user/repo/src/pkg",
+ "camlistore.org/r/p/camlistore",
+ "example.com/foo.git",
+ "launchpad.net/~user/foo/trunk",
+ "launchpad.net/~user/+junk/version",
+}
+
+var badImportPaths = []string{
+ "foobar",
+ "foo.",
+ ".bar",
+ "favicon.ico",
+ "exmpple.com",
+ "github.com/user/repo/testdata/x",
+ "github.com/user/repo/_ignore/x",
+ "github.com/user/repo/.ignore/x",
+}
+
+func TestIsValidRemotePath(t *testing.T) {
+ for _, importPath := range goodImportPaths {
+ if !IsValidRemotePath(importPath) {
+ t.Errorf("isBadImportPath(%q) -> true, want false", importPath)
+ }
+ }
+ for _, importPath := range badImportPaths {
+ if IsValidRemotePath(importPath) {
+ t.Errorf("isBadImportPath(%q) -> false, want true", importPath)
+ }
+ }
+}
diff --git a/gosrc/present.go b/gosrc/present.go
new file mode 100644
index 0000000..455804b
--- /dev/null
+++ b/gosrc/present.go
@@ -0,0 +1,73 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "regexp"
+ "time"
+)
+
+type Presentation struct {
+ Filename string
+ Files map[string][]byte
+ Updated time.Time
+}
+
+type presBuilder struct {
+ filename string
+ data []byte
+ resolveURL func(fname string) string
+ fetch func(fnames []string) ([]*File, error)
+}
+
+var assetPat = regexp.MustCompile(`(?m)^\.(play|code|image|iframe|html)\s+(?:-\S+\s+)*(\S+)`)
+
+func (b *presBuilder) build() (*Presentation, error) {
+ var data []byte
+ var fnames []string
+ i := 0
+ for _, m := range assetPat.FindAllSubmatchIndex(b.data, -1) {
+ name := string(b.data[m[4]:m[5]])
+ switch string(b.data[m[2]:m[3]]) {
+ case "iframe", "image":
+ data = append(data, b.data[i:m[4]]...)
+ data = append(data, b.resolveURL(name)...)
+ case "html":
+ // TODO: sanitize and fix relative URLs in HTML.
+ data = append(data, "\nERROR: .html not supported\n"...)
+ case "play", "code":
+ data = append(data, b.data[i:m[5]]...)
+ found := false
+ for _, n := range fnames {
+ if n == name {
+ found = true
+ break
+ }
+ }
+ if !found {
+ fnames = append(fnames, name)
+ }
+ default:
+ data = append(data, "\nERROR: unknown command\n"...)
+ }
+ i = m[5]
+ }
+ data = append(data, b.data[i:]...)
+ files, err := b.fetch(fnames)
+ if err != nil {
+ return nil, err
+ }
+ pres := &Presentation{
+ Updated: time.Now().UTC(),
+ Filename: b.filename,
+ Files: map[string][]byte{b.filename: data},
+ }
+ for _, f := range files {
+ pres.Files[f.Name] = f.Data
+ }
+ return pres, nil
+}
diff --git a/gosrc/print.go b/gosrc/print.go
new file mode 100644
index 0000000..f78bb9c
--- /dev/null
+++ b/gosrc/print.go
@@ -0,0 +1,79 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+// +build ignore
+
+// Command print fetches and prints package.
+//
+// Usage: go run print.go importPath
+package main
+
+import (
+ "flag"
+ "fmt"
+ "log"
+ "net/http"
+ "strings"
+
+ "github.com/garyburd/gosrc"
+)
+
+var (
+ etag = flag.String("etag", "", "Etag")
+ local = flag.String("local", "", "Get package from local workspace.")
+ present = flag.Bool("present", false, "Get presentation.")
+)
+
+func main() {
+ flag.Parse()
+ gosrc.SetUserAgent("print")
+ if len(flag.Args()) != 1 {
+ log.Fatal("Usage: go run print.go importPath")
+ }
+ if *present {
+ printPresentation(flag.Args()[0])
+ } else {
+ printDir(flag.Args()[0])
+ }
+}
+
+func printDir(path string) {
+ if *local != "" {
+ gosrc.SetLocalDevMode(*local)
+ }
+ dir, err := gosrc.Get(http.DefaultClient, path, *etag)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ fmt.Println("ImportPath ", dir.ImportPath)
+ fmt.Println("ResovledPath ", dir.ResolvedPath)
+ fmt.Println("ProjectRoot ", dir.ProjectRoot)
+ fmt.Println("ProjectName ", dir.ProjectName)
+ fmt.Println("ProjectURL ", dir.ProjectURL)
+ fmt.Println("VCS ", dir.VCS)
+ fmt.Println("Etag ", dir.Etag)
+ fmt.Println("BrowseURL ", dir.BrowseURL)
+ fmt.Println("Subdirectories", strings.Join(dir.Subdirectories, ", "))
+ fmt.Println("LineFmt ", dir.LineFmt)
+ fmt.Println("Files:")
+ for _, file := range dir.Files {
+ fmt.Printf("%30s %5d %s\n", file.Name, len(file.Data), file.BrowseURL)
+ }
+}
+
+func printPresentation(path string) {
+ pres, err := gosrc.GetPresentation(http.DefaultClient, path)
+ if err != nil {
+ log.Fatal(err)
+ }
+ fmt.Printf("%s\n", pres.Files[pres.Filename])
+ for name, data := range pres.Files {
+ if name != pres.Filename {
+ fmt.Printf("---------- %s ----------\n%s\n", name, data)
+ }
+ }
+}
diff --git a/gosrc/util.go b/gosrc/util.go
new file mode 100644
index 0000000..f5670dd
--- /dev/null
+++ b/gosrc/util.go
@@ -0,0 +1,70 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+var defaultTags = map[string]string{"git": "master", "hg": "default"}
+
+func bestTag(tags map[string]string, defaultTag string) (string, string, error) {
+ if commit, ok := tags["go1"]; ok {
+ return "go1", commit, nil
+ }
+ if commit, ok := tags[defaultTag]; ok {
+ return defaultTag, commit, nil
+ }
+ return "", "", NotFoundError{"Tag or branch not found."}
+}
+
+// expand replaces {k} in template with match[k] or subs[atoi(k)] if k is not in match.
+func expand(template string, match map[string]string, subs ...string) string {
+ var p []byte
+ var i int
+ for {
+ i = strings.Index(template, "{")
+ if i < 0 {
+ break
+ }
+ p = append(p, template[:i]...)
+ template = template[i+1:]
+ i = strings.Index(template, "}")
+ if s, ok := match[template[:i]]; ok {
+ p = append(p, s...)
+ } else {
+ j, _ := strconv.Atoi(template[:i])
+ p = append(p, subs[j]...)
+ }
+ template = template[i+1:]
+ }
+ p = append(p, template...)
+ return string(p)
+}
+
+var readmePat = regexp.MustCompile(`(?i)^readme(?:$|\.)`)
+
+// isDocFile returns true if a file with name n should be included in the
+// documentation.
+func isDocFile(n string) bool {
+ if strings.HasSuffix(n, ".go") && n[0] != '_' && n[0] != '.' {
+ return true
+ }
+ return readmePat.MatchString(n)
+}
+
+var linePat = regexp.MustCompile(`(?m)^//line .*$`)
+
+func OverwriteLineComments(p []byte) {
+ for _, m := range linePat.FindAllIndex(p, -1) {
+ for i := m[0] + 2; i < m[1]; i++ {
+ p[i] = ' '
+ }
+ }
+}
diff --git a/gosrc/util_test.go b/gosrc/util_test.go
new file mode 100644
index 0000000..2561e3a
--- /dev/null
+++ b/gosrc/util_test.go
@@ -0,0 +1,31 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+package gosrc
+
+import (
+ "testing"
+)
+
+var lineCommentTests = []struct {
+ in, out string
+}{
+ {"", ""},
+ {"//line 1", "// "},
+ {"//line x\n//line y", "// \n// "},
+ {"x\n//line ", "x\n// "},
+}
+
+func TestOverwriteLineComments(t *testing.T) {
+ for _, tt := range lineCommentTests {
+ p := []byte(tt.in)
+ OverwriteLineComments(p)
+ s := string(p)
+ if s != tt.out {
+ t.Errorf("in=%q, actual=%q, expect=%q", tt.in, s, tt.out)
+ }
+ }
+}
diff --git a/gosrc/vcs.go b/gosrc/vcs.go
new file mode 100644
index 0000000..6d2d78a
--- /dev/null
+++ b/gosrc/vcs.go
@@ -0,0 +1,255 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd.
+
+// +build !appengine
+
+package gosrc
+
+import (
+ "bytes"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strings"
+)
+
+func init() {
+ addService(&service{
+ pattern: regexp.MustCompile(`^(?P<repo>(?:[a-z0-9.\-]+\.)+[a-z0-9.\-]+(?::[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn)(?P<dir>/[A-Za-z0-9_.\-/]*)?$`),
+ prefix: "",
+ get: getVCSDir,
+ })
+ getVCSDirFn = getVCSDir
+}
+
+// Store temporary data in this directory.
+var TempDir = filepath.Join(os.TempDir(), "gddo")
+
+type urlTemplates struct {
+ re *regexp.Regexp
+ fileBrowse string
+ project string
+ line string
+}
+
+var vcsServices = []*urlTemplates{
+ {
+ regexp.MustCompile(`^git\.gitorious\.org/(?P<repo>[^/]+/[^/]+)$`),
+ "https://gitorious.org/{repo}/blobs/{tag}/{dir}{0}",
+ "https://gitorious.org/{repo}",
+ "%s#line%d",
+ },
+ {
+ regexp.MustCompile(`^git\.oschina\.net/(?P<repo>[^/]+/[^/]+)$`),
+ "http://git.oschina.net/{repo}/blob/{tag}/{dir}{0}",
+ "http://git.oschina.net/{repo}",
+ "%s#L%d",
+ },
+ {
+ regexp.MustCompile(`^(?P<r1>[^.]+)\.googlesource.com/(?P<r2>[^./]+)$`),
+ "https://{r1}.googlesource.com/{r2}/+/{tag}/{dir}{0}",
+ "https://{r1}.googlesource.com/{r2}/+/{tag}",
+ "",
+ },
+ {
+ regexp.MustCompile(`^gitcafe.com/(?P<repo>[^/]+/.[^/]+)$`),
+ "https://gitcafe.com/{repo}/tree/{tag}/{dir}{0}",
+ "https://gitcafe.com/{repo}",
+ "",
+ },
+}
+
+// lookupURLTemplate finds an expand() template, match map and line number
+// format for well known repositories.
+func lookupURLTemplate(repo, dir, tag string) (*urlTemplates, map[string]string) {
+ if strings.HasPrefix(dir, "/") {
+ dir = dir[1:] + "/"
+ }
+ for _, t := range vcsServices {
+ if m := t.re.FindStringSubmatch(repo); m != nil {
+ match := map[string]string{
+ "dir": dir,
+ "tag": tag,
+ }
+ for i, name := range t.re.SubexpNames() {
+ if name != "" {
+ match[name] = m[i]
+ }
+ }
+ return t, match
+ }
+ }
+ return &urlTemplates{}, nil
+}
+
+type vcsCmd struct {
+ schemes []string
+ download func([]string, string, string) (string, string, error)
+}
+
+var vcsCmds = map[string]*vcsCmd{
+ "git": {
+ schemes: []string{"http", "https", "git"},
+ download: downloadGit,
+ },
+}
+
+var lsremoteRe = regexp.MustCompile(`(?m)^([0-9a-f]{40})\s+refs/(?:tags|heads)/(.+)$`)
+
+func downloadGit(schemes []string, repo, savedEtag string) (string, string, error) {
+ var p []byte
+ var scheme string
+ for i := range schemes {
+ cmd := exec.Command("git", "ls-remote", "--heads", "--tags", schemes[i]+"://"+repo+".git")
+ log.Println(strings.Join(cmd.Args, " "))
+ var err error
+ p, err = cmd.Output()
+ if err == nil {
+ scheme = schemes[i]
+ break
+ }
+ }
+
+ if scheme == "" {
+ return "", "", NotFoundError{"VCS not found"}
+ }
+
+ tags := make(map[string]string)
+ for _, m := range lsremoteRe.FindAllSubmatch(p, -1) {
+ tags[string(m[2])] = string(m[1])
+ }
+
+ tag, commit, err := bestTag(tags, "master")
+ if err != nil {
+ return "", "", err
+ }
+
+ etag := scheme + "-" + commit
+
+ if etag == savedEtag {
+ return "", "", ErrNotModified
+ }
+
+ dir := path.Join(TempDir, repo+".git")
+ p, err = ioutil.ReadFile(path.Join(dir, ".git/HEAD"))
+ switch {
+ case err != nil:
+ if err := os.MkdirAll(dir, 0777); err != nil {
+ return "", "", err
+ }
+ cmd := exec.Command("git", "clone", scheme+"://"+repo+".git", dir)
+ log.Println(strings.Join(cmd.Args, " "))
+ if err := cmd.Run(); err != nil {
+ return "", "", err
+ }
+ case string(bytes.TrimRight(p, "\n")) == commit:
+ return tag, etag, nil
+ default:
+ cmd := exec.Command("git", "fetch")
+ log.Println(strings.Join(cmd.Args, " "))
+ cmd.Dir = dir
+ if err := cmd.Run(); err != nil {
+ return "", "", err
+ }
+ }
+
+ cmd := exec.Command("git", "checkout", "--detach", "--force", commit)
+ cmd.Dir = dir
+ if err := cmd.Run(); err != nil {
+ return "", "", err
+ }
+
+ return tag, etag, nil
+}
+
+func getVCSDir(client *http.Client, match map[string]string, etagSaved string) (*Directory, error) {
+ cmd := vcsCmds[match["vcs"]]
+ if cmd == nil {
+ return nil, NotFoundError{expand("VCS not supported: {vcs}", match)}
+ }
+
+ scheme := match["scheme"]
+ if scheme == "" {
+ i := strings.Index(etagSaved, "-")
+ if i > 0 {
+ scheme = etagSaved[:i]
+ }
+ }
+
+ schemes := cmd.schemes
+ if scheme != "" {
+ for i := range cmd.schemes {
+ if cmd.schemes[i] == scheme {
+ schemes = cmd.schemes[i : i+1]
+ break
+ }
+ }
+ }
+
+ // Download and checkout.
+
+ tag, etag, err := cmd.download(schemes, match["repo"], etagSaved)
+ if err != nil {
+ return nil, err
+ }
+
+ // Find source location.
+
+ template, urlMatch := lookupURLTemplate(match["repo"], match["dir"], tag)
+
+ // Slurp source files.
+
+ d := path.Join(TempDir, expand("{repo}.{vcs}", match), match["dir"])
+ f, err := os.Open(d)
+ if err != nil {
+ if os.IsNotExist(err) {
+ err = NotFoundError{err.Error()}
+ }
+ return nil, err
+ }
+ fis, err := f.Readdir(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ var files []*File
+ var subdirs []string
+ for _, fi := range fis {
+ switch {
+ case fi.IsDir():
+ if isValidPathElement(fi.Name()) {
+ subdirs = append(subdirs, fi.Name())
+ }
+ case isDocFile(fi.Name()):
+ b, err := ioutil.ReadFile(path.Join(d, fi.Name()))
+ if err != nil {
+ return nil, err
+ }
+ files = append(files, &File{
+ Name: fi.Name(),
+ BrowseURL: expand(template.fileBrowse, urlMatch, fi.Name()),
+ Data: b,
+ })
+ }
+ }
+
+ return &Directory{
+ LineFmt: template.line,
+ ProjectRoot: expand("{repo}.{vcs}", match),
+ ProjectName: path.Base(match["repo"]),
+ ProjectURL: expand(template.project, urlMatch),
+ BrowseURL: "",
+ Etag: etag,
+ VCS: match["vcs"],
+ Subdirectories: subdirs,
+ Files: files,
+ }, nil
+}
