Merge pull request #335 from tomheng/fixed_github_url_parse
fixed_github_url_parse
diff --git a/Dockerfile b/Dockerfile
index 80b1c2f..4d0a4ad 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,12 +19,12 @@
# Manually fetch and install gddo-server dependencies (faster than "go get").
ADD https://github.com/garyburd/redigo/archive/779af66db5668074a96f522d9025cb0a5ef50d89.zip /x/redigo.zip
-ADD https://snappy-go.googlecode.com/archive/12e4b4183793ac4b061921e7980845e750679fd0.tar.gz /x/snappy-go.tar.gz
-RUN unzip /x/redigo.zip -d /x && tar xzvf /x/snappy-go.tar.gz -C /x && \
+ADD https://github.com/golang/snappy/archive/master.zip /x/snappy-go.zip
+RUN unzip /x/redigo.zip -d /x && unzip /x/snappy-go.zip -d /x && \
mkdir -p /go/src/github.com/garyburd && \
- mkdir -p /go/src/code.google.com/p && \
+ mkdir -p /go/src/github.com/golang && \
mv /x/redigo-* /go/src/github.com/garyburd/redigo && \
- mv /x/snappy-go-* /go/src/code.google.com/p/snappy-go && \
+ mv /x/snappy-master /go/src/github.com/golang/snappy && \
rm -rf /x
# Build the local gddo files.
diff --git a/database/index.go b/database/index.go
index 3986cdf..69279bb 100644
--- a/database/index.go
+++ b/database/index.go
@@ -48,6 +48,60 @@
var httpPat = regexp.MustCompile(`https?://\S+`)
+func collectSynopsisTerms(terms map[string]bool, synopsis string) {
+
+ synopsis = httpPat.ReplaceAllLiteralString(synopsis, "")
+
+ fields := strings.FieldsFunc(synopsis, isTermSep)
+ for i := range fields {
+ fields[i] = strings.ToLower(fields[i])
+ }
+
+ // Ignore boilerplate in the following common patterns:
+ // Package foo ...
+ // Command foo ...
+ // Package foo implements ... (and provides, contains)
+ // The foo package ...
+ // The foo package implements ...
+ // The foo command ...
+
+ checkPackageVerb := false
+ switch {
+ case len(fields) >= 1 && fields[0] == "package":
+ fields = fields[1:]
+ checkPackageVerb = true
+ case len(fields) >= 1 && fields[0] == "command":
+ fields = fields[1:]
+ case len(fields) >= 3 && fields[0] == "the" && fields[2] == "package":
+ fields[2] = fields[1]
+ fields = fields[2:]
+ checkPackageVerb = true
+ case len(fields) >= 3 && fields[0] == "the" && fields[2] == "command":
+ fields[2] = fields[1]
+ fields = fields[2:]
+ }
+
+ if checkPackageVerb && len(fields) >= 2 &&
+ (fields[1] == "implements" || fields[1] == "provides" || fields[1] == "contains") {
+ fields[1] = fields[0]
+ fields = fields[1:]
+ }
+
+ for _, s := range fields {
+ if !stopWord[s] {
+ terms[term(s)] = true
+ }
+ }
+}
+
+func termSlice(terms map[string]bool) []string {
+ result := make([]string, 0, len(terms))
+ for term := range terms {
+ result = append(result, term)
+ }
+ return result
+}
+
func documentTerms(pdoc *doc.Package, score float64) []string {
terms := make(map[string]bool)
@@ -87,20 +141,11 @@
// Synopsis
- synopsis := httpPat.ReplaceAllLiteralString(pdoc.Synopsis, "")
- for i, s := range strings.FieldsFunc(synopsis, isTermSep) {
- s = strings.ToLower(s)
- if !stopWord[s] && (i > 3 || s != "package") {
- terms[term(s)] = true
- }
- }
+ collectSynopsisTerms(terms, pdoc.Synopsis)
+
}
- result := make([]string, 0, len(terms))
- for term := range terms {
- result = append(result, term)
- }
- return result
+ return termSlice(terms)
}
// vendorPat matches the path of a vendored package.
diff --git a/database/index_test.go b/database/index_test.go
index ea40f69..e0092bf 100644
--- a/database/index_test.go
+++ b/database/index_test.go
@@ -95,7 +95,7 @@
sort.Strings(terms)
sort.Strings(tt.terms)
if !reflect.DeepEqual(terms, tt.terms) {
- t.Errorf("documentTerms(%s)=%#v, want %#v", tt.pdoc.ImportPath, terms, tt.terms)
+ t.Errorf("documentTerms(%s) ->\n got: %#v\nwant: %#v", tt.pdoc.ImportPath, terms, tt.terms)
}
}
}
@@ -127,3 +127,76 @@
}
}
}
+
+var synopsisTermTests = []struct {
+ synopsis string
+ terms []string
+}{
+ {
+ "Package foo implements bar.",
+ []string{"bar", "foo"},
+ },
+ {
+ "Package foo provides bar.",
+ []string{"bar", "foo"},
+ },
+ {
+ "The foo package provides bar.",
+ []string{"bar", "foo"},
+ },
+ {
+ "Package foo contains an implementation of bar.",
+ []string{"bar", "foo", "impl"},
+ },
+ {
+ "Package foo is awesome",
+ []string{"awesom", "foo"},
+ },
+ {
+ "The foo package is awesome",
+ []string{"awesom", "foo"},
+ },
+ {
+ "The foo command is awesome",
+ []string{"awesom", "foo"},
+ },
+ {
+ "Command foo is awesome",
+ []string{"awesom", "foo"},
+ },
+ {
+ "The foo package",
+ []string{"foo"},
+ },
+ {
+ "Package foo",
+ []string{"foo"},
+ },
+ {
+ "Command foo",
+ []string{"foo"},
+ },
+ {
+ "Package",
+ []string{},
+ },
+ {
+ "Command",
+ []string{},
+ },
+}
+
+func TestSynopsisTerms(t *testing.T) {
+ for _, tt := range synopsisTermTests {
+ terms := make(map[string]bool)
+ collectSynopsisTerms(terms, tt.synopsis)
+
+ actual := termSlice(terms)
+ expected := tt.terms
+ sort.Strings(actual)
+ sort.Strings(expected)
+ if !reflect.DeepEqual(actual, expected) {
+ t.Errorf("%q ->\n got: %#v\nwant: %#v", tt.synopsis, actual, expected)
+ }
+ }
+}
diff --git a/database/stop.go b/database/stop.go
index 3b23e34..2a519fd 100644
--- a/database/stop.go
+++ b/database/stop.go
@@ -73,8 +73,6 @@
how
i
if
-implement
-implements
in
into
is
diff --git a/gddo-server/assets/site.css b/gddo-server/assets/site.css
index a2637f8..bb270d7 100644
--- a/gddo-server/assets/site.css
+++ b/gddo-server/assets/site.css
@@ -21,6 +21,10 @@
}
+.highlighted {
+ background-color: #FDFF9E;
+}
+
#x-pkginfo {
margin-top: 25px;
border-top: 1px solid #ccc;
diff --git a/gddo-server/assets/site.js b/gddo-server/assets/site.js
index b0381a5..4f52d58 100644
--- a/gddo-server/assets/site.js
+++ b/gddo-server/assets/site.js
@@ -119,6 +119,24 @@
});
+$(function() {
+
+ if ("onhashchange" in window) {
+ var highlightedSel = "";
+ window.onhashchange = function() {
+ if (highlightedSel) {
+ $(highlightedSel).removeClass("highlighted");
+ }
+ highlightedSel = window.location.hash.replace( /(:|\.|\[|\]|,)/g, "\\$1" );
+ if (highlightedSel && (highlightedSel.indexOf("example-") == -1)) {
+ $(highlightedSel).addClass("highlighted");
+ }
+ };
+ window.onhashchange();
+ }
+
+});
+
// keyboard shortcuts
$(function() {
var prevCh = null, prevTime = 0, modal = false;
diff --git a/gddo-server/assets/templates/notfound.html b/gddo-server/assets/templates/notfound.html
index 684d208..7a99c38 100644
--- a/gddo-server/assets/templates/notfound.html
+++ b/gddo-server/assets/templates/notfound.html
@@ -1,6 +1,7 @@
{{define "Head"}}<title>Not Found - GoDoc</title>{{end}}
{{define "Body"}}
+ {{template "FlashMessages" .flashMessages}}
<h1>Not Found</h1>
<p>Oh snap! Our team of gophers could not find the web page you are looking for. Try one of these pages:
<ul>
diff --git a/gddo-server/assets/templates/pkg.html b/gddo-server/assets/templates/pkg.html
index c98b74d..63ff746 100644
--- a/gddo-server/assets/templates/pkg.html
+++ b/gddo-server/assets/templates/pkg.html
@@ -178,7 +178,7 @@
<div id="ex-{{.ID}}" class="panel-collapse collapse"><div class="panel-body">
{{with .Example.Doc}}<p>{{.|comment}}{{end}}
<p>Code:{{if .Example.Play}}<span class="pull-right"><a href="?play={{.ID}}">play</a> </span>{{end}}
- <pre>{{code .Example.Code nil}}</pre>
+ {{code .Example.Code nil}}
{{with .Example.Output}}<p>Output:<pre>{{.}}</pre>{{end}}
</div></div>
</div>
diff --git a/gddo-server/crawl.go b/gddo-server/crawl.go
index d0a0645..fb23fa7 100644
--- a/gddo-server/crawl.go
+++ b/gddo-server/crawl.go
@@ -56,6 +56,9 @@
pdocNew, err = doc.Get(httpClient, importPath, etag)
message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
if err == nil && pdocNew.Name == "" && !hasSubdirs {
+ for _, e := range pdocNew.Errors {
+ message = append(message, "err:", e)
+ }
pdoc = nil
err = gosrc.NotFoundError{Message: "no Go files or subdirs"}
} else if err != gosrc.ErrNotModified {
diff --git a/gddo-server/main.go b/gddo-server/main.go
index 486addd..816971a 100644
--- a/gddo-server/main.go
+++ b/gddo-server/main.go
@@ -18,6 +18,7 @@
"html/template"
"io"
"log"
+ "net"
"net/http"
"os"
"path"
@@ -60,6 +61,7 @@
robotRequest
queryRequest
refreshRequest
+ apiRequest
)
type crawlResult struct {
@@ -84,7 +86,7 @@
needsCrawl := false
switch requestType {
- case queryRequest:
+ case queryRequest, apiRequest:
needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0
case humanRequest:
needsCrawl = nextCrawl.Before(time.Now())
@@ -598,7 +600,10 @@
var pkgs []database.Package
if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
- pdoc, _, err := getDoc(q, robotRequest)
+ pdoc, _, err := getDoc(q, apiRequest)
+ if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+ pdoc, _, err = getDoc(e.Redirect, robotRequest)
+ }
if err == nil && pdoc != nil {
pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}}
}
@@ -740,7 +745,9 @@
func handleError(resp http.ResponseWriter, req *http.Request, status int, err error) {
switch status {
case http.StatusNotFound:
- executeTemplate(resp, "notfound"+templateExt(req), status, nil, nil)
+ executeTemplate(resp, "notfound"+templateExt(req), status, nil, map[string]interface{}{
+ "flashMessages": getFlashMessages(resp, req),
+ })
default:
resp.Header().Set("Content-Type", textMIMEType)
resp.WriteHeader(http.StatusInternalServerError)
@@ -760,12 +767,29 @@
json.NewEncoder(resp).Encode(&data)
}
-type hostMux []struct {
+type rootHandler []struct {
prefix string
h http.Handler
}
-func (m hostMux) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (m rootHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ host := req.Host
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ host = h
+ }
+ if host == "godoc.org" {
+ if req.Header.Get("X-Scheme") != "https" {
+ u := *req.URL
+ u.Scheme = "https"
+ u.Host = host
+ http.Redirect(resp, req, u.String(), http.StatusFound)
+ return
+ }
+ // Because https is not used api.godoc.org, the includeSubDomains
+ // parameter is not used here.
+ resp.Header().Add("Strict-Transport-Security", "max-age=631138519; preload")
+ }
+
var h http.Handler
for _, ph := range m {
if strings.HasPrefix(req.Host, ph.prefix) {
@@ -773,6 +797,7 @@
break
}
}
+
h.ServeHTTP(resp, req)
}
@@ -898,7 +923,7 @@
cacheBusters.Handler = mux
- if err := http.ListenAndServe(*httpAddr, hostMux{{"api.", apiMux}, {"", mux}}); err != nil {
+ if err := http.ListenAndServe(*httpAddr, rootHandler{{"api.", apiMux}, {"", mux}}); err != nil {
log.Fatal(err)
}
}
diff --git a/gosrc/client.go b/gosrc/client.go
index be19e0f..09d54e3 100644
--- a/gosrc/client.go
+++ b/gosrc/client.go
@@ -30,6 +30,7 @@
return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: (%s)", resp.StatusCode, resp.Request.URL.String())}
}
+// get issues a GET to the specified URL.
func (c *httpClient) get(url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
@@ -45,6 +46,26 @@
return resp, err
}
+// getNoFollow issues a GET to the specified URL without following redirects.
+func (c *httpClient) getNoFollow(url string) (*http.Response, error) {
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return nil, err
+ }
+ for k, vs := range c.header {
+ req.Header[k] = vs
+ }
+ t := c.client.Transport
+ if t == nil {
+ t = http.DefaultTransport
+ }
+ resp, err := t.RoundTrip(req)
+ if err != nil {
+ return nil, &RemoteError{req.URL.Host, err}
+ }
+ return resp, err
+}
+
func (c *httpClient) getBytes(url string) ([]byte, error) {
resp, err := c.get(url)
if err != nil {
diff --git a/gosrc/google.go b/gosrc/google.go
index c36c223..1a47b59 100644
--- a/gosrc/google.go
+++ b/gosrc/google.go
@@ -30,10 +30,32 @@
googleFileRe = regexp.MustCompile(`<li><a href="([^"]+)"`)
)
+func checkGoogleRedir(c *httpClient, match map[string]string) error {
+ resp, err := c.getNoFollow(expand("https://code.google.com/{pr}/{repo}/", match))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusOK {
+ return nil
+ }
+ if resp.StatusCode == http.StatusMovedPermanently {
+ if u, err := url.Parse(resp.Header.Get("Location")); err == nil {
+ p := u.Host + u.Path + match["dir"]
+ return NotFoundError{Message: "Project moved", Redirect: p}
+ }
+ }
+ return c.err(resp)
+}
+
func getGoogleDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+ setupGoogleMatch(match)
c := &httpClient{client: client}
- setupGoogleMatch(match)
+ if err := checkGoogleRedir(c, match); err != nil {
+ return nil, err
+ }
+
if m := googleEtagRe.FindStringSubmatch(savedEtag); m != nil {
match["vcs"] = m[1]
} else if err := getGoogleVCS(c, match); err != nil {
diff --git a/gosrc/print.go b/gosrc/print.go
index 46de29c..f833bc3 100644
--- a/gosrc/print.go
+++ b/gosrc/print.go
@@ -44,8 +44,10 @@
gosrc.SetLocalDevMode(*local)
}
dir, err := gosrc.Get(http.DefaultClient, path, *etag)
- if err != nil {
- log.Fatal(err)
+ if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+ log.Fatalf("redirect to %s", e.Redirect)
+ } else if err != nil {
+ log.Fatalf("%+v", err)
}
fmt.Println("ImportPath ", dir.ImportPath)