all: propagate context throughout the codebase.
Change-Id: Icdb6f5911e49e121014077067ac8c9a83b0ea8cd
Reviewed-on: https://go-review.googlesource.com/66112
Reviewed-by: Ross Light <light@google.com>
diff --git a/database/database.go b/database/database.go
index 0eea7c8..4f112fb 100644
--- a/database/database.go
+++ b/database/database.go
@@ -31,6 +31,7 @@
import (
"bytes"
+ "context"
"encoding/gob"
"errors"
"fmt"
@@ -47,7 +48,6 @@
"github.com/garyburd/redigo/redis"
"github.com/golang/snappy"
- "golang.org/x/net/context"
"golang.org/x/oauth2/google"
"google.golang.org/appengine"
"google.golang.org/appengine/remote_api"
@@ -62,7 +62,7 @@
Get() redis.Conn
}
- AppEngineContext context.Context
+ RemoteClient *remote_api.Client
}
// Package represents the content of a package both for the search index and
@@ -121,7 +121,7 @@
}
}
-func newAppEngineContext(host string) (context.Context, error) {
+func newRemoteClient(host string) (*remote_api.Client, error) {
client, err := google.DefaultClient(context.TODO(),
"https://www.googleapis.com/auth/appengine.apis",
)
@@ -129,7 +129,7 @@
return nil, err
}
- return remote_api.NewRemoteContext(host, client)
+ return remote_api.NewClient(host, client)
}
// New creates a gddo database. serverURI, idleTimeout, and logConn configure
@@ -148,15 +148,15 @@
}
c.Close()
- gaeCtx := context.TODO()
+ var rc *remote_api.Client
if gaeEndpoint != "" {
var err error
- if gaeCtx, err = newAppEngineContext(gaeEndpoint); err != nil {
+ if rc, err = newRemoteClient(gaeEndpoint); err != nil {
return nil, err
}
}
- return &Database{Pool: pool, AppEngineContext: gaeCtx}, nil
+ return &Database{Pool: pool, RemoteClient: rc}, nil
}
// Exists returns true if package with import path exists in the database.
@@ -235,7 +235,7 @@
}
// Put adds the package documentation to the database.
-func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time, hide bool) error {
+func (db *Database) Put(ctx context.Context, pdoc *doc.Package, nextCrawl time.Time, hide bool) error {
c := db.Pool.Get()
defer c.Close()
@@ -284,7 +284,7 @@
// Get old version of the package to extract its imports.
// If the package does not exist, both oldDoc and err will be nil.
- old, _, err := db.getDoc(c, pdoc.ImportPath)
+ old, _, err := db.getDoc(ctx, c, pdoc.ImportPath)
if err != nil {
return err
}
@@ -300,17 +300,17 @@
}
if score > 0 {
- if err := db.PutIndex(db.AppEngineContext, pdoc, id, score, n); err != nil {
+ if err := db.PutIndex(ctx, pdoc, id, score, n); err != nil {
log.Printf("Cannot put %q in index: %v", pdoc.ImportPath, err)
}
if old != nil {
- if err := db.updateImportsIndex(c, db.AppEngineContext, old, pdoc); err != nil {
+ if err := db.updateImportsIndex(ctx, c, old, pdoc); err != nil {
return err
}
}
} else {
- if err := deleteIndex(db.AppEngineContext, id); err != nil {
+ if err := db.DeleteIndex(ctx, id); err != nil {
return err
}
}
@@ -364,7 +364,7 @@
return id, numImported, nil
}
-func (db *Database) updateImportsIndex(c redis.Conn, ctx context.Context, oldDoc, newDoc *doc.Package) error {
+func (db *Database) updateImportsIndex(ctx context.Context, c redis.Conn, oldDoc, newDoc *doc.Package) error {
// Create a map to store any import change since last time we indexed the package.
changes := make(map[string]bool)
for _, p := range oldDoc.Imports {
@@ -387,7 +387,7 @@
return err
}
if id != "" {
- db.PutIndex(db.AppEngineContext, nil, id, -1, n)
+ db.PutIndex(ctx, nil, id, -1, n)
}
}
return nil
@@ -481,7 +481,7 @@
return {gob, nextCrawl}
`)
-func (db *Database) getDoc(c redis.Conn, path string) (*doc.Package, time.Time, error) {
+func (db *Database) getDoc(ctx context.Context, c redis.Conn, path string) (*doc.Package, time.Time, error) {
r, err := redis.Values(getDocScript.Do(c, path))
if err == redis.ErrNil {
return nil, time.Time{}, nil
@@ -573,11 +573,11 @@
// Get gets the package documentation and sub-directories for the the given
// import path.
-func (db *Database) Get(path string) (*doc.Package, []Package, time.Time, error) {
+func (db *Database) Get(ctx context.Context, path string) (*doc.Package, []Package, time.Time, error) {
c := db.Pool.Get()
defer c.Close()
- pdoc, nextCrawl, err := db.getDoc(c, path)
+ pdoc, nextCrawl, err := db.getDoc(ctx, c, path)
if err != nil {
return nil, nil, time.Time{}, err
}
@@ -594,10 +594,10 @@
return pdoc, subdirs, nextCrawl, nil
}
-func (db *Database) GetDoc(path string) (*doc.Package, time.Time, error) {
+func (db *Database) GetDoc(ctx context.Context, path string) (*doc.Package, time.Time, error) {
c := db.Pool.Get()
defer c.Close()
- return db.getDoc(c, path)
+ return db.getDoc(ctx, c, path)
}
var deleteScript = redis.NewScript(0, `
@@ -620,7 +620,7 @@
`)
// Delete deletes the documentation for the given import path.
-func (db *Database) Delete(path string) error {
+func (db *Database) Delete(ctx context.Context, path string) error {
c := db.Pool.Get()
defer c.Close()
@@ -631,7 +631,7 @@
if err != nil {
return err
}
- if err := deleteIndex(db.AppEngineContext, id); err != nil {
+ if err := db.DeleteIndex(ctx, id); err != nil {
return err
}
@@ -1200,6 +1200,10 @@
// This will update the search index with the path, synopsis, score, import counts
// of all the packages in the database.
func (db *Database) Reindex(ctx context.Context) error {
+ if db.RemoteClient == nil {
+ return errors.New("database.Reindex: no App Engine endpoint given")
+ }
+
c := db.Pool.Get()
defer c.Close()
@@ -1246,7 +1250,7 @@
if err != nil {
return err
}
- if _, err := idx.Put(ctx, id, &Package{
+ if _, err := idx.Put(db.RemoteClient.NewContext(ctx), id, &Package{
Path: pdoc.ImportPath,
Synopsis: pdoc.Synopsis,
Score: score,
@@ -1265,9 +1269,24 @@
}
func (db *Database) Search(ctx context.Context, q string) ([]Package, error) {
- return searchAE(db.AppEngineContext, q)
+ if db.RemoteClient == nil {
+ return nil, errors.New("remote_api client not setup to use App Engine search")
+ }
+ return searchAE(db.RemoteClient.NewContext(ctx), q)
}
+// PutIndex puts a package into App Engine search index. ID is the package ID in the database.
func (db *Database) PutIndex(ctx context.Context, pdoc *doc.Package, id string, score float64, importCount int) error {
- return putIndex(db.AppEngineContext, pdoc, id, score, importCount)
+ if db.RemoteClient == nil {
+ return errors.New("remote_api client not setup to use App Engine search")
+ }
+ return putIndex(db.RemoteClient.NewContext(ctx), pdoc, id, score, importCount)
+}
+
+// DeleteIndex deletes a package from App Engine search index. ID is the package ID in the database.
+func (db *Database) DeleteIndex(ctx context.Context, id string) error {
+ if db.RemoteClient == nil {
+ return errors.New("database.DeleteIndex: no App Engine endpoint given")
+ }
+ return deleteIndex(db.RemoteClient.NewContext(ctx), id)
}
diff --git a/database/indexae.go b/database/indexae.go
index e86fa73..86b6169 100644
--- a/database/indexae.go
+++ b/database/indexae.go
@@ -8,6 +8,7 @@
import (
"bytes"
+ "context"
"errors"
"fmt"
"log"
@@ -15,7 +16,6 @@
"strings"
"unicode"
- "golang.org/x/net/context"
"google.golang.org/appengine/search"
"github.com/golang/gddo/doc"
@@ -181,6 +181,7 @@
}
// PurgeIndex deletes all the packages from the search index.
+// TODO(shantuo): wrap this with db and use db.RemoteClient to create the context.
func PurgeIndex(c context.Context) error {
idx, err := search.Open("packages")
if err != nil {
diff --git a/doc/get.go b/doc/get.go
index 67a21a6..1028d3f 100644
--- a/doc/get.go
+++ b/doc/get.go
@@ -8,6 +8,7 @@
package doc
import (
+ "context"
"go/doc"
"net/http"
"strings"
@@ -15,7 +16,7 @@
"github.com/golang/gddo/gosrc"
)
-func Get(client *http.Client, importPath string, etag string) (*Package, error) {
+func Get(ctx context.Context, client *http.Client, importPath string, etag string) (*Package, error) {
const versionPrefix = PackageVersion + "-"
@@ -25,7 +26,7 @@
etag = ""
}
- dir, err := gosrc.Get(client, importPath, etag)
+ dir, err := gosrc.Get(ctx, client, importPath, etag)
if err != nil {
return nil, err
}
@@ -41,7 +42,7 @@
pdoc.Name != "" &&
dir.ImportPath == dir.ProjectRoot &&
len(pdoc.Errors) == 0 {
- project, err := gosrc.GetProject(client, dir.ResolvedPath)
+ project, err := gosrc.GetProject(ctx, client, dir.ResolvedPath)
switch {
case err == nil:
pdoc.Synopsis = doc.Synopsis(project.Description)
diff --git a/gddo-admin/delete.go b/gddo-admin/delete.go
index 0ac2418..7a260f0 100644
--- a/gddo-admin/delete.go
+++ b/gddo-admin/delete.go
@@ -7,6 +7,7 @@
package main
import (
+ "context"
"log"
"os"
@@ -28,7 +29,7 @@
if err != nil {
log.Fatal(err)
}
- if err := db.Delete(c.flag.Args()[0]); err != nil {
+ if err := db.Delete(context.Background(), c.flag.Args()[0]); err != nil {
log.Fatal(err)
}
}
diff --git a/gddo-admin/reindex.go b/gddo-admin/reindex.go
index 4b3fb88..63d13fa 100644
--- a/gddo-admin/reindex.go
+++ b/gddo-admin/reindex.go
@@ -7,6 +7,7 @@
package main
import (
+ "context"
"log"
"os"
"time"
@@ -59,7 +60,7 @@
err = db.Do(func(pi *database.PackageInfo) error {
n++
fix(pi.PDoc)
- return db.Put(pi.PDoc, time.Time{}, false)
+ return db.Put(context.Background(), pi.PDoc, time.Time{}, false)
})
if err != nil {
log.Fatal(err)
diff --git a/gddo-server/background.go b/gddo-server/background.go
index 025e6e1..3a9650a 100644
--- a/gddo-server/background.go
+++ b/gddo-server/background.go
@@ -7,6 +7,7 @@
package main
import (
+ "context"
"log"
"time"
@@ -15,7 +16,7 @@
"github.com/golang/gddo/gosrc"
)
-func doCrawl() error {
+func doCrawl(ctx context.Context) error {
// Look for new package to crawl.
importPath, hasSubdirs, err := db.PopNewCrawl()
if err != nil {
@@ -23,7 +24,7 @@
return nil
}
if importPath != "" {
- if pdoc, err := crawlDoc("new", importPath, nil, hasSubdirs, time.Time{}); pdoc == nil && err == nil {
+ if pdoc, err := crawlDoc(ctx, "new", importPath, nil, hasSubdirs, time.Time{}); pdoc == nil && err == nil {
if err := db.AddBadCrawl(importPath); err != nil {
log.Printf("ERROR db.AddBadCrawl(%q): %v", importPath, err)
}
@@ -32,7 +33,7 @@
}
// Crawl existing doc.
- pdoc, pkgs, nextCrawl, err := db.Get("-")
+ pdoc, pkgs, nextCrawl, err := db.Get(ctx, "-")
if err != nil {
log.Printf("db.Get(\"-\") returned error %v", err)
return nil
@@ -40,7 +41,7 @@
if pdoc == nil || nextCrawl.After(time.Now()) {
return nil
}
- if _, err = crawlDoc("crawl", pdoc.ImportPath, pdoc, len(pkgs) > 0, nextCrawl); err != nil {
+ if _, err = crawlDoc(ctx, "crawl", pdoc.ImportPath, pdoc, len(pkgs) > 0, nextCrawl); err != nil {
// Touch package so that crawl advances to next package.
if err := db.SetNextCrawl(pdoc.ImportPath, time.Now().Add(viper.GetDuration(ConfigMaxAge)/3)); err != nil {
log.Printf("ERROR db.SetNextCrawl(%q): %v", pdoc.ImportPath, err)
@@ -49,13 +50,13 @@
return nil
}
-func readGitHubUpdates() error {
+func readGitHubUpdates(ctx context.Context) error {
const key = "gitHubUpdates"
var last string
if err := db.GetGob(key, &last); err != nil {
return err
}
- last, names, err := gosrc.GetGitHubUpdates(httpClient, last)
+ last, names, err := gosrc.GetGitHubUpdates(ctx, httpClient, last)
if err != nil {
return err
}
diff --git a/gddo-server/crawl.go b/gddo-server/crawl.go
index 959ab22..c4d1064 100644
--- a/gddo-server/crawl.go
+++ b/gddo-server/crawl.go
@@ -7,6 +7,7 @@
package main
import (
+ "context"
"fmt"
"log"
"regexp"
@@ -24,7 +25,7 @@
)
// crawlDoc fetches the package documentation from the VCS and updates the database.
-func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
+func crawlDoc(ctx context.Context, source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
message := []interface{}{source}
defer func() {
message = append(message, importPath)
@@ -58,7 +59,7 @@
err = gosrc.NotFoundError{Message: "testdata."}
} else {
var pdocNew *doc.Package
- pdocNew, err = doc.Get(httpClient, importPath, etag)
+ pdocNew, err = doc.Get(ctx, httpClient, importPath, etag)
message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
if err == nil && pdocNew.Name == "" && !hasSubdirs {
for _, e := range pdocNew.Errors {
@@ -83,7 +84,7 @@
if err == nil {
message = append(message, "put:", pdoc.Etag)
- if err := put(pdoc, nextCrawl); err != nil {
+ if err := put(ctx, pdoc, nextCrawl); err != nil {
log.Println(err)
}
return pdoc, nil
@@ -94,7 +95,7 @@
}
message = append(message, "archive", e)
pdoc.Status = e.Status
- if err := db.Put(pdoc, nextCrawl, false); err != nil {
+ if err := db.Put(ctx, pdoc, nextCrawl, false); err != nil {
log.Printf("ERROR db.Put(%q): %v", importPath, err)
}
} else {
@@ -107,7 +108,7 @@
return pdoc, nil
} else if e, ok := err.(gosrc.NotFoundError); ok {
message = append(message, "notfound:", e)
- if err := db.Delete(importPath); err != nil {
+ if err := db.Delete(ctx, importPath); err != nil {
log.Printf("ERROR db.Delete(%q): %v", importPath, err)
}
return nil, e
@@ -117,12 +118,12 @@
}
}
-func put(pdoc *doc.Package, nextCrawl time.Time) error {
+func put(ctx context.Context, pdoc *doc.Package, nextCrawl time.Time) error {
if pdoc.Status == gosrc.NoRecentCommits &&
isActivePkg(pdoc.ImportPath, gosrc.NoRecentCommits) {
pdoc.Status = gosrc.Active
}
- if err := db.Put(pdoc, nextCrawl, false); err != nil {
+ if err := db.Put(ctx, pdoc, nextCrawl, false); err != nil {
return fmt.Errorf("ERROR db.Put(%q): %v", pdoc.ImportPath, err)
}
return nil
diff --git a/gddo-server/main.go b/gddo-server/main.go
index da97770..c81a6c9 100644
--- a/gddo-server/main.go
+++ b/gddo-server/main.go
@@ -9,6 +9,7 @@
import (
"bytes"
+ "context"
"crypto/md5"
"encoding/json"
"errors"
@@ -30,8 +31,6 @@
"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/logging"
"github.com/spf13/viper"
- "golang.org/x/net/context"
- "google.golang.org/appengine"
"github.com/golang/gddo/database"
"github.com/golang/gddo/doc"
@@ -74,7 +73,7 @@
// getDoc gets the package documentation from the database or from the version
// control system as needed.
-func getDoc(path string, requestType int) (*doc.Package, []database.Package, error) {
+func getDoc(ctx context.Context, path string, requestType int) (*doc.Package, []database.Package, error) {
if path == "-" {
// A hack in the database package uses the path "-" to represent the
// next document to crawl. Block "-" here so that requests to /- always
@@ -82,7 +81,7 @@
return nil, nil, &httpError{status: http.StatusNotFound}
}
- pdoc, pkgs, nextCrawl, err := db.Get(path)
+ pdoc, pkgs, nextCrawl, err := db.Get(ctx, path)
if err != nil {
return nil, nil, err
}
@@ -103,7 +102,7 @@
c := make(chan crawlResult, 1)
go func() {
- pdoc, err := crawlDoc("web ", path, pdoc, len(pkgs) > 0, nextCrawl)
+ pdoc, err := crawlDoc(ctx, "web ", path, pdoc, len(pkgs) > 0, nextCrawl)
c <- crawlResult{pdoc, err}
}()
@@ -236,12 +235,12 @@
}
importPath := strings.TrimPrefix(req.URL.Path, "/")
- pdoc, pkgs, err := getDoc(importPath, requestType)
+ pdoc, pkgs, err := getDoc(req.Context(), importPath, requestType)
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
// To prevent dumb clients from following redirect loops, respond with
// status 404 if the target document is not found.
- if _, _, err := getDoc(e.Redirect, requestType); gosrc.IsNotFound(err) {
+ if _, _, err := getDoc(req.Context(), e.Redirect, requestType); gosrc.IsNotFound(err) {
return &httpError{status: http.StatusNotFound}
}
u := "/" + e.Redirect
@@ -262,7 +261,7 @@
if len(pkgs) == 0 {
return &httpError{status: http.StatusNotFound}
}
- pdocChild, _, _, err := db.Get(pkgs[0].Path)
+ pdocChild, _, _, err := db.Get(req.Context(), pkgs[0].Path)
if err != nil {
return err
}
@@ -420,13 +419,13 @@
func serveRefresh(resp http.ResponseWriter, req *http.Request) error {
importPath := req.Form.Get("path")
- _, pkgs, _, err := db.Get(importPath)
+ _, pkgs, _, err := db.Get(req.Context(), importPath)
if err != nil {
return err
}
c := make(chan error, 1)
go func() {
- _, err := crawlDoc("rfrsh", importPath, nil, len(pkgs) > 0, time.Time{})
+ _, err := crawlDoc(req.Context(), "rfrsh", importPath, nil, len(pkgs) > 0, time.Time{})
c <- err
}()
select {
@@ -552,7 +551,7 @@
}
if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
- pdoc, pkgs, err := getDoc(q, queryRequest)
+ pdoc, pkgs, err := getDoc(req.Context(), q, queryRequest)
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
return nil
@@ -611,9 +610,9 @@
var pkgs []database.Package
if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
- pdoc, _, err := getDoc(q, apiRequest)
+ pdoc, _, err := getDoc(req.Context(), q, apiRequest)
if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
- pdoc, _, err = getDoc(e.Redirect, robotRequest)
+ pdoc, _, err = getDoc(req.Context(), e.Redirect, robotRequest)
}
if err == nil && pdoc != nil {
pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}}
@@ -668,7 +667,7 @@
func serveAPIImports(resp http.ResponseWriter, req *http.Request) error {
importPath := strings.TrimPrefix(req.URL.Path, "/imports/")
- pdoc, _, err := getDoc(importPath, robotRequest)
+ pdoc, _, err := getDoc(req.Context(), importPath, robotRequest)
if err != nil {
return err
}
@@ -922,14 +921,14 @@
go func() {
for range time.Tick(viper.GetDuration(ConfigCrawlInterval)) {
- if err := doCrawl(); err != nil {
+ if err := doCrawl(context.Background()); err != nil {
log.Printf("Task Crawl: %v", err)
}
}
}()
go func() {
for range time.Tick(viper.GetDuration(ConfigGithubInterval)) {
- if err := readGitHubUpdates(); err != nil {
+ if err := readGitHubUpdates(context.Background()); err != nil {
log.Printf("Task GitHub updates: %v", err)
}
}
@@ -1010,6 +1009,5 @@
gceLogger = newGCELogger(logger)
}
- http.Handle("/", root)
- appengine.Main()
+ log.Fatal(http.ListenAndServe(viper.GetString(ConfigBindAddress), root))
}
diff --git a/gosrc/bitbucket.go b/gosrc/bitbucket.go
index 375e1f9..8435148 100644
--- a/gosrc/bitbucket.go
+++ b/gosrc/bitbucket.go
@@ -7,6 +7,7 @@
package gosrc
import (
+ "context"
"log"
"net/http"
"path"
@@ -40,14 +41,14 @@
Timestamp string `json:"utctimestamp"`
}
-func getBitbucketDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+func getBitbucketDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
var repo *bitbucketRepo
c := &httpClient{client: client}
if m := bitbucketEtagRe.FindStringSubmatch(savedEtag); m != nil {
match["vcs"] = m[1]
} else {
- repo, err := getBitbucketRepo(c, match)
+ repo, err := getBitbucketRepo(ctx, c, match)
if err != nil {
return nil, err
}
@@ -60,7 +61,7 @@
for _, nodeType := range []string{"branches", "tags"} {
var nodes map[string]bitbucketNode
- if _, err := c.getJSON(expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/{0}", match, nodeType), &nodes); err != nil {
return nil, err
}
for t, n := range nodes {
@@ -88,7 +89,7 @@
}
if repo == nil {
- repo, err = getBitbucketRepo(c, match)
+ repo, err = getBitbucketRepo(ctx, c, match)
if err != nil {
return nil, err
}
@@ -101,7 +102,7 @@
}
}
- if _, err := c.getJSON(expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), &contents); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/src/{tag}{dir}/", match), &contents); err != nil {
return nil, err
}
@@ -116,7 +117,7 @@
}
}
- if err := c.getFiles(dataURLs, files); err != nil {
+ if err := c.getFiles(ctx, dataURLs, files); err != nil {
return nil, err
}
@@ -141,9 +142,9 @@
}, nil
}
-func getBitbucketRepo(c *httpClient, match map[string]string) (*bitbucketRepo, error) {
+func getBitbucketRepo(ctx context.Context, c *httpClient, match map[string]string) (*bitbucketRepo, error) {
var repo bitbucketRepo
- if _, err := c.getJSON(expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.bitbucket.org/1.0/repositories/{owner}/{repo}", match), &repo); err != nil {
return nil, err
}
diff --git a/gosrc/client.go b/gosrc/client.go
index 09d54e3..2f4663f 100644
--- a/gosrc/client.go
+++ b/gosrc/client.go
@@ -7,6 +7,7 @@
package gosrc
import (
+ "context"
"encoding/json"
"fmt"
"io"
@@ -31,11 +32,12 @@
}
// get issues a GET to the specified URL.
-func (c *httpClient) get(url string) (*http.Response, error) {
+func (c *httpClient) get(ctx context.Context, url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
+
for k, vs := range c.header {
req.Header[k] = vs
}
@@ -66,8 +68,8 @@
return resp, err
}
-func (c *httpClient) getBytes(url string) ([]byte, error) {
- resp, err := c.get(url)
+func (c *httpClient) getBytes(ctx context.Context, url string) ([]byte, error) {
+ resp, err := c.get(ctx, url)
if err != nil {
return nil, err
}
@@ -79,8 +81,8 @@
return p, err
}
-func (c *httpClient) getReader(url string) (io.ReadCloser, error) {
- resp, err := c.get(url)
+func (c *httpClient) getReader(ctx context.Context, url string) (io.ReadCloser, error) {
+ resp, err := c.get(ctx, url)
if err != nil {
return nil, err
}
@@ -92,8 +94,8 @@
return resp.Body, nil
}
-func (c *httpClient) getJSON(url string, v interface{}) (*http.Response, error) {
- resp, err := c.get(url)
+func (c *httpClient) getJSON(ctx context.Context, url string, v interface{}) (*http.Response, error) {
+ resp, err := c.get(ctx, url)
if err != nil {
return resp, err
}
@@ -108,11 +110,11 @@
return resp, err
}
-func (c *httpClient) getFiles(urls []string, files []*File) error {
+func (c *httpClient) getFiles(ctx context.Context, urls []string, files []*File) error {
ch := make(chan error, len(files))
for i := range files {
go func(i int) {
- resp, err := c.get(urls[i])
+ resp, err := c.get(ctx, urls[i])
if err != nil {
ch <- err
return
diff --git a/gosrc/github.go b/gosrc/github.go
index 6445327..e1d9528 100644
--- a/gosrc/github.go
+++ b/gosrc/github.go
@@ -7,6 +7,7 @@
package gosrc
import (
+ "context"
"encoding/json"
"fmt"
"net/http"
@@ -57,7 +58,7 @@
return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: (%s)", resp.StatusCode, resp.Request.URL.String())}
}
-func getGitHubDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+func getGitHubDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
c := &httpClient{client: client, errFn: gitHubError}
@@ -69,7 +70,7 @@
DefaultBranch string `json:"default_branch"`
}
- if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
return nil, err
}
@@ -79,7 +80,7 @@
if match["dir"] != "" {
u += fmt.Sprintf("?path=%s", url.QueryEscape(match["dir"]))
}
- if _, err := c.getJSON(u, &commits); err != nil {
+ if _, err := c.getJSON(ctx, u, &commits); err != nil {
return nil, err
}
if len(commits) == 0 {
@@ -110,7 +111,7 @@
HTMLURL string `json:"html_url"`
}
- if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}", match), &contents); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}", match), &contents); err != nil {
// The GitHub content API returns array values for directories
// and object values for files. If there's a type mismatch at
// the beginning of the response, then assume that the path is
@@ -153,7 +154,7 @@
}
c.header = gitHubRawHeader
- if err := c.getFiles(dataURLs, files); err != nil {
+ if err := c.getFiles(ctx, dataURLs, files); err != nil {
return nil, err
}
@@ -200,18 +201,18 @@
return n < 3
}
-func getGitHubPresentation(client *http.Client, match map[string]string) (*Presentation, error) {
+func getGitHubPresentation(ctx context.Context, client *http.Client, match map[string]string) (*Presentation, error) {
c := &httpClient{client: client, header: gitHubRawHeader}
var repo struct {
DefaultBranch string `json:"default_branch"`
}
- if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
return nil, err
}
branch := repo.DefaultBranch
- p, err := c.getBytes(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}/{file}", match))
+ p, err := c.getBytes(ctx, expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}/{file}", match))
if err != nil {
return nil, err
}
@@ -242,7 +243,7 @@
files = append(files, &File{Name: fname})
dataURLs = append(dataURLs, u.String())
}
- err := c.getFiles(dataURLs, files)
+ err := c.getFiles(ctx, dataURLs, files)
return files, err
},
resolveURL: func(fname string) string {
@@ -262,7 +263,7 @@
// GetGitHubUpdates returns the full names ("owner/repo") of recently pushed GitHub repositories.
// by pushedAfter.
-func GetGitHubUpdates(client *http.Client, pushedAfter string) (maxPushedAt string, names []string, err error) {
+func GetGitHubUpdates(ctx context.Context, client *http.Client, pushedAfter string) (maxPushedAt string, names []string, err error) {
c := httpClient{client: client, header: gitHubPreviewHeader}
if pushedAfter == "" {
@@ -275,7 +276,7 @@
PushedAt string `json:"pushed_at"`
}
}
- _, err = c.getJSON(u, &updates)
+ _, err = c.getJSON(ctx, u, &updates)
if err != nil {
return pushedAfter, nil, err
}
@@ -290,14 +291,14 @@
return maxPushedAt, names, nil
}
-func getGitHubProject(client *http.Client, match map[string]string) (*Project, error) {
+func getGitHubProject(ctx context.Context, client *http.Client, match map[string]string) (*Project, error) {
c := &httpClient{client: client, errFn: gitHubError}
var repo struct {
Description string
}
- if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil {
return nil, err
}
@@ -306,7 +307,7 @@
}, nil
}
-func getGistDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+func getGistDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
c := &httpClient{client: client, errFn: gitHubError}
var gist struct {
@@ -319,7 +320,7 @@
}
}
- if _, err := c.getJSON(expand("https://api.github.com/gists/{gist}", match), &gist); err != nil {
+ if _, err := c.getJSON(ctx, expand("https://api.github.com/gists/{gist}", match), &gist); err != nil {
return nil, err
}
diff --git a/gosrc/golang.go b/gosrc/golang.go
index faed15a..d075349 100644
--- a/gosrc/golang.go
+++ b/gosrc/golang.go
@@ -7,6 +7,7 @@
package gosrc
import (
+ "context"
"errors"
"net/http"
"regexp"
@@ -18,11 +19,11 @@
golangFileRe = regexp.MustCompile(`<a href="([^"]+)"`)
)
-func getStandardDir(client *http.Client, importPath string, savedEtag string) (*Directory, error) {
+func getStandardDir(ctx context.Context, client *http.Client, importPath string, savedEtag string) (*Directory, error) {
c := &httpClient{client: client}
browseURL := "https://golang.org/src/" + importPath + "/"
- p, err := c.getBytes(browseURL)
+ p, err := c.getBytes(ctx, browseURL)
if err != nil {
return nil, err
}
@@ -47,7 +48,7 @@
}
}
- if err := c.getFiles(dataURLs, files); err != nil {
+ if err := c.getFiles(ctx, dataURLs, files); err != nil {
return nil, err
}
diff --git a/gosrc/google.go b/gosrc/google.go
index 3a2fc36..693f766 100644
--- a/gosrc/google.go
+++ b/gosrc/google.go
@@ -7,6 +7,7 @@
package gosrc
import (
+ "context"
"errors"
"net/http"
"net/url"
@@ -30,7 +31,7 @@
googleFileRe = regexp.MustCompile(`<li><a href="([^"]+)"`)
)
-func checkGoogleRedir(c *httpClient, match map[string]string) error {
+func checkGoogleRedir(ctx context.Context, c *httpClient, match map[string]string) error {
resp, err := c.getNoFollow(expand("https://code.google.com/{pr}/{repo}/", match))
if err != nil {
return err
@@ -48,22 +49,22 @@
return c.err(resp)
}
-func getGoogleDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+func getGoogleDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
setupGoogleMatch(match)
c := &httpClient{client: client}
- if err := checkGoogleRedir(c, match); err != nil {
+ if err := checkGoogleRedir(ctx, c, match); err != nil {
return nil, err
}
if m := googleEtagRe.FindStringSubmatch(savedEtag); m != nil {
match["vcs"] = m[1]
- } else if err := getGoogleVCS(c, match); err != nil {
+ } else if err := getGoogleVCS(ctx, c, match); err != nil {
return nil, err
}
// Scrape the repo browser to find the project revision and individual Go files.
- p, err := c.getBytes(expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match))
+ p, err := c.getBytes(ctx, expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/", match))
if err != nil {
return nil, err
}
@@ -95,7 +96,7 @@
}
}
- if err := c.getFiles(dataURLs, files); err != nil {
+ if err := c.getFiles(ctx, dataURLs, files); err != nil {
return nil, err
}
@@ -128,9 +129,9 @@
}
}
-func getGoogleVCS(c *httpClient, match map[string]string) error {
+func getGoogleVCS(ctx context.Context, c *httpClient, match map[string]string) error {
// Scrape the HTML project page to find the VCS.
- p, err := c.getBytes(expand("http://code.google.com/{pr}/{repo}/source/checkout", match))
+ p, err := c.getBytes(ctx, expand("http://code.google.com/{pr}/{repo}/source/checkout", match))
if err != nil {
return err
}
@@ -142,11 +143,11 @@
return nil
}
-func getGooglePresentation(client *http.Client, match map[string]string) (*Presentation, error) {
+func getGooglePresentation(ctx context.Context, client *http.Client, match map[string]string) (*Presentation, error) {
c := &httpClient{client: client}
setupGoogleMatch(match)
- if err := getGoogleVCS(c, match); err != nil {
+ if err := getGoogleVCS(ctx, c, match); err != nil {
return nil, err
}
@@ -155,7 +156,7 @@
return nil, err
}
- p, err := c.getBytes(expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{file}", match))
+ p, err := c.getBytes(ctx, expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{file}", match))
if err != nil {
return nil, err
}
@@ -174,7 +175,7 @@
files = append(files, &File{Name: fname})
dataURLs = append(dataURLs, u.String())
}
- err := c.getFiles(dataURLs, files)
+ err := c.getFiles(ctx, dataURLs, files)
return files, err
},
resolveURL: func(fname string) string {
diff --git a/gosrc/gosrc.go b/gosrc/gosrc.go
index 68181df..38005ec 100644
--- a/gosrc/gosrc.go
+++ b/gosrc/gosrc.go
@@ -8,6 +8,7 @@
package gosrc
import (
+ "context"
"encoding/xml"
"errors"
"fmt"
@@ -150,9 +151,9 @@
type service struct {
pattern *regexp.Regexp
prefix string
- get func(*http.Client, map[string]string, string) (*Directory, error)
- getPresentation func(*http.Client, map[string]string) (*Presentation, error)
- getProject func(*http.Client, map[string]string) (*Project, error)
+ get func(context.Context, *http.Client, map[string]string, string) (*Directory, error)
+ getPresentation func(context.Context, *http.Client, map[string]string) (*Presentation, error)
+ getProject func(context.Context, *http.Client, map[string]string) (*Project, error)
}
var services []*service
@@ -224,7 +225,7 @@
return ""
}
-func fetchMeta(client *http.Client, importPath string) (scheme string, im *importMeta, sm *sourceMeta, redir bool, err error) {
+func fetchMeta(ctx context.Context, client *http.Client, importPath string) (scheme string, im *importMeta, sm *sourceMeta, redir bool, err error) {
uri := importPath
if !strings.Contains(uri, "/") {
// Add slash for root of domain.
@@ -234,13 +235,13 @@
c := httpClient{client: client}
scheme = "https"
- resp, err := c.get(scheme + "://" + uri)
+ resp, err := c.get(ctx, scheme+"://"+uri)
if err != nil || resp.StatusCode != 200 {
if err == nil {
resp.Body.Close()
}
scheme = "http"
- resp, err = c.get(scheme + "://" + uri)
+ resp, err = c.get(ctx, scheme+"://"+uri)
if err != nil {
return scheme, nil, nil, false, err
}
@@ -340,20 +341,20 @@
// getVCSDirFn is called by getDynamic to fetch source using VCS commands. The
// default value here does nothing. If the code is not built for App Engine,
// then getVCSDirFn is set getVCSDir, the function that actually does the work.
-var getVCSDirFn = func(client *http.Client, m map[string]string, etag string) (*Directory, error) {
+var getVCSDirFn = func(ctx context.Context, client *http.Client, m map[string]string, etag string) (*Directory, error) {
return nil, errNoMatch
}
// getDynamic gets a directory from a service that is not statically known.
-func getDynamic(client *http.Client, importPath, etag string) (*Directory, error) {
- metaProto, im, sm, redir, err := fetchMeta(client, importPath)
+func getDynamic(ctx context.Context, client *http.Client, importPath, etag string) (*Directory, error) {
+ metaProto, im, sm, redir, err := fetchMeta(ctx, client, importPath)
if err != nil {
return nil, err
}
if im.projectRoot != importPath {
var imRoot *importMeta
- metaProto, imRoot, _, redir, err = fetchMeta(client, im.projectRoot)
+ metaProto, imRoot, _, redir, err = fetchMeta(ctx, client, im.projectRoot)
if err != nil {
return nil, err
}
@@ -376,7 +377,7 @@
dirName := importPath[len(im.projectRoot):]
resolvedPath := repo + dirName
- dir, err := getStatic(client, resolvedPath, etag)
+ dir, err := getStatic(ctx, client, resolvedPath, etag)
if err == errNoMatch {
resolvedPath = repo + "." + im.vcs + dirName
match := map[string]string{
@@ -387,7 +388,7 @@
"scheme": proto,
"vcs": im.vcs,
}
- dir, err = getVCSDirFn(client, match, etag)
+ dir, err = getVCSDirFn(ctx, client, match, etag)
}
if err != nil || dir == nil {
return nil, err
@@ -442,7 +443,7 @@
// getStatic gets a diretory from a statically known service. getStatic
// returns errNoMatch if the import path is not recognized.
-func getStatic(client *http.Client, importPath, etag string) (*Directory, error) {
+func getStatic(ctx context.Context, client *http.Client, importPath, etag string) (*Directory, error) {
for _, s := range services {
if s.get == nil {
continue
@@ -452,7 +453,7 @@
return nil, err
}
if match != nil {
- dir, err := s.get(client, match, etag)
+ dir, err := s.get(ctx, client, match, etag)
if dir != nil {
dir.ImportPath = importPath
dir.ResolvedPath = importPath
@@ -463,16 +464,16 @@
return nil, errNoMatch
}
-func Get(client *http.Client, importPath string, etag string) (dir *Directory, err error) {
+func Get(ctx context.Context, client *http.Client, importPath string, etag string) (dir *Directory, err error) {
switch {
case localPath != "":
dir, err = getLocal(importPath)
case IsGoRepoPath(importPath):
- dir, err = getStandardDir(client, importPath, etag)
+ dir, err = getStandardDir(ctx, client, importPath, etag)
case IsValidRemotePath(importPath):
- dir, err = getStatic(client, importPath, etag)
+ dir, err = getStatic(ctx, client, importPath, etag)
if err == errNoMatch {
- dir, err = getDynamic(client, importPath, etag)
+ dir, err = getDynamic(ctx, client, importPath, etag)
}
default:
err = errNoMatch
@@ -486,7 +487,7 @@
}
// GetPresentation gets a presentation from the the given path.
-func GetPresentation(client *http.Client, importPath string) (*Presentation, error) {
+func GetPresentation(ctx context.Context, client *http.Client, importPath string) (*Presentation, error) {
ext := path.Ext(importPath)
if ext != ".slide" && ext != ".article" {
return nil, NotFoundError{Message: "unknown file extension."}
@@ -504,14 +505,14 @@
}
if match != nil {
match["file"] = file
- return s.getPresentation(client, match)
+ return s.getPresentation(ctx, client, match)
}
}
return nil, NotFoundError{Message: "path does not match registered service"}
}
// GetProject gets information about a repository.
-func GetProject(client *http.Client, importPath string) (*Project, error) {
+func GetProject(ctx context.Context, client *http.Client, importPath string) (*Project, error) {
for _, s := range services {
if s.getProject == nil {
continue
@@ -521,7 +522,7 @@
return nil, err
}
if match != nil {
- return s.getProject(client, match)
+ return s.getProject(ctx, client, match)
}
}
return nil, NotFoundError{Message: "path does not match registered service"}
diff --git a/gosrc/launchpad.go b/gosrc/launchpad.go
index 3d561c8..a8c7714 100644
--- a/gosrc/launchpad.go
+++ b/gosrc/launchpad.go
@@ -10,6 +10,7 @@
"archive/tar"
"bytes"
"compress/gzip"
+ "context"
"crypto/md5"
"encoding/hex"
"io"
@@ -41,11 +42,11 @@
copy(p[j*md5.Size:], temp[:])
}
-func getLaunchpadDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
+func getLaunchpadDir(ctx context.Context, client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
c := &httpClient{client: client}
if match["project"] != "" && match["series"] != "" {
- rc, err := c.getReader(expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match))
+ rc, err := c.getReader(ctx, expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match))
switch {
case err == nil:
rc.Close()
@@ -59,7 +60,7 @@
}
}
- p, err := c.getBytes(expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match))
+ p, err := c.getBytes(ctx, expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match))
if err != nil {
return nil, err
}
diff --git a/gosrc/vcs.go b/gosrc/vcs.go
index bd7cc06..f21670d 100644
--- a/gosrc/vcs.go
+++ b/gosrc/vcs.go
@@ -10,6 +10,7 @@
import (
"bytes"
+ "context"
"errors"
"io/ioutil"
"log"
@@ -245,7 +246,7 @@
return "", NotFoundError{Message: "Last changed revision not found"}
}
-func getVCSDir(client *http.Client, match map[string]string, etagSaved string) (*Directory, error) {
+func getVCSDir(ctx context.Context, client *http.Client, match map[string]string, etagSaved string) (*Directory, error) {
cmd := vcsCmds[match["vcs"]]
if cmd == nil {
return nil, NotFoundError{Message: expand("VCS not supported: {vcs}", match)}
