gddo-server: Use memcache to cache HTTP transport.
The memcache underneath is using the gomemcache package, which can be
used both as a standalone app or in App Engine. The cache will reduce a
significant amount of HTTP requests we make when updating packages. Most
of the requests are to GitHub. So this change will save a lot against
GitHub rate limits.
Change-Id: I46c7c4bd658b7b2ab18ca67e8c8d25549181dfb4
Reviewed-on: https://go-review.googlesource.com/24326
Reviewed-by: Alan Donovan <adonovan@google.com>
Reviewed-by: Andrew Gerrand <adg@golang.org>
diff --git a/gddo-server/client.go b/gddo-server/client.go
index 75b7c6f..bd40e7e 100644
--- a/gddo-server/client.go
+++ b/gddo-server/client.go
@@ -11,10 +11,15 @@
import (
"flag"
+ "fmt"
"net"
"net/http"
+ "os"
"time"
+ "github.com/gregjones/httpcache"
+ "github.com/gregjones/httpcache/memcache"
+
"github.com/golang/gddo/httputil"
)
@@ -24,18 +29,36 @@
)
func newHTTPClient() *http.Client {
- return &http.Client{
- Transport: httputil.NewAuthTransport(
- &http.Transport{
- Proxy: http.ProxyFromEnvironment,
- Dial: (&net.Dialer{
- Timeout: *dialTimeout,
- KeepAlive: *requestTimeout / 2,
- }).Dial,
- ResponseHeaderTimeout: *requestTimeout / 2,
- TLSHandshakeTimeout: *requestTimeout / 2,
- },
- ),
- Timeout: *requestTimeout,
+ t := newCacheTransport()
+ t.Transport = &http.Transport{
+ Proxy: http.ProxyFromEnvironment,
+ Dial: (&net.Dialer{
+ Timeout: *dialTimeout,
+ KeepAlive: *requestTimeout / 2,
+ }).Dial,
+ ResponseHeaderTimeout: *requestTimeout / 2,
+ TLSHandshakeTimeout: *requestTimeout / 2,
}
+ return &http.Client{
+ // Wrap the cached transport with GitHub authentication.
+ Transport: httputil.NewAuthTransport(t),
+ Timeout: *requestTimeout,
+ }
+}
+
+func newCacheTransport() *httpcache.Transport {
+ // host and port are set by GAE Flex runtime, can be left blank locally.
+ host := os.Getenv("MEMCACHE_PORT_11211_TCP_ADDR")
+ if host == "" {
+ host = "localhost"
+ }
+ port := os.Getenv("MEMCACHE_PORT_11211_TCP_PORT")
+ if port == "" {
+ port = "11211"
+ }
+ addr := fmt.Sprintf("%s:%s", host, port)
+
+ return httpcache.NewTransport(
+ memcache.New(addr),
+ )
}
diff --git a/gosrc/bitbucket.go b/gosrc/bitbucket.go
index 3593cc5..c89feb3 100644
--- a/gosrc/bitbucket.go
+++ b/gosrc/bitbucket.go
@@ -30,7 +30,7 @@
ForkOf struct {
Scm string
} `json:"fork_of"`
- Followers int `json:"followers"`
+ Followers int `json:"followers_count"`
IsFork bool `json:"is_fork"`
}
diff --git a/gosrc/github.go b/gosrc/github.go
index e02fb32..ad8f4a3 100644
--- a/gosrc/github.go
+++ b/gosrc/github.go
@@ -162,7 +162,7 @@
var repo = struct {
Fork bool `json:"fork"`
- Stars int `json:"stars"`
+ Stars int `json:"stargazers_count"`
CreatedAt time.Time `json:"created_at"`
PushedAt time.Time `json:"pushed_at"`
}{}
