gddo-server: use App Engine remoteapi for search.
App Engine Flexible Environment does not allow access to the Search API. To
temporarily get around this limitation, we are adding support for using an
App Engine standard application as a proxy. This will support future work
getting rid of App Engine specific code from the codebase entirely.
Change-Id: Id0537f4a86d1a588601e3d97d30f2be906ab23cf
Reviewed-on: https://go-review.googlesource.com/38397
Reviewed-by: Tuo Shan <shantuo@google.com>
diff --git a/database/database.go b/database/database.go
index c6fb1e4..234a2e6 100644
--- a/database/database.go
+++ b/database/database.go
@@ -48,7 +48,9 @@
"github.com/garyburd/redigo/redis"
"github.com/golang/snappy"
"golang.org/x/net/context"
+ "golang.org/x/oauth2/google"
"google.golang.org/appengine"
+ "google.golang.org/appengine/remote_api"
"google.golang.org/appengine/search"
"github.com/golang/gddo/doc"
@@ -59,6 +61,8 @@
Pool interface {
Get() redis.Conn
}
+
+ AppEngineContext context.Context
}
// Package represents the content of a package both for the search index and
@@ -117,8 +121,21 @@
}
}
-// New creates a gddo database
-func New(serverUri string, idleTimeout time.Duration, logConn bool) (*Database, error) {
+func newAppEngineContext(host string) (context.Context, error) {
+ client, err := google.DefaultClient(context.TODO(),
+ "https://www.googleapis.com/auth/appengine.apis",
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return remote_api.NewRemoteContext(host, client)
+}
+
+// New creates a gddo database. serverUri, idleTimeout, and logConn configure
+// the use of redis. gaeEndpoint is the target of the App Engine remoteapi
+// endpoint.
+func New(serverUri string, idleTimeout time.Duration, logConn bool, gaeEndpoint string) (*Database, error) {
pool := &redis.Pool{
Dial: newDBDialer(serverUri, logConn),
MaxIdle: 10,
@@ -131,7 +148,12 @@
}
c.Close()
- return &Database{Pool: pool}, nil
+ gaeCtx, err := newAppEngineContext(gaeEndpoint)
+ if err != nil {
+ return nil, err
+ }
+
+ return &Database{Pool: pool, AppEngineContext: gaeCtx}, nil
}
// Exists returns true if package with import path exists in the database.
@@ -174,7 +196,7 @@
for term, x in pairs(update) do
if x == 1 then
redis.call('SREM', 'index:' .. term, id)
- elseif x == 2 then
+ elseif x == 2 then
redis.call('SADD', 'index:' .. term, id)
end
end
@@ -209,7 +231,8 @@
return err
}
-var bgCtx = appengine.BackgroundContext // replaced by tests
+// TODO(stephenmw): bgCtx is not necessary anymore.
+var bgCtx = context.Background // replaced by tests
// Put adds the package documentation to the database.
func (db *Database) Put(pdoc *doc.Package, nextCrawl time.Time, hide bool) error {
@@ -278,12 +301,12 @@
ctx := bgCtx()
if score > 0 {
- if err := PutIndex(ctx, pdoc, id, score, n); err != nil {
+ if err := db.PutIndex(ctx, pdoc, id, score, n); err != nil {
log.Printf("Cannot put %q in index: %v", pdoc.ImportPath, err)
}
if old != nil {
- if err := updateImportsIndex(c, ctx, old, pdoc); err != nil {
+ if err := db.updateImportsIndex(c, ctx, old, pdoc); err != nil {
return err
}
}
@@ -342,7 +365,7 @@
return id, numImported, nil
}
-func updateImportsIndex(c redis.Conn, ctx context.Context, oldDoc, newDoc *doc.Package) error {
+func (db *Database) updateImportsIndex(c redis.Conn, ctx context.Context, oldDoc, newDoc *doc.Package) error {
// Create a map to store any import change since last time we indexed the package.
changes := make(map[string]bool)
for _, p := range oldDoc.Imports {
@@ -365,7 +388,7 @@
return err
}
if id != "" {
- PutIndex(ctx, nil, id, -1, n)
+ db.PutIndex(ctx, nil, id, -1, n)
}
}
return nil
@@ -449,13 +472,13 @@
end
local nextCrawl = redis.call('HGET', 'pkg:' .. id, 'crawl')
- if not nextCrawl then
+ if not nextCrawl then
nextCrawl = redis.call('ZSCORE', 'nextCrawl', id)
if not nextCrawl then
nextCrawl = 0
end
end
-
+
return {gob, nextCrawl}
`)
@@ -1238,3 +1261,13 @@
log.Printf("%d packages are reindexed", npkgs)
return nil
}
+
+func (db *Database) Search(ctx context.Context, q string) ([]Package, error) {
+ // TODO(stephenmw): merge ctx with AppEngineContext
+ return searchAE(db.AppEngineContext, q)
+}
+
+func (db *Database) PutIndex(ctx context.Context, pdoc *doc.Package, id string, score float64, importCount int) error {
+ // TODO(stephenmw): merge ctx with AppEngineContext
+ return putIndex(db.AppEngineContext, pdoc, id, score, importCount)
+}
diff --git a/database/indexae.go b/database/indexae.go
index 4259025..e86fa73 100644
--- a/database/indexae.go
+++ b/database/indexae.go
@@ -84,11 +84,11 @@
return fields, meta, nil
}
-// PutIndex creates or updates a package entry in the search index. id identifies the document in the index.
-// If pdoc is non-nil, PutIndex will update the package's name, path and synopsis supplied by pdoc.
-// pdoc must be non-nil for a package's first call to PutIndex.
-// PutIndex updates the Score to score, if non-negative.
-func PutIndex(c context.Context, pdoc *doc.Package, id string, score float64, importCount int) error {
+// putIndex creates or updates a package entry in the search index. id identifies the document in the index.
+// If pdoc is non-nil, putIndex will update the package's name, path and synopsis supplied by pdoc.
+// pdoc must be non-nil for a package's first call to putIndex.
+// putIndex updates the Score to score, if non-negative.
+func putIndex(c context.Context, pdoc *doc.Package, id string, score float64, importCount int) error {
if id == "" {
return errors.New("indexae: no id assigned")
}
@@ -127,9 +127,9 @@
return nil
}
-// Search searches the packages index for a given query. A path-like query string
+// searchAE searches the packages index for a given query. A path-like query string
// will be passed in unchanged, whereas single words will be stemmed.
-func Search(c context.Context, q string) ([]Package, error) {
+func searchAE(c context.Context, q string) ([]Package, error) {
index, err := search.Open("packages")
if err != nil {
return nil, err
diff --git a/gddo-server/config.go b/gddo-server/config.go
index c335fe3..fbda23e 100644
--- a/gddo-server/config.go
+++ b/gddo-server/config.go
@@ -2,6 +2,7 @@
import (
"context"
+ "fmt"
"os"
"path/filepath"
"strings"
@@ -29,6 +30,7 @@
ConfigDBServer = "db-server"
ConfigDBIdleTimeout = "db-idle-timeout"
ConfigDBLog = "db-log"
+ ConfigGAERemoteAPI = "remoteapi-endpoint"
// Display Config
ConfigSidebar = "sidebar"
@@ -75,9 +77,22 @@
// Read from config.
readViperConfig(ctx)
+ // Set defaults based on other configs
+ setDefaults()
+
log.Info(ctx, "config values loaded", "values", viper.AllSettings())
}
+// setDefaults sets defaults for configuration options that depend on other
+// configuration options. This allows for smart defaults but allows for
+// overrides.
+func setDefaults() {
+ // ConfigGAERemoteAPI is based on project.
+ project := viper.GetString(ConfigProject)
+ defaultEndpoint := fmt.Sprintf("serviceproxy-dot-%s.appspot.com", project)
+ viper.SetDefault(ConfigGAERemoteAPI, defaultEndpoint)
+}
+
func buildFlags() *pflag.FlagSet {
flags := pflag.NewFlagSet("default", pflag.ExitOnError)
@@ -102,6 +117,7 @@
flags.Duration(ConfigDBIdleTimeout, 250*time.Second, "Close Redis connections after remaining idle for this duration.")
flags.Bool(ConfigDBLog, false, "Log database commands")
flags.String(ConfigMemcacheAddr, "", "Address in the format host:port gddo uses to point to the memcache backend.")
+ flags.String(ConfigGAERemoteAPI, "", "Remoteapi endpoint for App Engine Search. Defaults to serviceproxy-dot-${project}.appspot.com.")
return flags
}
diff --git a/gddo-server/main.go b/gddo-server/main.go
index a6d9eac..f48b6c1 100644
--- a/gddo-server/main.go
+++ b/gddo-server/main.go
@@ -573,8 +573,7 @@
}
}
- ctx := appengine.NewContext(req)
- pkgs, err := database.Search(ctx, q)
+ pkgs, err := db.Search(req.Context(), q)
if err != nil {
return err
}
@@ -633,8 +632,7 @@
if pkgs == nil {
var err error
- ctx := appengine.NewContext(req)
- pkgs, err = database.Search(ctx, q)
+ pkgs, err = db.Search(req.Context(), q)
if err != nil {
return err
}
@@ -911,6 +909,7 @@
viper.GetString(ConfigDBServer),
viper.GetDuration(ConfigDBIdleTimeout),
viper.GetBool(ConfigDBLog),
+ viper.GetString(ConfigGAERemoteAPI),
)
if err != nil {
log.Fatalf("Error opening database: %v", err)
