gddo-admin: use the new database API for admin commands

Use default context.TODO() when initializing database for usage without
remote_api for search. This CL mainly fixes the gddo-admin commands.

Change-Id: I8518f3aeff57d3e733a99a45c50177cffaf35621
Reviewed-on: https://go-review.googlesource.com/57730
Reviewed-by: Ross Light <light@google.com>

database/database.go

diff --git a/database/database.go b/database/database.go
index 32fa2f2..0eea7c8 100644
--- a/database/database.go
+++ b/database/database.go
@@ -132,12 +132,12 @@
 	return remote_api.NewRemoteContext(host, client)
 }
 
-// New creates a gddo database. serverUri, idleTimeout, and logConn configure
+// New creates a gddo database. serverURI, idleTimeout, and logConn configure
 // the use of redis. gaeEndpoint is the target of the App Engine remoteapi
 // endpoint.
-func New(serverUri string, idleTimeout time.Duration, logConn bool, gaeEndpoint string) (*Database, error) {
+func New(serverURI string, idleTimeout time.Duration, logConn bool, gaeEndpoint string) (*Database, error) {
 	pool := &redis.Pool{
-		Dial:        newDBDialer(serverUri, logConn),
+		Dial:        newDBDialer(serverURI, logConn),
 		MaxIdle:     10,
 		IdleTimeout: idleTimeout,
 	}
@@ -148,9 +148,12 @@
 	}
 	c.Close()
 
-	gaeCtx, err := newAppEngineContext(gaeEndpoint)
-	if err != nil {
-		return nil, err
+	gaeCtx := context.TODO()
+	if gaeEndpoint != "" {
+		var err error
+		if gaeCtx, err = newAppEngineContext(gaeEndpoint); err != nil {
+			return nil, err
+		}
 	}
 
 	return &Database{Pool: pool, AppEngineContext: gaeCtx}, nil
@@ -753,6 +756,8 @@
 	return db.getPackages("index:import:"+path, false)
 }
 
+// Block puts a domain, repo or package into the block set, removes all the
+// packages under it from the database and prevents future crawling from it.
 func (db *Database) Block(root string) error {
 	c := db.Pool.Get()
 	defer c.Close()
@@ -785,6 +790,8 @@
     return  0
 `)
 
+// IsBlocked returns whether the package is blocked or belongs to a blocked
+// domain/repo.
 func (db *Database) IsBlocked(path string) (bool, error) {
 	c := db.Pool.Get()
 	defer c.Close()

doc/get.go

diff --git a/doc/get.go b/doc/get.go
index 611bfd0..67a21a6 100644
--- a/doc/get.go
+++ b/doc/get.go
@@ -8,10 +8,11 @@
 package doc
 
 import (
-	"github.com/golang/gddo/gosrc"
 	"go/doc"
 	"net/http"
 	"strings"
+
+	"github.com/golang/gddo/gosrc"
 )
 
 func Get(client *http.Client, importPath string, etag string) (*Package, error) {

gddo-admin/block.go

diff --git a/gddo-admin/block.go b/gddo-admin/block.go
index 70fe4c1..589cba7 100644
--- a/gddo-admin/block.go
+++ b/gddo-admin/block.go
@@ -7,9 +7,10 @@
 package main
 
 import (
-	"github.com/golang/gddo/database"
 	"log"
 	"os"
+
+	"github.com/golang/gddo/database"
 )
 
 var blockCommand = &command{
@@ -23,7 +24,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-admin/crawl.go

diff --git a/gddo-admin/crawl.go b/gddo-admin/crawl.go
index 62ebe71..4537790 100644
--- a/gddo-admin/crawl.go
+++ b/gddo-admin/crawl.go
@@ -28,7 +28,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-admin/dangle.go

diff --git a/gddo-admin/dangle.go b/gddo-admin/dangle.go
index b0473f6..0c34286 100644
--- a/gddo-admin/dangle.go
+++ b/gddo-admin/dangle.go
@@ -26,7 +26,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-admin/delete.go

diff --git a/gddo-admin/delete.go b/gddo-admin/delete.go
index a9d188e..0ac2418 100644
--- a/gddo-admin/delete.go
+++ b/gddo-admin/delete.go
@@ -24,7 +24,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-admin/main.go

diff --git a/gddo-admin/main.go b/gddo-admin/main.go
index 223c94c..47f19af 100644
--- a/gddo-admin/main.go
+++ b/gddo-admin/main.go
@@ -12,6 +12,7 @@
 	"fmt"
 	"os"
 	"strings"
+	"time"
 )
 
 type command struct {
@@ -26,6 +27,14 @@
 	c.flag.PrintDefaults()
 }
 
+var (
+	project       = flag.String("project", "", "App Engine project ID used to interact with remote API.")
+	redisServer   = flag.String("db-server", "redis://127.0.0.1:6379", "URI of Redis server.")
+	dbIdleTimeout = flag.Duration("db-idle-timeout", 250*time.Second, "Close database connections after remaining idle for this duration.")
+)
+
+var gaeEndpoint string
+
 var commands = []*command{
 	blockCommand,
 	reindexCommand,
@@ -48,10 +57,18 @@
 	}
 }
 
+func setDefaults() {
+	if *project != "" {
+		gaeEndpoint = fmt.Sprintf("serviceproxy-dot-%s.appspot.com", *project)
+	}
+}
+
 func main() {
 	flag.Usage = printUsage
 	flag.Parse()
 	args := flag.Args()
+	setDefaults()
+
 	if len(args) >= 1 {
 		for _, c := range commands {
 			if args[0] == c.name {

gddo-admin/popular.go

diff --git a/gddo-admin/popular.go b/gddo-admin/popular.go
index cdb678d..f698a5e 100644
--- a/gddo-admin/popular.go
+++ b/gddo-admin/popular.go
@@ -30,7 +30,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-admin/reindex.go

diff --git a/gddo-admin/reindex.go b/gddo-admin/reindex.go
index 3c0210f..4b3fb88 100644
--- a/gddo-admin/reindex.go
+++ b/gddo-admin/reindex.go
@@ -51,7 +51,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-admin/stats.go

diff --git a/gddo-admin/stats.go b/gddo-admin/stats.go
index 987087f..07d35b0 100644
--- a/gddo-admin/stats.go
+++ b/gddo-admin/stats.go
@@ -37,7 +37,7 @@
 		c.printUsage()
 		os.Exit(1)
 	}
-	db, err := database.New()
+	db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
 	if err != nil {
 		log.Fatal(err)
 	}

gddo-server/config.go

diff --git a/gddo-server/config.go b/gddo-server/config.go
index fbda23e..5f07548 100644
--- a/gddo-server/config.go
+++ b/gddo-server/config.go
@@ -89,8 +89,10 @@
 func setDefaults() {
 	// ConfigGAERemoteAPI is based on project.
 	project := viper.GetString(ConfigProject)
-	defaultEndpoint := fmt.Sprintf("serviceproxy-dot-%s.appspot.com", project)
-	viper.SetDefault(ConfigGAERemoteAPI, defaultEndpoint)
+	if project != "" {
+		defaultEndpoint := fmt.Sprintf("serviceproxy-dot-%s.appspot.com", project)
+		viper.SetDefault(ConfigGAERemoteAPI, defaultEndpoint)
+	}
 }
 
 func buildFlags() *pflag.FlagSet {

gddo-server/crawl.go

diff --git a/gddo-server/crawl.go b/gddo-server/crawl.go
index 9afdc00..959ab22 100644
--- a/gddo-server/crawl.go
+++ b/gddo-server/crawl.go
@@ -20,8 +20,7 @@
 )
 
 var (
-	testdataPat      = regexp.MustCompile(`/testdata(?:/|$)`)
-	blockedDomainPat = regexp.MustCompile(`^zxq\.co/`)
+	testdataPat = regexp.MustCompile(`/testdata(?:/|$)`)
 )
 
 // crawlDoc fetches the package documentation from the VCS and updates the database.
@@ -54,9 +53,6 @@
 	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
 		pdoc = nil
 		err = gosrc.NotFoundError{Message: "blocked."}
-	} else if blockedDomainPat.MatchString(importPath) {
-		pdoc = nil
-		err = gosrc.NotFoundError{Message: "blocked domain."}
 	} else if testdataPat.MatchString(importPath) {
 		pdoc = nil
 		err = gosrc.NotFoundError{Message: "testdata."}

gosrc/gosrc.go

diff --git a/gosrc/gosrc.go b/gosrc/gosrc.go
index 966821f..68181df 100644
--- a/gosrc/gosrc.go
+++ b/gosrc/gosrc.go
@@ -339,7 +339,7 @@
 
 // getVCSDirFn is called by getDynamic to fetch source using VCS commands. The
 // default value here does nothing. If the code is not built for App Engine,
-// then getvCSDirFn is set getVCSDir, the function that actually does the work.
+// then getVCSDirFn is set getVCSDir, the function that actually does the work.
 var getVCSDirFn = func(client *http.Client, m map[string]string, etag string) (*Directory, error) {
 	return nil, errNoMatch
 }