gddo-admin: use the new database API for admin commands
Use default context.TODO() when initializing database for usage without
remote_api for search. This CL mainly fixes the gddo-admin commands.
Change-Id: I8518f3aeff57d3e733a99a45c50177cffaf35621
Reviewed-on: https://go-review.googlesource.com/57730
Reviewed-by: Ross Light <light@google.com>
diff --git a/database/database.go b/database/database.go
index 32fa2f2..0eea7c8 100644
--- a/database/database.go
+++ b/database/database.go
@@ -132,12 +132,12 @@
return remote_api.NewRemoteContext(host, client)
}
-// New creates a gddo database. serverUri, idleTimeout, and logConn configure
+// New creates a gddo database. serverURI, idleTimeout, and logConn configure
// the use of redis. gaeEndpoint is the target of the App Engine remoteapi
// endpoint.
-func New(serverUri string, idleTimeout time.Duration, logConn bool, gaeEndpoint string) (*Database, error) {
+func New(serverURI string, idleTimeout time.Duration, logConn bool, gaeEndpoint string) (*Database, error) {
pool := &redis.Pool{
- Dial: newDBDialer(serverUri, logConn),
+ Dial: newDBDialer(serverURI, logConn),
MaxIdle: 10,
IdleTimeout: idleTimeout,
}
@@ -148,9 +148,12 @@
}
c.Close()
- gaeCtx, err := newAppEngineContext(gaeEndpoint)
- if err != nil {
- return nil, err
+ gaeCtx := context.TODO()
+ if gaeEndpoint != "" {
+ var err error
+ if gaeCtx, err = newAppEngineContext(gaeEndpoint); err != nil {
+ return nil, err
+ }
}
return &Database{Pool: pool, AppEngineContext: gaeCtx}, nil
@@ -753,6 +756,8 @@
return db.getPackages("index:import:"+path, false)
}
+// Block puts a domain, repo or package into the block set, removes all the
+// packages under it from the database and prevents future crawling from it.
func (db *Database) Block(root string) error {
c := db.Pool.Get()
defer c.Close()
@@ -785,6 +790,8 @@
return 0
`)
+// IsBlocked returns whether the package is blocked or belongs to a blocked
+// domain/repo.
func (db *Database) IsBlocked(path string) (bool, error) {
c := db.Pool.Get()
defer c.Close()
diff --git a/doc/get.go b/doc/get.go
index 611bfd0..67a21a6 100644
--- a/doc/get.go
+++ b/doc/get.go
@@ -8,10 +8,11 @@
package doc
import (
- "github.com/golang/gddo/gosrc"
"go/doc"
"net/http"
"strings"
+
+ "github.com/golang/gddo/gosrc"
)
func Get(client *http.Client, importPath string, etag string) (*Package, error) {
diff --git a/gddo-admin/block.go b/gddo-admin/block.go
index 70fe4c1..589cba7 100644
--- a/gddo-admin/block.go
+++ b/gddo-admin/block.go
@@ -7,9 +7,10 @@
package main
import (
- "github.com/golang/gddo/database"
"log"
"os"
+
+ "github.com/golang/gddo/database"
)
var blockCommand = &command{
@@ -23,7 +24,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-admin/crawl.go b/gddo-admin/crawl.go
index 62ebe71..4537790 100644
--- a/gddo-admin/crawl.go
+++ b/gddo-admin/crawl.go
@@ -28,7 +28,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-admin/dangle.go b/gddo-admin/dangle.go
index b0473f6..0c34286 100644
--- a/gddo-admin/dangle.go
+++ b/gddo-admin/dangle.go
@@ -26,7 +26,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-admin/delete.go b/gddo-admin/delete.go
index a9d188e..0ac2418 100644
--- a/gddo-admin/delete.go
+++ b/gddo-admin/delete.go
@@ -24,7 +24,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-admin/main.go b/gddo-admin/main.go
index 223c94c..47f19af 100644
--- a/gddo-admin/main.go
+++ b/gddo-admin/main.go
@@ -12,6 +12,7 @@
"fmt"
"os"
"strings"
+ "time"
)
type command struct {
@@ -26,6 +27,14 @@
c.flag.PrintDefaults()
}
+var (
+ project = flag.String("project", "", "App Engine project ID used to interact with remote API.")
+ redisServer = flag.String("db-server", "redis://127.0.0.1:6379", "URI of Redis server.")
+ dbIdleTimeout = flag.Duration("db-idle-timeout", 250*time.Second, "Close database connections after remaining idle for this duration.")
+)
+
+var gaeEndpoint string
+
var commands = []*command{
blockCommand,
reindexCommand,
@@ -48,10 +57,18 @@
}
}
+func setDefaults() {
+ if *project != "" {
+ gaeEndpoint = fmt.Sprintf("serviceproxy-dot-%s.appspot.com", *project)
+ }
+}
+
func main() {
flag.Usage = printUsage
flag.Parse()
args := flag.Args()
+ setDefaults()
+
if len(args) >= 1 {
for _, c := range commands {
if args[0] == c.name {
diff --git a/gddo-admin/popular.go b/gddo-admin/popular.go
index cdb678d..f698a5e 100644
--- a/gddo-admin/popular.go
+++ b/gddo-admin/popular.go
@@ -30,7 +30,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-admin/reindex.go b/gddo-admin/reindex.go
index 3c0210f..4b3fb88 100644
--- a/gddo-admin/reindex.go
+++ b/gddo-admin/reindex.go
@@ -51,7 +51,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-admin/stats.go b/gddo-admin/stats.go
index 987087f..07d35b0 100644
--- a/gddo-admin/stats.go
+++ b/gddo-admin/stats.go
@@ -37,7 +37,7 @@
c.printUsage()
os.Exit(1)
}
- db, err := database.New()
+ db, err := database.New(*redisServer, *dbIdleTimeout, false, gaeEndpoint)
if err != nil {
log.Fatal(err)
}
diff --git a/gddo-server/config.go b/gddo-server/config.go
index fbda23e..5f07548 100644
--- a/gddo-server/config.go
+++ b/gddo-server/config.go
@@ -89,8 +89,10 @@
func setDefaults() {
// ConfigGAERemoteAPI is based on project.
project := viper.GetString(ConfigProject)
- defaultEndpoint := fmt.Sprintf("serviceproxy-dot-%s.appspot.com", project)
- viper.SetDefault(ConfigGAERemoteAPI, defaultEndpoint)
+ if project != "" {
+ defaultEndpoint := fmt.Sprintf("serviceproxy-dot-%s.appspot.com", project)
+ viper.SetDefault(ConfigGAERemoteAPI, defaultEndpoint)
+ }
}
func buildFlags() *pflag.FlagSet {
diff --git a/gddo-server/crawl.go b/gddo-server/crawl.go
index 9afdc00..959ab22 100644
--- a/gddo-server/crawl.go
+++ b/gddo-server/crawl.go
@@ -20,8 +20,7 @@
)
var (
- testdataPat = regexp.MustCompile(`/testdata(?:/|$)`)
- blockedDomainPat = regexp.MustCompile(`^zxq\.co/`)
+ testdataPat = regexp.MustCompile(`/testdata(?:/|$)`)
)
// crawlDoc fetches the package documentation from the VCS and updates the database.
@@ -54,9 +53,6 @@
} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
pdoc = nil
err = gosrc.NotFoundError{Message: "blocked."}
- } else if blockedDomainPat.MatchString(importPath) {
- pdoc = nil
- err = gosrc.NotFoundError{Message: "blocked domain."}
} else if testdataPat.MatchString(importPath) {
pdoc = nil
err = gosrc.NotFoundError{Message: "testdata."}
diff --git a/gosrc/gosrc.go b/gosrc/gosrc.go
index 966821f..68181df 100644
--- a/gosrc/gosrc.go
+++ b/gosrc/gosrc.go
@@ -339,7 +339,7 @@
// getVCSDirFn is called by getDynamic to fetch source using VCS commands. The
// default value here does nothing. If the code is not built for App Engine,
-// then getvCSDirFn is set getVCSDir, the function that actually does the work.
+// then getVCSDirFn is set getVCSDir, the function that actually does the work.
var getVCSDirFn = func(client *http.Client, m map[string]string, etag string) (*Directory, error) {
return nil, errNoMatch
}
