Use Redis SCAN command in db.Do().
diff --git a/database/database.go b/database/database.go
index e15339a..5e4ef6b 100644
--- a/database/database.go
+++ b/database/database.go
@@ -714,7 +714,6 @@
type PackageInfo struct {
PDoc *doc.Package
- Pkgs []Package
Score float64
Kind string
Size int
@@ -724,48 +723,62 @@
func (db *Database) Do(f func(*PackageInfo) error) error {
c := db.Pool.Get()
defer c.Close()
- keys, err := redis.Values(c.Do("KEYS", "pkg:*"))
- if err != nil {
- return err
- }
- for _, key := range keys {
- values, err := redis.Values(c.Do("HMGET", key, "gob", "score", "kind", "path", "terms", "synopis"))
+ cursor := 0
+ c.Send("SCAN", cursor, "MATCH", "pkg:*")
+ c.Flush()
+ for {
+ // Recieve previous SCAN.
+ values, err := redis.Values(c.Receive())
if err != nil {
return err
}
-
- var (
- pi PackageInfo
- p []byte
- path string
- terms string
- synopsis string
- )
-
- if _, err := redis.Scan(values, &p, &pi.Score, &pi.Kind, &path, &terms, &synopsis); err != nil {
+ var keys [][]byte
+ if _, err := redis.Scan(values, &cursor, &keys); err != nil {
return err
}
-
- if p == nil {
- continue
+ if cursor == 0 {
+ break
}
-
- pi.Size = len(path) + len(p) + len(terms) + len(synopsis)
-
- p, err = snappy.Decode(nil, p)
- if err != nil {
- return fmt.Errorf("snappy decoding %s: %v", path, err)
+ for _, key := range keys {
+ c.Send("HMGET", key, "gob", "score", "kind", "path", "terms", "synopis")
}
+ c.Send("SCAN", cursor, "MATCH", "pkg:*")
+ c.Flush()
+ for _ = range keys {
+ values, err := redis.Values(c.Receive())
+ if err != nil {
+ return err
+ }
- if err := gob.NewDecoder(bytes.NewReader(p)).Decode(&pi.PDoc); err != nil {
- return fmt.Errorf("gob decoding %s: %v", path, err)
- }
- pi.Pkgs, err = db.getSubdirs(c, pi.PDoc.ImportPath, pi.PDoc)
- if err != nil {
- return fmt.Errorf("get subdirs %s: %v", path, err)
- }
- if err := f(&pi); err != nil {
- return fmt.Errorf("func %s: %v", path, err)
+ var (
+ pi PackageInfo
+ p []byte
+ path string
+ terms string
+ synopsis string
+ )
+
+ if _, err := redis.Scan(values, &p, &pi.Score, &pi.Kind, &path, &terms, &synopsis); err != nil {
+ return err
+ }
+
+ if p == nil {
+ continue
+ }
+
+ pi.Size = len(path) + len(p) + len(terms) + len(synopsis)
+
+ p, err = snappy.Decode(nil, p)
+ if err != nil {
+ return fmt.Errorf("snappy decoding %s: %v", path, err)
+ }
+
+ if err := gob.NewDecoder(bytes.NewReader(p)).Decode(&pi.PDoc); err != nil {
+ return fmt.Errorf("gob decoding %s: %v", path, err)
+ }
+ if err := f(&pi); err != nil {
+ return fmt.Errorf("func %s: %v", path, err)
+ }
}
}
return nil
