client: only fetch necessary packages when cache missing
When a httpSource is used without a Cache implementation, only fetch
packages which are represented in the index. Previously if there
wasn't a Cache implementation every pacakge was enumerated, ignoring
the contents of the retrieved index.
Change-Id: I8e8e0ce412b3ded188afd6bb109d96efb7e7f27c
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/333455
Trust: Roland Shoemaker <roland@golang.org>
Trust: Zvonimir Pavlinovic <zpavlinovic@google.com>
Run-TryBot: Roland Shoemaker <roland@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/client/client.go b/client/client.go
index c7a22a0..acf29d3 100644
--- a/client/client.go
+++ b/client/client.go
@@ -158,12 +158,12 @@
}
var stillNeed []string
- if hs.cache != nil {
- for _, p := range packages {
- lastModified, present := index[p]
- if !present {
- continue
- }
+ for _, p := range packages {
+ lastModified, present := index[p]
+ if !present {
+ continue
+ }
+ if hs.cache != nil {
if cached, err := hs.cache.ReadEntries(hs.dbName, p); err != nil {
return nil, err
} else if len(cached) != 0 {
@@ -179,10 +179,8 @@
continue
}
}
- stillNeed = append(stillNeed, p)
}
- } else {
- stillNeed = packages
+ stillNeed = append(stillNeed, p)
}
for _, p := range stillNeed {
diff --git a/client/client_test.go b/client/client_test.go
index 1000797..c8c04cd 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -5,12 +5,15 @@
package client
import (
+ "encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
+ "net/http/httptest"
"os"
"path"
+ "reflect"
"runtime"
"testing"
"time"
@@ -156,3 +159,30 @@
}
}
}
+
+func TestCorrectFetchesNoCache(t *testing.T) {
+ fetches := map[string]int{}
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fetches[r.URL.Path]++
+ if r.URL.Path == "/index.json" {
+ j, _ := json.Marshal(osv.DBIndex{
+ "a": time.Now(),
+ "b": time.Now(),
+ })
+ w.Write(j)
+ } else {
+ w.Write([]byte("[]"))
+ }
+ }))
+ defer ts.Close()
+
+ hs := &httpSource{url: ts.URL, c: new(http.Client)}
+ _, err := hs.Get([]string{"a", "b", "c"})
+ if err != nil {
+ t.Fatalf("unexpected error: %s", err)
+ }
+ expectedFetches := map[string]int{"/index.json": 1, "/a.json": 1, "/b.json": 1}
+ if !reflect.DeepEqual(fetches, expectedFetches) {
+ t.Errorf("unexpected fetches, got %v, want %v", fetches, expectedFetches)
+ }
+}
