cmd/go/internal/modfetch: check go.mod content using go.sum
go.sum (formerly go.modverify) already contains lines like
github.com/pkg/foo v1.2.3 h1:hashhashhashhashhash
where the hash is a checksum over the whole file tree of the module.
After this CL, it will also contain lines like:
github.com/pkg/foo v1.2.3/go.mod h1:hashhashhashhashhash
(note the /go.mod suffix on the version), where the hash is a checksum
over the one-file tree containing only the go.mod for that module.
This hash is recorded even for auto-generated go.mod files,
because no matter how it arises, the exact go.mod is important
for later reproducibility.
Fixes golang/go#25526.
Change-Id: Ib0ff32f45dd30421907f3ba181db726c7ac9379c
Reviewed-on: https://go-review.googlesource.com/121301
Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/vendor/cmd/go/internal/modfetch/cache.go b/vendor/cmd/go/internal/modfetch/cache.go
index 14def8d..1f6551e 100644
--- a/vendor/cmd/go/internal/modfetch/cache.go
+++ b/vendor/cmd/go/internal/modfetch/cache.go
@@ -133,10 +133,20 @@
c := r.cache.Do("gomod:"+rev, func() interface{} {
file, text, err := readDiskGoMod(r.path, rev)
if err == nil {
+ // Note: readDiskGoMod already called checkGoMod.
return cached{text, nil}
}
+ // Convert rev to canonical version
+ // so that we use the right identifier in the go.sum check.
+ info, err := r.Stat(rev)
+ if err != nil {
+ return cached{nil, err}
+ }
+ rev = info.Version
+
text, err = r.r.GoMod(rev)
+ checkGoMod(r.path, rev, text)
if err == nil {
if err := writeDiskGoMod(file, text); err != nil {
fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
@@ -267,6 +277,10 @@
data = nil
}
+ if err == nil {
+ checkGoMod(path, rev, data)
+ }
+
return file, data, err
}
diff --git a/vendor/cmd/go/internal/modfetch/fetch.go b/vendor/cmd/go/internal/modfetch/fetch.go
index 7f82e4a..984acae 100644
--- a/vendor/cmd/go/internal/modfetch/fetch.go
+++ b/vendor/cmd/go/internal/modfetch/fetch.go
@@ -81,6 +81,7 @@
if err != nil {
return err
}
+ checkOneSum(mod, hash) // check before installing the zip file
r, err := os.Open(tmpfile)
if err != nil {
return err
@@ -163,11 +164,35 @@
data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash"))
if err != nil {
- base.Fatalf("vgo: verifying %s %s: %v", mod.Path, mod.Version, err)
+ base.Fatalf("vgo: verifying %s@%s: %v", mod.Path, mod.Version, err)
}
h := strings.TrimSpace(string(data))
if !strings.HasPrefix(h, "h1:") {
- base.Fatalf("vgo: verifying %s %s: unexpected ziphash: %q", mod.Path, mod.Version, h)
+ base.Fatalf("vgo: verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
+ }
+
+ checkOneSum(mod, h)
+}
+
+func checkGoMod(path, version string, data []byte) {
+ initGoSum()
+ if !useGoSum {
+ return
+ }
+
+ h, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
+ return ioutil.NopCloser(bytes.NewReader(data)), nil
+ })
+ if err != nil {
+ base.Fatalf("vgo: verifying %s %s go.mod: %v", path, version, err)
+ }
+ checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
+}
+
+func checkOneSum(mod module.Version, h string) {
+ initGoSum()
+ if !useGoSum {
+ return
}
for _, vh := range goSum[mod] {
@@ -175,11 +200,11 @@
return
}
if strings.HasPrefix(vh, "h1:") {
- base.Fatalf("vgo: verifying %s %s: module hash mismatch\n\tdownloaded: %v\n\tgo.sum: %v", mod.Path, mod.Version, h, vh)
+ base.Fatalf("vgo: verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v", mod.Path, mod.Version, h, vh)
}
}
if len(goSum[mod]) > 0 {
- fmt.Fprintf(os.Stderr, "warning: verifying %s %s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum[mod], ", "), h)
+ fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum[mod], ", "), h)
}
goSum[mod] = append(goSum[mod], h)
}
diff --git a/vendor/cmd/go/internal/module/module.go b/vendor/cmd/go/internal/module/module.go
index 93e8cb6..7fad513 100644
--- a/vendor/cmd/go/internal/module/module.go
+++ b/vendor/cmd/go/internal/module/module.go
@@ -210,6 +210,21 @@
if mi.Path != mj.Path {
return mi.Path < mj.Path
}
- return semver.Compare(mi.Version, mj.Version) < 0
+ // To help go.sum formatting, allow version/file.
+ // Compare semver prefix by semver rules,
+ // file by string order.
+ vi := mi.Version
+ vj := mj.Version
+ var fi, fj string
+ if k := strings.Index(vi, "/"); k >= 0 {
+ vi, fi = vi[:k], vi[k:]
+ }
+ if k := strings.Index(vj, "/"); k >= 0 {
+ vj, fj = vj[:k], vj[k:]
+ }
+ if vi != vj {
+ return semver.Compare(vi, vj) < 0
+ }
+ return fi < fj
})
}
diff --git a/vendor/cmd/go/internal/vgo/init.go b/vendor/cmd/go/internal/vgo/init.go
index a88633a..5352fd3 100644
--- a/vendor/cmd/go/internal/vgo/init.go
+++ b/vendor/cmd/go/internal/vgo/init.go
@@ -189,6 +189,7 @@
}
modfetch.SrcMod = SrcMod
+ modfetch.GoSumFile = filepath.Join(ModRoot, "go.sum")
codehost.WorkRoot = filepath.Join(SrcMod, "cache/vcs")
if CmdModInit {
diff --git a/vendor/cmd/go/vgo_test.go b/vendor/cmd/go/vgo_test.go
index db8feea..9b240fc 100644
--- a/vendor/cmd/go/vgo_test.go
+++ b/vendor/cmd/go/vgo_test.go
@@ -12,6 +12,7 @@
"path/filepath"
"runtime"
"sort"
+ "strings"
"testing"
"cmd/go/internal/modconv"
@@ -660,24 +661,60 @@
tg.grepStdout("v0.6.0", "expected github.com/pkg/errors at v0.6.0")
}
-func TestVerifyNotDownloaded(t *testing.T) {
+func TestVerify(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
tg := testgo(t)
defer tg.cleanup()
tg.makeTempdir()
- tg.setenv("GOPATH", tg.path("gp"))
+ gopath := tg.path("gp")
+ tg.setenv("GOPATH", gopath)
tg.must(os.MkdirAll(tg.path("x"), 0777))
tg.must(ioutil.WriteFile(tg.path("x/go.mod"), []byte(`
module x
require github.com/pkg/errors v0.8.0
`), 0666))
+ tg.must(ioutil.WriteFile(tg.path("x/x.go"), []byte(`package x; import _ "github.com/pkg/errors"`), 0666))
+
+ // With correct go.sum,verify succeeds but avoids download.
tg.must(ioutil.WriteFile(tg.path("x/go.sum"), []byte(`github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
`), 0666))
- tg.must(ioutil.WriteFile(tg.path("x/x.go"), []byte(`package x`), 0666))
tg.cd(tg.path("x"))
tg.run("-vgo", "mod", "-verify")
- tg.mustNotExist(filepath.Join(tg.path("gp"), "/src/mod/cache/github.com/pkg/errors/@v/v0.8.0.zip"))
- tg.mustNotExist(filepath.Join(tg.path("gp"), "/src/mod/github.com/pkg"))
+ tg.mustNotExist(filepath.Join(gopath, "src/mod/cache/download/github.com/pkg/errors/@v/v0.8.0.zip"))
+ tg.mustNotExist(filepath.Join(gopath, "src/mod/github.com/pkg"))
+
+ // With incorrect sum, sync (which must download) fails.
+ // Even if the incorrect sum is in the old legacy go.modverify file.
+ tg.must(ioutil.WriteFile(tg.path("x/go.sum"), []byte(`
+`), 0666))
+ tg.must(ioutil.WriteFile(tg.path("x/go.modverify"), []byte(`github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf1+qw=
+`), 0666))
+ tg.runFail("-vgo", "mod", "-sync") // downloads pkg/errors
+ tg.grepStderr("checksum mismatch", "must detect mismatch")
+ tg.mustNotExist(filepath.Join(gopath, "src/mod/cache/download/github.com/pkg/errors/@v/v0.8.0.zip"))
+ tg.mustNotExist(filepath.Join(gopath, "src/mod/github.com/pkg"))
+
+ // With corrected sum, sync works.
+ tg.must(ioutil.WriteFile(tg.path("x/go.modverify"), []byte(`github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+`), 0666))
+ tg.run("-vgo", "mod", "-sync")
+ tg.mustExist(filepath.Join(gopath, "src/mod/cache/download/github.com/pkg/errors/@v/v0.8.0.zip"))
+ tg.mustExist(filepath.Join(gopath, "src/mod/github.com/pkg"))
+ tg.mustNotExist(tg.path("x/go.modverify")) // moved into go.sum
+
+ // Sync should have added sum for go.mod.
+ data, err := ioutil.ReadFile(tg.path("x/go.sum"))
+ if !strings.Contains(string(data), "\ngithub.com/pkg/errors v0.8.0/go.mod ") {
+ t.Fatalf("cannot find go.mod hash in go.sum: %v\n%s", err, data)
+ }
+
+ // Even the most basic attempt to load the module graph should detect incorrect go.mod files.
+ tg.run("-vgo", "mod", "-graph") // loads module graph, is OK
+ tg.must(ioutil.WriteFile(tg.path("x/go.sum"), []byte(`github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl1=
+`), 0666))
+ tg.runFail("-vgo", "mod", "-graph") // loads module graph, fails (even though sum is in old go.modverify file)
+ tg.grepStderr("go.mod: checksum mismatch", "must detect mismatch")
}
func TestVendorWithoutDeps(t *testing.T) {
