blob: 89d5555eb4d940602301663409d6bff1b870947f [file] [log] [blame]
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zip_test
import (
"archive/zip"
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"golang.org/x/mod/module"
"golang.org/x/mod/sumdb/dirhash"
modzip "golang.org/x/mod/zip"
"golang.org/x/tools/txtar"
)
const emptyHash = "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
var gitOnce struct {
path string
err error
sync.Once
}
func init() {
if os.Getenv("GO_BUILDER_NAME") != "" || os.Getenv("GIT_TRACE_CURL") == "1" {
// Enable extra Git logging to diagnose networking issues.
// (These environment variables will be inherited by subprocesses.)
os.Setenv("GIT_TRACE_CURL", "1")
os.Setenv("GIT_TRACE_CURL_NO_DATA", "1")
os.Setenv("GIT_REDACT_COOKIES", "o,SSO,GSSO_Uberproxy")
}
}
// gitPath returns the path to a usable "git" command,
// or a non-nil error.
func gitPath() (string, error) {
gitOnce.Do(func() {
path, err := exec.LookPath("git")
if err != nil {
gitOnce.err = err
return
}
if runtime.GOOS == "plan9" {
gitOnce.err = errors.New("plan9 git does not support the full git command line")
}
gitOnce.path = path
})
return gitOnce.path, gitOnce.err
}
func mustHaveGit(t testing.TB) {
if _, err := gitPath(); err != nil {
t.Helper()
t.Skipf("skipping: %v", err)
}
}
type testParams struct {
path, version, wantErr, hash string
archive *txtar.Archive
}
// readTest loads a test from a txtar file. The comment section of the file
// should contain lines with key=value pairs. Valid keys are the field names
// from testParams.
func readTest(file string) (testParams, error) {
var test testParams
var err error
test.archive, err = txtar.ParseFile(file)
if err != nil {
return testParams{}, err
}
lines := strings.Split(string(test.archive.Comment), "\n")
for n, line := range lines {
n++ // report line numbers starting with 1
if i := strings.IndexByte(line, '#'); i >= 0 {
line = line[:i]
}
line = strings.TrimSpace(line)
if line == "" {
continue
}
eq := strings.IndexByte(line, '=')
if eq < 0 {
return testParams{}, fmt.Errorf("%s:%d: missing = separator", file, n)
}
key, value := strings.TrimSpace(line[:eq]), strings.TrimSpace(line[eq+1:])
switch key {
case "path":
test.path = value
case "version":
test.version = value
case "wantErr":
test.wantErr = value
case "hash":
test.hash = value
default:
return testParams{}, fmt.Errorf("%s:%d: unknown key %q", file, n, key)
}
}
return test, nil
}
func extractTxtarToTempDir(t testing.TB, arc *txtar.Archive) (dir string, err error) {
dir = t.TempDir()
for _, f := range arc.Files {
filePath := filepath.Join(dir, f.Name)
if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil {
return "", err
}
if err := ioutil.WriteFile(filePath, f.Data, 0666); err != nil {
return "", err
}
}
return dir, nil
}
func extractTxtarToTempZip(t *testing.T, arc *txtar.Archive) (zipPath string, err error) {
zipPath = filepath.Join(t.TempDir(), "txtar.zip")
zipFile, err := os.Create(zipPath)
if err != nil {
return "", err
}
defer func() {
if cerr := zipFile.Close(); err == nil && cerr != nil {
err = cerr
}
}()
zw := zip.NewWriter(zipFile)
for _, f := range arc.Files {
zf, err := zw.Create(f.Name)
if err != nil {
return "", err
}
if _, err := zf.Write(f.Data); err != nil {
return "", err
}
}
if err := zw.Close(); err != nil {
return "", err
}
return zipFile.Name(), nil
}
type fakeFile struct {
name string
size uint64
data []byte // if nil, Open will access a sequence of 0-bytes
}
func (f fakeFile) Path() string { return f.name }
func (f fakeFile) Lstat() (os.FileInfo, error) { return fakeFileInfo{f}, nil }
func (f fakeFile) Open() (io.ReadCloser, error) {
if f.data != nil {
return ioutil.NopCloser(bytes.NewReader(f.data)), nil
}
if f.size >= uint64(modzip.MaxZipFile<<1) {
return nil, fmt.Errorf("cannot open fakeFile of size %d", f.size)
}
return ioutil.NopCloser(io.LimitReader(zeroReader{}, int64(f.size))), nil
}
type fakeFileInfo struct {
f fakeFile
}
func (fi fakeFileInfo) Name() string { return path.Base(fi.f.name) }
func (fi fakeFileInfo) Size() int64 { return int64(fi.f.size) }
func (fi fakeFileInfo) Mode() os.FileMode { return 0644 }
func (fi fakeFileInfo) ModTime() time.Time { return time.Time{} }
func (fi fakeFileInfo) IsDir() bool { return false }
func (fi fakeFileInfo) Sys() interface{} { return nil }
type zeroReader struct{}
func (r zeroReader) Read(b []byte) (int, error) {
for i := range b {
b[i] = 0
}
return len(b), nil
}
func formatCheckedFiles(cf modzip.CheckedFiles) string {
buf := &bytes.Buffer{}
fmt.Fprintf(buf, "valid:\n")
for _, f := range cf.Valid {
fmt.Fprintln(buf, f)
}
fmt.Fprintf(buf, "\nomitted:\n")
for _, f := range cf.Omitted {
fmt.Fprintf(buf, "%s: %v\n", f.Path, f.Err)
}
fmt.Fprintf(buf, "\ninvalid:\n")
for _, f := range cf.Invalid {
fmt.Fprintf(buf, "%s: %v\n", f.Path, f.Err)
}
return buf.String()
}
// TestCheckFiles verifies behavior of CheckFiles. Note that CheckFiles is also
// covered by TestCreate, TestCreateDir, and TestCreateSizeLimits, so this test
// focuses on how multiple errors and omissions are reported, rather than trying
// to cover every case.
func TestCheckFiles(t *testing.T) {
testPaths, err := filepath.Glob(filepath.FromSlash("testdata/check_files/*.txt"))
if err != nil {
t.Fatal(err)
}
for _, testPath := range testPaths {
testPath := testPath
name := strings.TrimSuffix(filepath.Base(testPath), ".txt")
t.Run(name, func(t *testing.T) {
t.Parallel()
// Load the test.
test, err := readTest(testPath)
if err != nil {
t.Fatal(err)
}
files := make([]modzip.File, 0, len(test.archive.Files))
var want string
for _, tf := range test.archive.Files {
if tf.Name == "want" {
want = string(tf.Data)
continue
}
files = append(files, fakeFile{
name: tf.Name,
size: uint64(len(tf.Data)),
data: tf.Data,
})
}
// Check the files.
cf, _ := modzip.CheckFiles(files)
got := formatCheckedFiles(cf)
if got != want {
t.Errorf("got:\n%s\n\nwant:\n%s", got, want)
}
// Check that the error (if any) is just a list of invalid files.
// SizeError is not covered in this test.
var gotErr, wantErr string
if len(cf.Invalid) > 0 {
wantErr = modzip.FileErrorList(cf.Invalid).Error()
}
if err := cf.Err(); err != nil {
gotErr = err.Error()
}
if gotErr != wantErr {
t.Errorf("got error:\n%s\n\nwant error:\n%s", gotErr, wantErr)
}
})
}
}
// TestCheckDir verifies behavior of the CheckDir function. Note that CheckDir
// relies on CheckFiles and listFilesInDir (called by CreateFromDir), so this
// test focuses on how multiple errors and omissions are reported, rather than
// trying to cover every case.
func TestCheckDir(t *testing.T) {
testPaths, err := filepath.Glob(filepath.FromSlash("testdata/check_dir/*.txt"))
if err != nil {
t.Fatal(err)
}
for _, testPath := range testPaths {
testPath := testPath
name := strings.TrimSuffix(filepath.Base(testPath), ".txt")
t.Run(name, func(t *testing.T) {
t.Parallel()
// Load the test and extract the files to a temporary directory.
test, err := readTest(testPath)
if err != nil {
t.Fatal(err)
}
var want string
for i, f := range test.archive.Files {
if f.Name == "want" {
want = string(f.Data)
test.archive.Files = append(test.archive.Files[:i], test.archive.Files[i+1:]...)
break
}
}
tmpDir, err := extractTxtarToTempDir(t, test.archive)
if err != nil {
t.Fatal(err)
}
// Check the directory.
cf, err := modzip.CheckDir(tmpDir)
if err != nil && err.Error() != cf.Err().Error() {
// I/O error
t.Fatal(err)
}
rep := strings.NewReplacer(tmpDir, "$work", `'\''`, `'\''`, string(os.PathSeparator), "/")
got := rep.Replace(formatCheckedFiles(cf))
if got != want {
t.Errorf("got:\n%s\n\nwant:\n%s", got, want)
}
// Check that the error (if any) is just a list of invalid files.
// SizeError is not covered in this test.
var gotErr, wantErr string
if len(cf.Invalid) > 0 {
wantErr = modzip.FileErrorList(cf.Invalid).Error()
}
if err := cf.Err(); err != nil {
gotErr = err.Error()
}
if gotErr != wantErr {
t.Errorf("got error:\n%s\n\nwant error:\n%s", gotErr, wantErr)
}
})
}
}
// TestCheckZip verifies behavior of CheckZip. Note that CheckZip is also
// covered by TestUnzip, so this test focuses on how multiple errors are
// reported, rather than trying to cover every case.
func TestCheckZip(t *testing.T) {
testPaths, err := filepath.Glob(filepath.FromSlash("testdata/check_zip/*.txt"))
if err != nil {
t.Fatal(err)
}
for _, testPath := range testPaths {
testPath := testPath
name := strings.TrimSuffix(filepath.Base(testPath), ".txt")
t.Run(name, func(t *testing.T) {
t.Parallel()
// Load the test and extract the files to a temporary zip file.
test, err := readTest(testPath)
if err != nil {
t.Fatal(err)
}
var want string
for i, f := range test.archive.Files {
if f.Name == "want" {
want = string(f.Data)
test.archive.Files = append(test.archive.Files[:i], test.archive.Files[i+1:]...)
break
}
}
tmpZipPath, err := extractTxtarToTempZip(t, test.archive)
if err != nil {
t.Fatal(err)
}
// Check the zip.
m := module.Version{Path: test.path, Version: test.version}
cf, err := modzip.CheckZip(m, tmpZipPath)
if err != nil && err.Error() != cf.Err().Error() {
// I/O error
t.Fatal(err)
}
got := formatCheckedFiles(cf)
if got != want {
t.Errorf("got:\n%s\n\nwant:\n%s", got, want)
}
// Check that the error (if any) is just a list of invalid files.
// SizeError is not covered in this test.
var gotErr, wantErr string
if len(cf.Invalid) > 0 {
wantErr = modzip.FileErrorList(cf.Invalid).Error()
}
if err := cf.Err(); err != nil {
gotErr = err.Error()
}
if gotErr != wantErr {
t.Errorf("got error:\n%s\n\nwant error:\n%s", gotErr, wantErr)
}
})
}
}
func TestCreate(t *testing.T) {
testDir := filepath.FromSlash("testdata/create")
testInfos, err := ioutil.ReadDir(testDir)
if err != nil {
t.Fatal(err)
}
for _, testInfo := range testInfos {
testInfo := testInfo
base := filepath.Base(testInfo.Name())
if filepath.Ext(base) != ".txt" {
continue
}
t.Run(base[:len(base)-len(".txt")], func(t *testing.T) {
t.Parallel()
// Load the test.
testPath := filepath.Join(testDir, testInfo.Name())
test, err := readTest(testPath)
if err != nil {
t.Fatal(err)
}
// Write zip to temporary file.
tmpZip, err := ioutil.TempFile("", "TestCreate-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZip.Name()
defer func() {
tmpZip.Close()
os.Remove(tmpZipPath)
}()
m := module.Version{Path: test.path, Version: test.version}
files := make([]modzip.File, len(test.archive.Files))
for i, tf := range test.archive.Files {
files[i] = fakeFile{
name: tf.Name,
size: uint64(len(tf.Data)),
data: tf.Data,
}
}
if err := modzip.Create(tmpZip, m, files); err != nil {
if test.wantErr == "" {
t.Fatalf("unexpected error: %v", err)
} else if !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("got error %q; want error containing %q", err.Error(), test.wantErr)
} else {
return
}
} else if test.wantErr != "" {
t.Fatalf("unexpected success; wanted error containing %q", test.wantErr)
}
if err := tmpZip.Close(); err != nil {
t.Fatal(err)
}
// Hash zip file, compare with known value.
if hash, err := dirhash.HashZip(tmpZipPath, dirhash.Hash1); err != nil {
t.Fatal(err)
} else if hash != test.hash {
t.Fatalf("got hash: %q\nwant: %q", hash, test.hash)
}
})
}
}
func TestCreateFromDir(t *testing.T) {
testDir := filepath.FromSlash("testdata/create_from_dir")
testInfos, err := ioutil.ReadDir(testDir)
if err != nil {
t.Fatal(err)
}
for _, testInfo := range testInfos {
testInfo := testInfo
base := filepath.Base(testInfo.Name())
if filepath.Ext(base) != ".txt" {
continue
}
t.Run(base[:len(base)-len(".txt")], func(t *testing.T) {
t.Parallel()
// Load the test.
testPath := filepath.Join(testDir, testInfo.Name())
test, err := readTest(testPath)
if err != nil {
t.Fatal(err)
}
// Write files to a temporary directory.
tmpDir, err := extractTxtarToTempDir(t, test.archive)
if err != nil {
t.Fatal(err)
}
// Create zip from the directory.
tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZip.Name()
defer func() {
tmpZip.Close()
os.Remove(tmpZipPath)
}()
m := module.Version{Path: test.path, Version: test.version}
if err := modzip.CreateFromDir(tmpZip, m, tmpDir); err != nil {
if test.wantErr == "" {
t.Fatalf("unexpected error: %v", err)
} else if !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("got error %q; want error containing %q", err, test.wantErr)
} else {
return
}
} else if test.wantErr != "" {
t.Fatalf("unexpected success; want error containing %q", test.wantErr)
}
// Hash zip file, compare with known value.
if hash, err := dirhash.HashZip(tmpZipPath, dirhash.Hash1); err != nil {
t.Fatal(err)
} else if hash != test.hash {
t.Fatalf("got hash: %q\nwant: %q", hash, test.hash)
}
})
}
}
func TestCreateFromDirSpecial(t *testing.T) {
for _, test := range []struct {
desc string
setup func(t *testing.T, tmpDir string) string
wantHash string
}{
{
desc: "ignore_empty_dir",
setup: func(t *testing.T, tmpDir string) string {
if err := os.Mkdir(filepath.Join(tmpDir, "empty"), 0777); err != nil {
t.Fatal(err)
}
return tmpDir
},
wantHash: emptyHash,
}, {
desc: "ignore_symlink",
setup: func(t *testing.T, tmpDir string) string {
if err := os.Symlink(tmpDir, filepath.Join(tmpDir, "link")); err != nil {
switch runtime.GOOS {
case "plan9", "windows":
t.Skipf("could not create symlink: %v", err)
default:
t.Fatal(err)
}
}
return tmpDir
},
wantHash: emptyHash,
}, {
desc: "dir_is_vendor",
setup: func(t *testing.T, tmpDir string) string {
vendorDir := filepath.Join(tmpDir, "vendor")
if err := os.Mkdir(vendorDir, 0777); err != nil {
t.Fatal(err)
}
goModData := []byte("module example.com/m\n\ngo 1.13\n")
if err := ioutil.WriteFile(filepath.Join(vendorDir, "go.mod"), goModData, 0666); err != nil {
t.Fatal(err)
}
return vendorDir
},
wantHash: "h1:XduFAgX/GaspZa8Jv4pfzoGEzNaU/r88PiCunijw5ok=",
},
} {
t.Run(test.desc, func(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "TestCreateFromDirSpecial-"+test.desc)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
dir := test.setup(t, tmpDir)
tmpZipFile, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZipFile.Name()
defer func() {
tmpZipFile.Close()
os.Remove(tmpZipPath)
}()
m := module.Version{Path: "example.com/m", Version: "v1.0.0"}
if err := modzip.CreateFromDir(tmpZipFile, m, dir); err != nil {
t.Fatal(err)
}
if err := tmpZipFile.Close(); err != nil {
t.Fatal(err)
}
if hash, err := dirhash.HashZip(tmpZipPath, dirhash.Hash1); err != nil {
t.Fatal(err)
} else if hash != test.wantHash {
t.Fatalf("got hash %q; want %q", hash, emptyHash)
}
})
}
}
func TestUnzip(t *testing.T) {
testDir := filepath.FromSlash("testdata/unzip")
testInfos, err := ioutil.ReadDir(testDir)
if err != nil {
t.Fatal(err)
}
for _, testInfo := range testInfos {
base := filepath.Base(testInfo.Name())
if filepath.Ext(base) != ".txt" {
continue
}
t.Run(base[:len(base)-len(".txt")], func(t *testing.T) {
// Load the test.
testPath := filepath.Join(testDir, testInfo.Name())
test, err := readTest(testPath)
if err != nil {
t.Fatal(err)
}
// Convert txtar to temporary zip file.
tmpZipPath, err := extractTxtarToTempZip(t, test.archive)
if err != nil {
t.Fatal(err)
}
// Extract to a temporary directory.
tmpDir, err := ioutil.TempDir("", "TestUnzip")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
m := module.Version{Path: test.path, Version: test.version}
if err := modzip.Unzip(tmpDir, m, tmpZipPath); err != nil {
if test.wantErr == "" {
t.Fatalf("unexpected error: %v", err)
} else if !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("got error %q; want error containing %q", err.Error(), test.wantErr)
} else {
return
}
} else if test.wantErr != "" {
t.Fatalf("unexpected success; wanted error containing %q", test.wantErr)
}
// Hash the directory, compare to known value.
prefix := fmt.Sprintf("%s@%s/", test.path, test.version)
if hash, err := dirhash.HashDir(tmpDir, prefix, dirhash.Hash1); err != nil {
t.Fatal(err)
} else if hash != test.hash {
t.Fatalf("got hash %q\nwant: %q", hash, test.hash)
}
})
}
}
type sizeLimitTest struct {
desc string
files []modzip.File
wantErr string
wantCheckFilesErr string
wantCreateErr string
wantCheckZipErr string
wantUnzipErr string
}
// sizeLimitTests is shared by TestCreateSizeLimits and TestUnzipSizeLimits.
var sizeLimitTests = [...]sizeLimitTest{
{
desc: "one_large",
files: []modzip.File{fakeFile{
name: "large.go",
size: modzip.MaxZipFile,
}},
}, {
desc: "one_too_large",
files: []modzip.File{fakeFile{
name: "large.go",
size: modzip.MaxZipFile + 1,
}},
wantCheckFilesErr: "module source tree too large",
wantCreateErr: "module source tree too large",
wantCheckZipErr: "total uncompressed size of module contents too large",
wantUnzipErr: "total uncompressed size of module contents too large",
}, {
desc: "total_large",
files: []modzip.File{
fakeFile{
name: "small.go",
size: 10,
},
fakeFile{
name: "large.go",
size: modzip.MaxZipFile - 10,
},
},
}, {
desc: "total_too_large",
files: []modzip.File{
fakeFile{
name: "small.go",
size: 10,
},
fakeFile{
name: "large.go",
size: modzip.MaxZipFile - 9,
},
},
wantCheckFilesErr: "module source tree too large",
wantCreateErr: "module source tree too large",
wantCheckZipErr: "total uncompressed size of module contents too large",
wantUnzipErr: "total uncompressed size of module contents too large",
}, {
desc: "large_gomod",
files: []modzip.File{fakeFile{
name: "go.mod",
size: modzip.MaxGoMod,
}},
}, {
desc: "too_large_gomod",
files: []modzip.File{fakeFile{
name: "go.mod",
size: modzip.MaxGoMod + 1,
}},
wantErr: "go.mod file too large",
}, {
desc: "large_license",
files: []modzip.File{fakeFile{
name: "LICENSE",
size: modzip.MaxLICENSE,
}},
}, {
desc: "too_large_license",
files: []modzip.File{fakeFile{
name: "LICENSE",
size: modzip.MaxLICENSE + 1,
}},
wantErr: "LICENSE file too large",
},
}
var sizeLimitVersion = module.Version{Path: "example.com/large", Version: "v1.0.0"}
func TestCreateSizeLimits(t *testing.T) {
if testing.Short() {
t.Skip("creating large files takes time")
}
tests := append(sizeLimitTests[:], sizeLimitTest{
// negative file size may happen when size is represented as uint64
// but is cast to int64, as is the case in zip files.
desc: "negative",
files: []modzip.File{fakeFile{
name: "neg.go",
size: 0x8000000000000000,
}},
wantErr: "module source tree too large",
}, sizeLimitTest{
desc: "size_is_a_lie",
files: []modzip.File{fakeFile{
name: "lie.go",
size: 1,
data: []byte(`package large`),
}},
wantCreateErr: "larger than declared size",
})
for _, test := range tests {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
wantCheckFilesErr := test.wantCheckFilesErr
if wantCheckFilesErr == "" {
wantCheckFilesErr = test.wantErr
}
if _, err := modzip.CheckFiles(test.files); err == nil && wantCheckFilesErr != "" {
t.Fatalf("CheckFiles: unexpected success; want error containing %q", wantCheckFilesErr)
} else if err != nil && wantCheckFilesErr == "" {
t.Fatalf("CheckFiles: got error %q; want success", err)
} else if err != nil && !strings.Contains(err.Error(), wantCheckFilesErr) {
t.Fatalf("CheckFiles: got error %q; want error containing %q", err, wantCheckFilesErr)
}
wantCreateErr := test.wantCreateErr
if wantCreateErr == "" {
wantCreateErr = test.wantErr
}
if err := modzip.Create(ioutil.Discard, sizeLimitVersion, test.files); err == nil && wantCreateErr != "" {
t.Fatalf("Create: unexpected success; want error containing %q", wantCreateErr)
} else if err != nil && wantCreateErr == "" {
t.Fatalf("Create: got error %q; want success", err)
} else if err != nil && !strings.Contains(err.Error(), wantCreateErr) {
t.Fatalf("Create: got error %q; want error containing %q", err, wantCreateErr)
}
})
}
}
func TestUnzipSizeLimits(t *testing.T) {
if testing.Short() {
t.Skip("creating large files takes time")
}
for _, test := range sizeLimitTests {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
tmpZipFile, err := ioutil.TempFile("", "TestUnzipSizeLimits-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZipFile.Name()
defer func() {
tmpZipFile.Close()
if err := os.Remove(tmpZipPath); err != nil {
t.Errorf("removing temp zip file: %v", err)
}
}()
zw := zip.NewWriter(tmpZipFile)
prefix := fmt.Sprintf("%s@%s/", sizeLimitVersion.Path, sizeLimitVersion.Version)
for _, tf := range test.files {
zf, err := zw.Create(prefix + tf.Path())
if err != nil {
t.Fatal(err)
}
rc, err := tf.Open()
if err != nil {
t.Fatal(err)
}
_, err = io.Copy(zf, rc)
rc.Close()
if err != nil {
t.Fatal(err)
}
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if err := tmpZipFile.Close(); err != nil {
t.Fatal(err)
}
tmpDir, err := ioutil.TempDir("", "TestUnzipSizeLimits")
if err != nil {
t.Fatal(err)
}
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Errorf("removing temp dir: %v", err)
}
}()
wantCheckZipErr := test.wantCheckZipErr
if wantCheckZipErr == "" {
wantCheckZipErr = test.wantErr
}
cf, err := modzip.CheckZip(sizeLimitVersion, tmpZipPath)
if err == nil {
err = cf.Err()
}
if err == nil && wantCheckZipErr != "" {
t.Fatalf("CheckZip: unexpected success; want error containing %q", wantCheckZipErr)
} else if err != nil && wantCheckZipErr == "" {
t.Fatalf("CheckZip: got error %q; want success", err)
} else if err != nil && !strings.Contains(err.Error(), wantCheckZipErr) {
t.Fatalf("CheckZip: got error %q; want error containing %q", err, wantCheckZipErr)
}
wantUnzipErr := test.wantUnzipErr
if wantUnzipErr == "" {
wantUnzipErr = test.wantErr
}
if err := modzip.Unzip(tmpDir, sizeLimitVersion, tmpZipPath); err == nil && wantUnzipErr != "" {
t.Fatalf("Unzip: unexpected success; want error containing %q", wantUnzipErr)
} else if err != nil && wantUnzipErr == "" {
t.Fatalf("Unzip: got error %q; want success", err)
} else if err != nil && !strings.Contains(err.Error(), wantUnzipErr) {
t.Fatalf("Unzip: got error %q; want error containing %q", err, wantUnzipErr)
}
})
}
}
func TestUnzipSizeLimitsSpecial(t *testing.T) {
if testing.Short() {
t.Skip("skipping test; creating large files takes time")
}
for _, test := range []struct {
desc string
wantErr1, wantErr2 string
m module.Version
writeZip func(t *testing.T, zipFile *os.File)
}{
{
desc: "large_zip",
m: module.Version{Path: "example.com/m", Version: "v1.0.0"},
writeZip: func(t *testing.T, zipFile *os.File) {
if err := zipFile.Truncate(modzip.MaxZipFile); err != nil {
t.Fatal(err)
}
},
// this is not an error we care about; we're just testing whether
// Unzip checks the size of the file before opening.
// It's harder to create a valid zip file of exactly the right size.
wantErr1: "not a valid zip file",
}, {
desc: "too_large_zip",
m: module.Version{Path: "example.com/m", Version: "v1.0.0"},
writeZip: func(t *testing.T, zipFile *os.File) {
if err := zipFile.Truncate(modzip.MaxZipFile + 1); err != nil {
t.Fatal(err)
}
},
wantErr1: "module zip file is too large",
}, {
desc: "size_is_a_lie",
m: module.Version{Path: "example.com/m", Version: "v1.0.0"},
writeZip: func(t *testing.T, zipFile *os.File) {
// Create a normal zip file in memory containing one file full of zero
// bytes. Use a distinctive size so we can find it later.
zipBuf := &bytes.Buffer{}
zw := zip.NewWriter(zipBuf)
f, err := zw.Create("example.com/m@v1.0.0/go.mod")
if err != nil {
t.Fatal(err)
}
realSize := 0x0BAD
buf := make([]byte, realSize)
if _, err := f.Write(buf); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
// Replace the uncompressed size of the file. As a shortcut, we just
// search-and-replace the byte sequence. It should occur twice because
// the 32- and 64-byte sizes are stored separately. All multi-byte
// values are little-endian.
zipData := zipBuf.Bytes()
realSizeData := []byte{0xAD, 0x0B}
fakeSizeData := []byte{0xAC, 0x00}
s := zipData
n := 0
for {
if i := bytes.Index(s, realSizeData); i < 0 {
break
} else {
s = s[i:]
}
copy(s[:len(fakeSizeData)], fakeSizeData)
n++
}
if n != 2 {
t.Fatalf("replaced size %d times; expected 2", n)
}
// Write the modified zip to the actual file.
if _, err := zipFile.Write(zipData); err != nil {
t.Fatal(err)
}
},
// wantErr1 is for 1.18 and earlier,
// wantErr2 is for 1.19 and later.
wantErr1: "uncompressed size of file example.com/m@v1.0.0/go.mod is larger than declared size",
wantErr2: "not a valid zip file",
},
} {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
tmpZipFile, err := ioutil.TempFile("", "TestUnzipSizeLimitsSpecial-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZipFile.Name()
defer func() {
tmpZipFile.Close()
os.Remove(tmpZipPath)
}()
test.writeZip(t, tmpZipFile)
if err := tmpZipFile.Close(); err != nil {
t.Fatal(err)
}
tmpDir, err := ioutil.TempDir("", "TestUnzipSizeLimitsSpecial")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
want := func() string {
s := fmt.Sprintf("%q", test.wantErr1)
if test.wantErr2 != "" {
s = fmt.Sprintf("%q or %q", test.wantErr1, test.wantErr2)
}
return s
}
if err := modzip.Unzip(tmpDir, test.m, tmpZipPath); err == nil && test.wantErr1 != "" {
t.Fatalf("unexpected success; want error containing %s", want())
} else if err != nil && test.wantErr1 == "" {
t.Fatalf("got error %q; want success", err)
} else if err != nil && !strings.Contains(err.Error(), test.wantErr1) && (test.wantErr2 == "" || !strings.Contains(err.Error(), test.wantErr2)) {
t.Fatalf("got error %q; want error containing %s", err, want())
}
})
}
}
// TestVCS clones a repository, creates a zip for a known version,
// and verifies the zip file itself has the same SHA-256 hash as the one
// 'go mod download' produces.
//
// This test is intended to build confidence that this implementation produces
// the same output as the go command, given the same VCS zip input. This is
// not intended to be a complete conformance test. The code that produces zip
// archives from VCS repos is based on the go command, but it's for testing
// only, and we don't export it.
//
// Note that we test the hash of the zip file itself. This is stricter than
// testing the hash of the content, which is what we've promised users.
// It's okay if the zip hash changes without changing the content hash, but
// we should not let that happen accidentally.
func TestVCS(t *testing.T) {
if testing.Short() {
t.Skip("skipping VCS cloning in -short mode")
}
var downloadErrorCount int32
const downloadErrorLimit = 3
_, gitErr := gitPath()
_, hgErr := exec.LookPath("hg")
haveVCS := map[string]bool{
"git": gitErr == nil,
"hg": hgErr == nil,
}
for _, test := range []struct {
m module.Version
vcs, url, subdir, rev string
wantContentHash, wantZipHash string
}{
// Simple tests: all versions of rsc.io/quote + newer major versions
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.0.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.0.0",
wantContentHash: "h1:haUSojyo3j2M9g7CEUFG8Na09dtn7QKxvPGaPVQdGwM=",
wantZipHash: "5c08ba2c09a364f93704aaa780e7504346102c6ef4fe1333a11f09904a732078",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.1.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.1.0",
wantContentHash: "h1:n/ElL9GOlVEwL0mVjzaYj0UxTI/TX9aQ7lR5LHqP/Rw=",
wantZipHash: "730a5ae6e5c4e216e4f84bb93aa9785a85630ad73f96954ebb5f9daa123dcaa9",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.2.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.2.0",
wantContentHash: "h1:fFMCNi0A97hfNrtUZVQKETbuc3h7bmfFQHnjutpPYCg=",
wantZipHash: "fe1bd62652e9737a30d6b7fd396ea13e54ad13fb05f295669eb63d6d33290b06",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.2.1"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.2.1",
wantContentHash: "h1:l+HtgC05eds8qgXNApuv6g1oK1q3B144BM5li1akqXY=",
wantZipHash: "9f0e74de55a6bd20c1567a81e707814dc221f07df176af2a0270392c6faf32fd",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.3.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.3.0",
wantContentHash: "h1:aPUoHx/0Cd7BTZs4SAaknT4TaKryH766GcFTvJjVbHU=",
wantZipHash: "03872ee7d6747bc2ee0abadbd4eb09e60f6df17d0a6142264abe8a8a00af50e7",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.4.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.4.0",
wantContentHash: "h1:tYuJspOzwTRMUOX6qmSDRTEKFVV80GM0/l89OLZuVNg=",
wantZipHash: "f60be8193c607bf197da01da4bedb3d683fe84c30de61040eb5d7afaf7869f2e",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.5.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.5.0",
wantContentHash: "h1:mVjf/WMWxfIw299sOl/O3EXn5qEaaJPMDHMsv7DBDlw=",
wantZipHash: "a2d281834ce159703540da94425fa02c7aec73b88b560081ed0d3681bfe9cd1f",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.5.1"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.5.1",
wantContentHash: "h1:ptSemFtffEBvMed43o25vSUpcTVcqxfXU8Jv0sfFVJs=",
wantZipHash: "4ecd78a6d9f571e84ed2baac1688fd150400db2c5b017b496c971af30aaece02",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.5.2"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.5.2",
wantContentHash: "h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=",
wantZipHash: "643fcf8ef4e4cbb8f910622c42df3f9a81f3efe8b158a05825a81622c121ca0a",
},
{
m: module.Version{Path: "rsc.io/quote", Version: "v1.5.3-pre1"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v1.5.3-pre1",
wantContentHash: "h1:c3EJ21kn75/hyrOL/Dvj45+ifxGFSY8Wf4WBcoWTxF0=",
wantZipHash: "24106f0f15384949df51fae5d34191bf120c3b80c1c904721ca2872cf83126b2",
},
{
m: module.Version{Path: "rsc.io/quote/v2", Version: "v2.0.1"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v2.0.1",
wantContentHash: "h1:DF8hmGbDhgiIa2tpqLjHLIKkJx6WjCtLEqZBAU+hACI=",
wantZipHash: "009ed42474a59526fe56a14a9dd02bd7f977d1bd3844398bd209d0da0484aade",
},
{
m: module.Version{Path: "rsc.io/quote/v3", Version: "v3.0.0"},
vcs: "git",
url: "https://github.com/rsc/quote",
rev: "v3.0.0",
subdir: "v3",
wantContentHash: "h1:OEIXClZHFMyx5FdatYfxxpNEvxTqHlu5PNdla+vSYGg=",
wantZipHash: "cf3ff89056b785d7b3ef3a10e984efd83b47d9e65eabe8098b927b3370d5c3eb",
},
// Test cases from vcs-test.golang.org
{
m: module.Version{Path: "vcs-test.golang.org/git/v3pkg.git/v3", Version: "v3.0.0"},
vcs: "git",
url: "https://vcs-test.golang.org/git/v3pkg",
rev: "v3.0.0",
wantContentHash: "h1:mZhljS1BaiW8lODR6wqY5pDxbhXja04rWPFXPwRAtvA=",
wantZipHash: "9c65f0d235e531008dc04e977f6fa5d678febc68679bb63d4148dadb91d3fe57",
},
{
m: module.Version{Path: "vcs-test.golang.org/go/custom-hg-hello", Version: "v0.0.0-20171010233936-a8c8e7a40da9"},
vcs: "hg",
url: "https://vcs-test.golang.org/hg/custom-hg-hello",
rev: "a8c8e7a40da9",
wantContentHash: "h1:LU6jFCbwn5VVgTcj+y4LspOpJHLZvl5TGPE+LwwpMw4=",
wantZipHash: "a1b12047da979d618c639ee98f370767a13d0507bd77785dc2f8dad66b40e2e6",
},
// Latest versions of selected golang.org/x repos
{
m: module.Version{Path: "golang.org/x/arch", Version: "v0.0.0-20190927153633-4e8777c89be4"},
vcs: "git",
url: "https://go.googlesource.com/arch",
rev: "4e8777c89be4d9e61691fbe5d4e6c8838a7806f3",
wantContentHash: "h1:QlVATYS7JBoZMVaf+cNjb90WD/beKVHnIxFKT4QaHVI=",
wantZipHash: "d17551a0c4957180ec1507065d13dcdd0f5cd8bfd7dd735fb81f64f3e2b31b68",
},
{
m: module.Version{Path: "golang.org/x/blog", Version: "v0.0.0-20191017104857-0cd0cdff05c2"},
vcs: "git",
url: "https://go.googlesource.com/blog",
rev: "0cd0cdff05c251ad0c796cc94d7059e013311fc6",
wantContentHash: "h1:IKGICrORhR1aH2xG/WqrnpggSNolSj5urQxggCfmj28=",
wantZipHash: "0fed6b400de54da34b52b464ef2cdff45167236aaaf9a99ba8eba8855036faff",
},
{
m: module.Version{Path: "golang.org/x/crypto", Version: "v0.0.0-20191011191535-87dc89f01550"},
vcs: "git",
url: "https://go.googlesource.com/crypto",
rev: "87dc89f01550277dc22b74ffcf4cd89fa2f40f4c",
wantContentHash: "h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=",
wantZipHash: "88e47aa05eb25c6abdad7387ccccfc39e74541896d87b7b1269e9dd2fa00100d",
},
{
m: module.Version{Path: "golang.org/x/net", Version: "v0.0.0-20191014212845-da9a3fd4c582"},
vcs: "git",
url: "https://go.googlesource.com/net",
rev: "da9a3fd4c5820e74b24a6cb7fb438dc9b0dd377c",
wantContentHash: "h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA=",
wantZipHash: "34901a85e6c15475a40457c2393ce66fb0999accaf2d6aa5b64b4863751ddbde",
},
{
m: module.Version{Path: "golang.org/x/sync", Version: "v0.0.0-20190911185100-cd5d95a43a6e"},
vcs: "git",
url: "https://go.googlesource.com/sync",
rev: "cd5d95a43a6e21273425c7ae415d3df9ea832eeb",
wantContentHash: "h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=",
wantZipHash: "9c63fe51b0c533b258d3acc30d9319fe78679ce1a051109c9dea3105b93e2eef",
},
{
m: module.Version{Path: "golang.org/x/sys", Version: "v0.0.0-20191010194322-b09406accb47"},
vcs: "git",
url: "https://go.googlesource.com/sys",
rev: "b09406accb4736d857a32bf9444cd7edae2ffa79",
wantContentHash: "h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=",
wantZipHash: "f26f2993757670b4d1fee3156d331513259757f17133a36966c158642c3f61df",
},
{
m: module.Version{Path: "golang.org/x/talks", Version: "v0.0.0-20191010201600-067e0d331fee"},
vcs: "git",
url: "https://go.googlesource.com/talks",
rev: "067e0d331feee4f8d0fa17d47444db533bd904e7",
wantContentHash: "h1:8fnBMBUwliuiHuzfFw6kSSx79AzQpqkjZi3FSNIoqYs=",
wantZipHash: "fab2129f3005f970dbf2247378edb3220f6bd36726acdc7300ae3bb0f129e2f2",
},
{
m: module.Version{Path: "golang.org/x/tools", Version: "v0.0.0-20191017205301-920acffc3e65"},
vcs: "git",
url: "https://go.googlesource.com/tools",
rev: "920acffc3e65862cb002dae6b227b8d9695e3d29",
wantContentHash: "h1:GwXwgmbrvlcHLDsENMqrQTTIC2C0kIPszsq929NruKI=",
wantZipHash: "7f0ab7466448190f8ad1b8cfb05787c3fb08f4a8f9953cd4b40a51c76ddebb28",
},
{
m: module.Version{Path: "golang.org/x/tour", Version: "v0.0.0-20191002171047-6bb846ce41cd"},
vcs: "git",
url: "https://go.googlesource.com/tour",
rev: "6bb846ce41cdca087b14c8e3560a679691c424b6",
wantContentHash: "h1:EUlK3Rq8iTkQERnCnveD654NvRJ/ZCM9XCDne+S5cJ8=",
wantZipHash: "d6a7e03e02e5f7714bd12653d319a3b0f6e1099c01b1f9a17bc3613fb31c9170",
},
} {
test := test
testName := strings.ReplaceAll(test.m.String(), "/", "_")
t.Run(testName, func(t *testing.T) {
if have, ok := haveVCS[test.vcs]; !ok {
t.Fatalf("unknown vcs: %s", test.vcs)
} else if !have {
t.Skipf("no %s executable in path", test.vcs)
}
t.Parallel()
repo, dl, err := downloadVCSZip(t, test.vcs, test.url, test.rev, test.subdir)
if err != nil {
// This may fail if there's a problem with the network or upstream
// repository. The package being tested doesn't directly interact with
// VCS tools; the test just does this to simulate what the go command
// does. So an error should cause a skip instead of a failure. But we
// should fail after too many errors so we don't lose test coverage
// when something changes permanently.
n := atomic.AddInt32(&downloadErrorCount, 1)
if n < downloadErrorLimit {
t.Skipf("failed to download zip from repository: %v", err)
} else {
t.Fatalf("failed to download zip from repository (repeated failure): %v", err)
}
}
// Create a module zip from that archive.
// (adapted from cmd/go/internal/modfetch.codeRepo.Zip)
info, err := dl.Stat()
if err != nil {
t.Fatal(err)
}
zr, err := zip.NewReader(dl, info.Size())
if err != nil {
t.Fatal(err)
}
var files []modzip.File
topPrefix := ""
subdir := test.subdir
if subdir != "" && !strings.HasSuffix(subdir, "/") {
subdir += "/"
}
haveLICENSE := false
for _, f := range zr.File {
if !f.FileInfo().Mode().IsRegular() {
continue
}
if topPrefix == "" {
i := strings.Index(f.Name, "/")
if i < 0 {
t.Fatal("missing top-level directory prefix")
}
topPrefix = f.Name[:i+1]
}
if strings.HasSuffix(f.Name, "/") { // drop directory dummy entries
continue
}
if !strings.HasPrefix(f.Name, topPrefix) {
t.Fatal("zip file contains more than one top-level directory")
}
name := strings.TrimPrefix(f.Name, topPrefix)
if !strings.HasPrefix(name, subdir) {
continue
}
name = strings.TrimPrefix(name, subdir)
if name == ".hg_archival.txt" {
// Inserted by hg archive.
// Not correct to drop from other version control systems, but too bad.
continue
}
if name == "LICENSE" {
haveLICENSE = true
}
files = append(files, zipFile{name: name, f: f})
}
if !haveLICENSE && subdir != "" {
license, err := downloadVCSFile(t, test.vcs, repo, test.rev, "LICENSE")
if err != nil {
t.Fatal(err)
}
files = append(files, fakeFile{
name: "LICENSE",
size: uint64(len(license)),
data: license,
})
}
tmpModZipFile, err := ioutil.TempFile("", "TestVCS-*.zip")
if err != nil {
t.Fatal(err)
}
tmpModZipPath := tmpModZipFile.Name()
defer func() {
tmpModZipFile.Close()
os.Remove(tmpModZipPath)
}()
h := sha256.New()
w := io.MultiWriter(tmpModZipFile, h)
if err := modzip.Create(w, test.m, files); err != nil {
t.Fatal(err)
}
if err := tmpModZipFile.Close(); err != nil {
t.Fatal(err)
}
gotZipHash := hex.EncodeToString(h.Sum(nil))
if test.wantZipHash != gotZipHash {
// If the test fails because the hash of the zip file itself differs,
// that may be okay as long as the hash of the data within the zip file
// does not change. For example, we might change the compression,
// order, or alignment of files without affecting the extracted output.
// We shouldn't make such a change unintentionally though, so this
// test will fail either way.
if gotSum, err := dirhash.HashZip(tmpModZipPath, dirhash.Hash1); err == nil && test.wantContentHash != gotSum {
t.Fatalf("zip content hash: got %s, want %s", gotSum, test.wantContentHash)
} else {
t.Fatalf("zip file hash: got %s, want %s", gotZipHash, test.wantZipHash)
}
}
})
}
}
func downloadVCSZip(t testing.TB, vcs, url, rev, subdir string) (repoDir string, dl *os.File, err error) {
repoDir = t.TempDir()
switch vcs {
case "git":
// Create a repository and download the revision we want.
if _, err := run(t, repoDir, "git", "init", "--bare"); err != nil {
return "", nil, err
}
if err := os.MkdirAll(filepath.Join(repoDir, "info"), 0777); err != nil {
return "", nil, err
}
attrFile, err := os.OpenFile(filepath.Join(repoDir, "info", "attributes"), os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
if err != nil {
return "", nil, err
}
if _, err := attrFile.Write([]byte("\n* -export-subst -export-ignore\n")); err != nil {
attrFile.Close()
return "", nil, err
}
if err := attrFile.Close(); err != nil {
return "", nil, err
}
if _, err := run(t, repoDir, "git", "remote", "add", "origin", "--", url); err != nil {
return "", nil, err
}
var refSpec string
if strings.HasPrefix(rev, "v") {
refSpec = fmt.Sprintf("refs/tags/%[1]s:refs/tags/%[1]s", rev)
} else {
refSpec = fmt.Sprintf("%s:refs/dummy", rev)
}
if _, err := run(t, repoDir, "git", "fetch", "-f", "--depth=1", "origin", refSpec); err != nil {
return "", nil, err
}
// Create an archive.
zipPath := filepath.Join(t.TempDir(), "vcs.zip")
tmpZipFile, err := os.Create(zipPath)
if err != nil {
return "", nil, err
}
t.Cleanup(func() { tmpZipFile.Close() })
subdirArg := subdir
if subdir == "" {
subdirArg = "."
}
cmd := exec.Command("git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", rev, "--", subdirArg)
cmd.Dir = repoDir
cmd.Stdout = tmpZipFile
stderr := new(strings.Builder)
cmd.Stderr = stderr
err = cmd.Run()
if stderr.Len() > 0 && (err != nil || testing.Verbose()) {
t.Logf("%v: %v\n%s", err, cmd, stderr)
} else if err != nil {
t.Logf("%v: %v", err, cmd)
} else {
t.Logf("%v", cmd)
}
if err != nil {
return "", nil, err
}
if _, err := tmpZipFile.Seek(0, 0); err != nil {
return "", nil, err
}
return repoDir, tmpZipFile, nil
case "hg":
// Clone the whole repository.
if _, err := run(t, repoDir, "hg", "clone", "-U", "--", url, "."); err != nil {
return "", nil, err
}
// Create an archive.
tmpZipFile, err := ioutil.TempFile("", "downloadVCSZip-*.zip")
if err != nil {
return "", nil, err
}
tmpZipPath := tmpZipFile.Name()
tmpZipFile.Close()
t.Cleanup(func() { os.Remove(tmpZipPath) })
args := []string{"archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/"}
if subdir != "" {
args = append(args, "-I", subdir+"/**")
}
args = append(args, "--", tmpZipPath)
if _, err := run(t, repoDir, "hg", args...); err != nil {
return "", nil, err
}
if tmpZipFile, err = os.Open(tmpZipPath); err != nil {
return "", nil, err
}
t.Cleanup(func() { tmpZipFile.Close() })
return repoDir, tmpZipFile, err
default:
return "", nil, fmt.Errorf("vcs %q not supported", vcs)
}
}
func downloadVCSFile(t testing.TB, vcs, repo, rev, file string) ([]byte, error) {
t.Helper()
switch vcs {
case "git":
return run(t, repo, "git", "cat-file", "blob", rev+":"+file)
default:
return nil, fmt.Errorf("vcs %q not supported", vcs)
}
}
func run(t testing.TB, dir string, name string, args ...string) ([]byte, error) {
t.Helper()
cmd := exec.Command(name, args...)
cmd.Dir = dir
stderr := new(strings.Builder)
cmd.Stderr = stderr
out, err := cmd.Output()
if stderr.Len() > 0 && (err != nil || testing.Verbose()) {
t.Logf("%v: %v\n%s", err, cmd, stderr)
} else if err != nil {
t.Logf("%v: %v", err, cmd)
} else {
t.Logf("%v", cmd)
}
return out, err
}
type zipFile struct {
name string
f *zip.File
}
func (f zipFile) Path() string { return f.name }
func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil }
func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
func TestCreateFromVCS_basic(t *testing.T) {
mustHaveGit(t)
// Write files to a temporary directory.
tmpDir, err := extractTxtarToTempDir(t, txtar.Parse([]byte(`-- go.mod --
module example.com/foo/bar
go 1.12
-- a.go --
package a
var A = 5
-- b.go --
package a
var B = 5
-- c/c.go --
package c
var C = 5
-- d/d.go --
package c
var D = 5
-- .gitignore --
b.go
c/`)))
if err != nil {
t.Fatal(err)
}
gitInit(t, tmpDir)
gitCommit(t, tmpDir)
for _, tc := range []struct {
desc string
subdir string
wantFiles []string
}{
{
desc: "from root",
subdir: "",
wantFiles: []string{"go.mod", "a.go", "d/d.go", ".gitignore"},
},
{
desc: "from subdir",
subdir: "d/",
// Note: File paths are zipped as if the subdir were the root. ie d.go instead of d/d.go.
wantFiles: []string{"d.go"},
},
} {
t.Run(tc.desc, func(t *testing.T) {
// Create zip from the directory.
tmpZip := &bytes.Buffer{}
m := module.Version{Path: "example.com/foo/bar", Version: "v0.0.1"}
if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", tc.subdir); err != nil {
t.Fatal(err)
}
readerAt := bytes.NewReader(tmpZip.Bytes())
r, err := zip.NewReader(readerAt, int64(tmpZip.Len()))
if err != nil {
t.Fatal(err)
}
var gotFiles []string
gotMap := map[string]bool{}
for _, f := range r.File {
gotMap[f.Name] = true
gotFiles = append(gotFiles, f.Name)
}
wantMap := map[string]bool{}
for _, f := range tc.wantFiles {
p := filepath.Join("example.com", "foo", "bar@v0.0.1", f)
wantMap[p] = true
}
// The things that should be there.
for f := range gotMap {
if !wantMap[f] {
t.Errorf("CreatedFromVCS: zipped file contains %s, but expected it not to", f)
}
}
// The things that are missing.
for f := range wantMap {
if !gotMap[f] {
t.Errorf("CreatedFromVCS: zipped file doesn't contain %s, but expected it to. all files: %v", f, gotFiles)
}
}
})
}
}
// Test what the experience of creating a zip from a v2 module is like.
func TestCreateFromVCS_v2(t *testing.T) {
mustHaveGit(t)
// Write files to a temporary directory.
tmpDir, err := extractTxtarToTempDir(t, txtar.Parse([]byte(`-- go.mod --
module example.com/foo/bar
go 1.12
-- a.go --
package a
var A = 5
-- b.go --
package a
var B = 5
-- go.mod --
module example.com/foo/bar
go 1.12
-- gaz/v2/a_2.go --
package a
var C = 9
-- gaz/v2/b_2.go --
package a
var B = 11
-- gaz/v2/go.mod --
module example.com/foo/bar/v2
go 1.12
-- .gitignore --
`)))
if err != nil {
t.Fatal(err)
}
gitInit(t, tmpDir)
gitCommit(t, tmpDir)
// Create zip from the directory.
tmpZip := &bytes.Buffer{}
m := module.Version{Path: "example.com/foo/bar/v2", Version: "v2.0.0"}
if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", "gaz/v2"); err != nil {
t.Fatal(err)
}
readerAt := bytes.NewReader(tmpZip.Bytes())
r, err := zip.NewReader(readerAt, int64(tmpZip.Len()))
if err != nil {
t.Fatal(err)
}
var gotFiles []string
gotMap := map[string]bool{}
for _, f := range r.File {
gotMap[f.Name] = true
gotFiles = append(gotFiles, f.Name)
}
wantMap := map[string]bool{
"example.com/foo/bar/v2@v2.0.0/a_2.go": true,
"example.com/foo/bar/v2@v2.0.0/b_2.go": true,
"example.com/foo/bar/v2@v2.0.0/go.mod": true,
}
// The things that should be there.
for f := range gotMap {
if !wantMap[f] {
t.Errorf("CreatedFromVCS: zipped file contains %s, but expected it not to", f)
}
}
// The things that are missing.
for f := range wantMap {
if !gotMap[f] {
t.Errorf("CreatedFromVCS: zipped file doesn't contain %s, but expected it to. all files: %v", f, gotFiles)
}
}
}
func TestCreateFromVCS_nonGitDir(t *testing.T) {
mustHaveGit(t)
// Write files to a temporary directory.
tmpDir, err := extractTxtarToTempDir(t, txtar.Parse([]byte(`-- go.mod --
module example.com/foo/bar
go 1.12
-- a.go --
package a
var A = 5
`)))
if err != nil {
t.Fatal(err)
}
// Create zip from the directory.
tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZip.Name()
defer func() {
tmpZip.Close()
os.Remove(tmpZipPath)
}()
m := module.Version{Path: "example.com/foo/bar", Version: "v0.0.1"}
err = modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", "")
if err == nil {
t.Fatal("CreateFromVCS: expected error, got nil")
}
var gotErr *modzip.UnrecognizedVCSError
if !errors.As(err, &gotErr) {
t.Errorf("CreateFromVCS: returned error does not unwrap to modzip.ErrUnrecognisedVCS, but expected it to. returned error: %v", err)
} else if gotErr.RepoRoot != tmpDir {
t.Errorf("CreateFromVCS: returned error has RepoRoot %q, but want %q. returned error: %v", gotErr.RepoRoot, tmpDir, err)
}
}
func TestCreateFromVCS_zeroCommitsGitDir(t *testing.T) {
mustHaveGit(t)
// Write files to a temporary directory.
tmpDir, err := extractTxtarToTempDir(t, txtar.Parse([]byte(`-- go.mod --
module example.com/foo/bar
go 1.12
-- a.go --
package a
var A = 5
`)))
if err != nil {
t.Fatal(err)
}
gitInit(t, tmpDir)
// Create zip from the directory.
tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
if err != nil {
t.Fatal(err)
}
tmpZipPath := tmpZip.Name()
defer func() {
tmpZip.Close()
os.Remove(tmpZipPath)
}()
m := module.Version{Path: "example.com/foo/bar", Version: "v0.0.1"}
if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", ""); err == nil {
t.Error("CreateFromVCS: expected error, got nil")
}
}
// gitInit runs "git init" at the specified dir.
//
// Note: some environments - and trybots - don't have git installed. This
// function will cause the calling test to be skipped if that's the case.
func gitInit(t testing.TB, dir string) {
t.Helper()
mustHaveGit(t)
if _, err := run(t, dir, "git", "init"); err != nil {
t.Fatal(err)
}
if _, err := run(t, dir, "git", "config", "user.name", "Go Gopher"); err != nil {
t.Fatal(err)
}
if _, err := run(t, dir, "git", "config", "user.email", "gopher@golang.org"); err != nil {
t.Fatal(err)
}
}
func gitCommit(t testing.TB, dir string) {
t.Helper()
mustHaveGit(t)
if _, err := run(t, dir, "git", "add", "-A"); err != nil {
t.Fatal(err)
}
if _, err := run(t, dir, "git", "commit", "-m", "some commit"); err != nil {
t.Fatal(err)
}
}