sweet: switch to a zip archive format for assets
Currently sweet stores assets in GCS as a gzipped tar archive, and while
this is amenable to streaming, it is not amenable to saving disk space
where possible. So, switch to a zip archive format which allows
decompression of individual files found within, and store the assets as
compressed in the cache in "sweet get", with an option to extract them
out for development.
On that note, "sweet run" needs to support both decompressing an
individual benchmark's assets and working from an uncompressed
development directory, so this change implements that too.
All of this is in service of using less disk space to hopefully allow
testing on regular builders.
Change-Id: I8c028620271b44d5dc4ee95d39693119578717ab
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/382655
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
diff --git a/sweet/assets.hash b/sweet/assets.hash
index 1fa2683..22df8f8 100644
--- a/sweet/assets.hash
+++ b/sweet/assets.hash
@@ -1 +1 @@
-{"v0.2.1":"9d1fb576fbf174cfa3e23b6bdba3ca1474e54bf43d5a2645e597bfd4b7a922ce"}
+{"v0.2.1":"79eb43748f32174138f008b64d9d400520de938e12db653df625ed01340279e1"}
diff --git a/sweet/cli/bootstrap/cache.go b/sweet/cli/bootstrap/cache.go
index 15221e9..774ae7d 100644
--- a/sweet/cli/bootstrap/cache.go
+++ b/sweet/cli/bootstrap/cache.go
@@ -14,7 +14,7 @@
var ErrNotInCache = errors.New("not found in cache")
func CachedAssets(cache, version string) (string, error) {
- name := VersionDirName(version)
+ name := VersionArchiveName(version)
if err := os.MkdirAll(cache, os.ModePerm); err != nil {
return "", fmt.Errorf("failed to create cache directory: %v", err)
}
diff --git a/sweet/cli/bootstrap/version.go b/sweet/cli/bootstrap/version.go
index 219ea45..6f850e5 100644
--- a/sweet/cli/bootstrap/version.go
+++ b/sweet/cli/bootstrap/version.go
@@ -19,9 +19,5 @@
}
func VersionArchiveName(version string) string {
- return fmt.Sprintf("%s.tar.gz", VersionDirName(version))
-}
-
-func VersionDirName(version string) string {
- return fmt.Sprintf("assets-%s", version)
+ return fmt.Sprintf("assets-%s.zip", version)
}
diff --git a/sweet/cmd/sweet/benchmark.go b/sweet/cmd/sweet/benchmark.go
index 50bc6c5..dd88971 100644
--- a/sweet/cmd/sweet/benchmark.go
+++ b/sweet/cmd/sweet/benchmark.go
@@ -5,7 +5,9 @@
package main
import (
+ "errors"
"fmt"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -132,7 +134,7 @@
func copyDirContents(dst, src string) error {
log.CommandPrintf("cp -r %s/* %s", src, dst)
- return fileutil.CopyDir(dst, src)
+ return fileutil.CopyDir(dst, src, nil)
}
func rmDirContents(dir string) error {
@@ -160,13 +162,26 @@
log.Printf("Setting up benchmark: %s", b.name)
// Compute top-level directories for this benchmark to work in.
- topAssetsDir := filepath.Join(r.assetsDir, b.name)
benchDir := filepath.Join(r.benchDir, b.name)
topDir := filepath.Join(r.workDir, b.name)
srcDir := filepath.Join(topDir, "src")
- hasAssets, err := fileutil.FileExists(topAssetsDir)
- if err != nil {
+ // Check if assets for this benchmark exist. Not all benchmarks have assets!
+ var hasAssets bool
+ assetsFSDir := b.name
+ if f, err := r.assetsFS.Open(assetsFSDir); err == nil {
+ fi, err := f.Stat()
+ if err != nil {
+ f.Close()
+ return err
+ }
+ if !fi.IsDir() {
+ f.Close()
+ return fmt.Errorf("found assets file for %s instead of directory", b.name)
+ }
+ f.Close()
+ hasAssets = true
+ } else if !errors.Is(err, fs.ErrNotExist) {
return err
}
@@ -271,7 +286,8 @@
for i, setup := range setups {
if hasAssets {
// Set up assets directory for test run.
- if err := copyDirContents(setup.AssetsDir, topAssetsDir); err != nil {
+ r.logCopyDirCommand(b.name, setup.AssetsDir)
+ if err := fileutil.CopyDir(setup.AssetsDir, assetsFSDir, r.assetsFS); err != nil {
return err
}
}
diff --git a/sweet/cmd/sweet/get.go b/sweet/cmd/sweet/get.go
index bc605cd..2d80290 100644
--- a/sweet/cmd/sweet/get.go
+++ b/sweet/cmd/sweet/get.go
@@ -5,8 +5,7 @@
package main
import (
- "archive/tar"
- "compress/gzip"
+ "archive/zip"
"flag"
"fmt"
"io"
@@ -16,7 +15,6 @@
"golang.org/x/benchmarks/sweet/cli/bootstrap"
"golang.org/x/benchmarks/sweet/common"
- "golang.org/x/benchmarks/sweet/common/fileutil"
"golang.org/x/benchmarks/sweet/common/log"
)
@@ -42,10 +40,9 @@
type getCmd struct {
auth bootstrap.AuthOption
force bool
- copyAssets bool
cache string
bucket string
- assetsDir string
+ copyDir string
assetsHashFile string
version string
}
@@ -59,11 +56,10 @@
func (c *getCmd) SetFlags(f *flag.FlagSet) {
f.Var(&c.auth, "auth", fmt.Sprintf("authentication method (options: %s)", authOpts(true)))
f.BoolVar(&c.force, "force", false, "force download even if assets for this version exist in the cache")
- f.BoolVar(&c.copyAssets, "copy", false, "copy assets to assets-dir instead of symlinking")
- f.StringVar(&c.cache, "cache", bootstrap.CacheDefault(), "cache location for tar'd and compressed assets, if set to \"\" will ignore cache")
+ f.StringVar(&c.cache, "cache", bootstrap.CacheDefault(), "cache location for assets")
f.StringVar(&c.version, "version", common.Version, "the version to download assets for")
f.StringVar(&c.bucket, "bucket", "go-sweet-assets", "GCS bucket to download assets from")
- f.StringVar(&c.assetsDir, "assets-dir", "./assets", "location to extract assets into")
+ f.StringVar(&c.copyDir, "copy", "", "location to extract assets into, useful for development")
f.StringVar(&c.assetsHashFile, "assets-hash-file", "./assets.hash", "file to check SHA256 hash of the downloaded artifact against")
}
@@ -72,49 +68,78 @@
if err := bootstrap.ValidateVersion(c.version); err != nil {
return err
}
- installAssets := func(todir string, readonly bool) error {
- return downloadAndExtract(todir, c.bucket, c.assetsHashFile, c.version, c.auth, readonly)
+ if c.copyDir == "" && c.cache == "" {
+ log.Printf("No cache to populate and assets are not copied. Nothing to do.")
+ return nil
}
+
+ // Create a file that we'll download assets into.
+ var (
+ f *os.File
+ fName string
+ err error
+ )
if c.cache == "" {
- log.Printf("Skipping cache...")
- return installAssets(c.assetsDir, false)
- }
- log.Printf("Checking cache: %s", c.cache)
- t, err := bootstrap.CachedAssets(c.cache, c.version)
- if err == bootstrap.ErrNotInCache || (err == nil && c.force) {
- if err := installAssets(t, true); err != nil {
+ // There's no cache, which means we'll be extracting directly.
+ // Just create a temporary file. zip archives cannot be streamed
+ // out unfortunately (the API requires a ReaderAt).
+ f, err = os.CreateTemp("", "go-sweet-assets")
+ if err != nil {
return err
}
- } else if err != nil {
- return err
- }
- if !c.copyAssets {
- log.Printf("Creating symlink to %s", c.assetsDir)
- if info, err := os.Lstat(c.assetsDir); err == nil {
- if info.Mode()&os.ModeSymlink != 0 {
- // We have a symlink, so just delete it so we can replace it.
- if err := os.Remove(c.assetsDir); err != nil {
- return fmt.Errorf("installing assets: removing %s: %v", c.assetsDir, err)
- }
- } else {
- return fmt.Errorf("installing assets: %s is not a symlink; to install assets here, remove it and re-run this command", c.assetsDir)
+ defer f.Close()
+ fName = f.Name()
+ } else {
+ // There is a cache, so create a file in the cache if there isn't
+ // one already.
+ log.Printf("Checking cache: %s", c.cache)
+ fName, err = bootstrap.CachedAssets(c.cache, c.version)
+ if err == bootstrap.ErrNotInCache || (err == nil && c.force) {
+ f, err = os.Create(fName)
+ if err != nil {
+ return err
}
- } else if !os.IsNotExist(err) {
- return fmt.Errorf("stat %s: %v", c.assetsDir, err)
+ defer f.Close()
+ } else if err != nil {
+ return err
}
- return os.Symlink(t, c.assetsDir)
}
- if _, err := os.Stat(c.assetsDir); err == nil {
- return fmt.Errorf("installing assets: %s exists; to copy assets here, remove it and re-run this command", c.assetsDir)
+
+ // If f is not nil, then we need to download assets.
+ // Otherwise they're in a cache.
+ if f != nil {
+ // Download the compressed assets into f.
+ if err := downloadAssets(f, c.bucket, c.assetsHashFile, c.version, c.auth); err != nil {
+ return err
+ }
+ }
+ // If we're not copying, we're done.
+ if c.copyDir == "" {
+ return nil
+ }
+ if f == nil {
+ // Since f is nil, and we'll be extracting, we need to open the file.
+ f, err := os.Open(fName)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ }
+
+ // Check to make sure out destination is clear.
+ if _, err := os.Stat(c.copyDir); err == nil {
+ return fmt.Errorf("installing assets: %s exists; to copy assets here, remove it and re-run this command", c.copyDir)
} else if !os.IsNotExist(err) {
- return fmt.Errorf("stat %s: %v", c.assetsDir, err)
+ return fmt.Errorf("stat %s: %v", c.copyDir, err)
}
- log.Printf("Copying assets %s", c.assetsDir)
- return fileutil.CopyDir(c.assetsDir, t)
+
+ // Extract assets into assetsDir.
+ log.Printf("Copying assets to %s", c.copyDir)
+ return extractAssets(f, c.copyDir)
}
-func downloadAndExtract(todir, bucket, hashfile, version string, auth bootstrap.AuthOption, readonly bool) error {
- log.Printf("Downloading assets archive for version %s to %s", version, todir)
+func downloadAssets(toFile *os.File, bucket, hashfile, version string, auth bootstrap.AuthOption) error {
+ log.Printf("Downloading assets archive for version %s to %s", version, toFile.Name())
// Create storage reader for streaming.
rc, err := bootstrap.NewStorageReader(bucket, version, auth)
@@ -127,8 +152,8 @@
hash := bootstrap.Hash()
r := io.TeeReader(rc, hash)
- // Stream and extract the results.
- if err := extractAssets(r, todir, readonly); err != nil {
+ // Stream the results.
+ if _, err := io.Copy(toFile, r); err != nil {
return err
}
@@ -151,56 +176,45 @@
return nil
}
-func extractAssets(r io.Reader, outdir string, readonly bool) error {
+func extractAssets(archive *os.File, outdir string) error {
if err := os.MkdirAll(outdir, os.ModePerm); err != nil {
return fmt.Errorf("create assets directory: %v", err)
}
- gr, err := gzip.NewReader(r)
+ archiveInfo, err := archive.Stat()
if err != nil {
return err
}
- defer gr.Close()
-
- tr := tar.NewReader(gr)
- for {
- hdr, err := tr.Next()
- if err == io.EOF {
- break
- } else if err != nil {
- return err
- }
- fullpath := filepath.Join(outdir, hdr.Name)
- if err := os.MkdirAll(filepath.Dir(fullpath), os.ModePerm); err != nil {
- return err
- }
- f, err := os.Create(fullpath)
- if err != nil {
- return err
- }
- if _, err := io.Copy(f, tr); err != nil {
- f.Close()
- return err
- }
- fperm := os.FileMode(uint32(hdr.Mode))
- if readonly {
- fperm = 0444 | (fperm & 0555)
- }
- if err := f.Chmod(fperm); err != nil {
- f.Close()
- return err
- }
- f.Close()
+ zr, err := zip.NewReader(archive, archiveInfo.Size())
+ if err != nil {
+ return err
}
- if readonly {
- return filepath.Walk(outdir, func(path string, info os.FileInfo, err error) error {
+ for _, zf := range zr.File {
+ err := func(zf *zip.File) error {
+ fullpath := filepath.Join(outdir, zf.Name)
+ if err := os.MkdirAll(filepath.Dir(fullpath), os.ModePerm); err != nil {
+ return err
+ }
+ inFile, err := zf.Open()
if err != nil {
return err
}
- if info.IsDir() {
- return os.Chmod(path, 0555)
+ defer inFile.Close()
+ outFile, err := os.Create(fullpath)
+ if err != nil {
+ return err
+ }
+ defer outFile.Close()
+ if _, err := io.Copy(outFile, inFile); err != nil {
+ return err
+ }
+ if err := outFile.Chmod(zf.Mode()); err != nil {
+ return err
}
return nil
- })
+ }(zf)
+ if err != nil {
+ return err
+ }
}
return nil
}
diff --git a/sweet/cmd/sweet/put.go b/sweet/cmd/sweet/put.go
index bce707f..893c40f 100644
--- a/sweet/cmd/sweet/put.go
+++ b/sweet/cmd/sweet/put.go
@@ -5,8 +5,7 @@
package main
import (
- "archive/tar"
- "compress/gzip"
+ "archive/zip"
"flag"
"fmt"
"io"
@@ -49,7 +48,7 @@
f.BoolVar(&c.force, "force", false, "force upload even if assets for this version exist")
f.StringVar(&c.version, "version", common.Version, "the version to upload assets for")
f.StringVar(&c.bucket, "bucket", "go-sweet-assets", "GCS bucket to upload assets to")
- f.StringVar(&c.assetsDir, "assets-dir", "./assets", "assets directory to tar, compress, and upload")
+ f.StringVar(&c.assetsDir, "assets-dir", "./assets", "assets directory to zip and upload")
f.StringVar(&c.assetsHashFile, "assets-hash-file", "./assets.hash", "file containing assets SHA256 hashes")
}
@@ -83,50 +82,48 @@
return updateAssetsHash(bootstrap.CanonicalizeHash(hash), c.assetsHashFile, c.version, c.force)
}
-func createAssetsArchive(w io.Writer, assetsDir, version string) error {
- gw := gzip.NewWriter(w)
- defer gw.Close()
-
- tw := tar.NewWriter(gw)
- defer tw.Close()
-
+func createAssetsArchive(w io.Writer, assetsDir, version string) (err error) {
+ zw := zip.NewWriter(w)
+ defer func() {
+ if cerr := zw.Close(); cerr != nil && err == nil {
+ err = fmt.Errorf("closing zip archive: %w", cerr)
+ }
+ }()
return filepath.Walk(assetsDir, func(fpath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
+ outPath, err := filepath.Rel(assetsDir, fpath)
+ if err != nil {
+ // By the guarantees of filepath.Walk, this shouldn't happen.
+ panic(err)
+ }
if info.IsDir() {
- return nil
+ // Add a trailing slash to indicate we're creating a directory.
+ _, err := zw.Create(outPath + "/")
+ return err
}
- isSymlink := info.Mode()&os.ModeSymlink != 0
- link := ""
- if isSymlink {
- l, err := os.Readlink(fpath)
- if err != nil {
- return err
- }
- link = l
+ if info.Mode()&os.ModeSymlink != 0 {
+ return fmt.Errorf("encountered symlink %s: symbolic links are not supported in assets", fpath)
}
+ // Create a file in our zip archive for writing.
+ fh := new(zip.FileHeader)
+ fh.Name = outPath
+ fh.Method = zip.Deflate
+ fh.SetMode(info.Mode())
+ zf, err := zw.CreateHeader(fh)
+ if err != nil {
+ return err
+ }
+ // Open the original file for reading.
f, err := os.Open(fpath)
if err != nil {
return err
}
defer f.Close()
- header, err := tar.FileInfoHeader(info, link)
- if err != nil {
- return err
- }
- header.Name, err = filepath.Rel(assetsDir, fpath)
- if err != nil {
- panic(err)
- }
- if err := tw.WriteHeader(header); err != nil {
- return err
- }
- if isSymlink {
- // We don't need to copy any data for the symlink.
- return nil
- }
- _, err = io.Copy(tw, f)
+
+ // Copy data into the archive.
+ _, err = io.Copy(zf, f)
return err
})
}
diff --git a/sweet/cmd/sweet/run.go b/sweet/cmd/sweet/run.go
index 4239149..3a2f142 100644
--- a/sweet/cmd/sweet/run.go
+++ b/sweet/cmd/sweet/run.go
@@ -5,9 +5,11 @@
package main
import (
+ "archive/zip"
"flag"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -15,6 +17,7 @@
"strings"
"unicode/utf8"
+ "golang.org/x/benchmarks/sweet/cli/bootstrap"
"golang.org/x/benchmarks/sweet/common"
"golang.org/x/benchmarks/sweet/common/log"
@@ -40,17 +43,29 @@
)
type runCfg struct {
- count int
- resultsDir string
- benchDir string
- assetsDir string
- workDir string
- dumpCore bool
- cpuProfile bool
- memProfile bool
- perf bool
- perfFlags string
- short bool
+ count int
+ resultsDir string
+ benchDir string
+ assetsDir string
+ workDir string
+ assetsCache string
+ dumpCore bool
+ cpuProfile bool
+ memProfile bool
+ perf bool
+ perfFlags string
+ short bool
+
+ assetsFS fs.FS
+}
+
+func (r *runCfg) logCopyDirCommand(fromRelDir, toDir string) {
+ if r.assetsDir == "" {
+ assetsFile, _ := bootstrap.CachedAssets(r.assetsCache, common.Version)
+ log.CommandPrintf("unzip %s '%s/*' -d %s", assetsFile, fromRelDir, toDir)
+ } else {
+ log.CommandPrintf("cp -r %s/* %s", filepath.Join(r.assetsDir, fromRelDir), toDir)
+ }
}
type runCmd struct {
@@ -113,8 +128,9 @@
func (c *runCmd) SetFlags(f *flag.FlagSet) {
f.StringVar(&c.runCfg.resultsDir, "results", "./results", "location to write benchmark results to")
f.StringVar(&c.runCfg.benchDir, "bench-dir", "./benchmarks", "the benchmarks directory in the sweet source")
- f.StringVar(&c.runCfg.assetsDir, "assets-dir", "./assets", "the directory containing assets for sweet benchmarks")
+ f.StringVar(&c.runCfg.assetsDir, "assets-dir", "", "the directory containing uncompressed assets for sweet benchmarks (overrides -cache)")
f.StringVar(&c.runCfg.workDir, "work-dir", "", "work directory for benchmarks (default: temporary directory)")
+ f.StringVar(&c.runCfg.assetsCache, "cache", bootstrap.CacheDefault(), "cache location for assets")
f.BoolVar(&c.runCfg.dumpCore, "dump-core", false, "whether to dump core files for each benchmark process when it completes a benchmark")
f.BoolVar(&c.runCfg.cpuProfile, "cpuprofile", false, "whether to dump a CPU profile for each benchmark (ensures all benchmarks do the same amount of work)")
f.BoolVar(&c.runCfg.memProfile, "memprofile", false, "whether to dump a memory profile for each benchmark (ensures all executions do the same amount of work")
@@ -152,10 +168,6 @@
if err != nil {
return fmt.Errorf("creating absolute path from provided work root: %v", err)
}
- c.assetsDir, err = filepath.Abs(c.assetsDir)
- if err != nil {
- return fmt.Errorf("creating absolute path from assets path: %v", err)
- }
c.benchDir, err = filepath.Abs(c.benchDir)
if err != nil {
return fmt.Errorf("creating absolute path from benchmarks path: %v", err)
@@ -164,14 +176,53 @@
if err != nil {
return fmt.Errorf("creating absolute path from results path: %v", err)
}
-
- // Make sure the assets directory is there.
- if info, err := os.Stat(c.assetsDir); os.IsNotExist(err) {
- return fmt.Errorf("assets not found at %q: forgot to run `sweet get`?", c.assetsDir)
- } else if err != nil {
- return fmt.Errorf("stat assets %q: %v", c.assetsDir, err)
- } else if info.Mode()&os.ModeDir == 0 {
- return fmt.Errorf("%q is not a directory", c.assetsDir)
+ if c.assetsDir != "" {
+ c.assetsDir, err = filepath.Abs(c.assetsDir)
+ if err != nil {
+ return fmt.Errorf("creating absolute path from assets path: %v", err)
+ }
+ if info, err := os.Stat(c.assetsDir); os.IsNotExist(err) {
+ return fmt.Errorf("assets not found at %q: did you mean to specify assets-dir?", c.assetsDir)
+ } else if err != nil {
+ return fmt.Errorf("stat assets %q: %v", c.assetsDir, err)
+ } else if info.Mode()&os.ModeDir == 0 {
+ return fmt.Errorf("%q is not a directory", c.assetsDir)
+ }
+ c.assetsFS = os.DirFS(c.assetsDir)
+ } else {
+ if c.assetsCache == "" {
+ return fmt.Errorf("missing assets cache and assets directory: cannot proceed without assets")
+ }
+ c.assetsCache, err = filepath.Abs(c.assetsCache)
+ if err != nil {
+ return fmt.Errorf("creating absolute path from assets cache path: %v", err)
+ }
+ if info, err := os.Stat(c.assetsCache); os.IsNotExist(err) {
+ return fmt.Errorf("assets not found at %q: forgot to run `sweet get`?", c.assetsDir)
+ } else if err != nil {
+ return fmt.Errorf("stat assets %q: %v", c.assetsDir, err)
+ } else if info.Mode()&os.ModeDir == 0 {
+ return fmt.Errorf("%q is not a directory", c.assetsDir)
+ }
+ assetsFile, err := bootstrap.CachedAssets(c.assetsCache, common.Version)
+ if err == bootstrap.ErrNotInCache {
+ return fmt.Errorf("assets for version %q not found in %q", common.Version, c.assetsCache)
+ } else if err != nil {
+ return err
+ }
+ f, err := os.Open(assetsFile)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ fi, err := f.Stat()
+ if err != nil {
+ return err
+ }
+ c.assetsFS, err = zip.NewReader(f, fi.Size())
+ if err != nil {
+ return err
+ }
}
log.Printf("Work directory: %s", c.workDir)
diff --git a/sweet/common/fileutil/copy.go b/sweet/common/fileutil/copy.go
index 288e8f5..2fc4004 100644
--- a/sweet/common/fileutil/copy.go
+++ b/sweet/common/fileutil/copy.go
@@ -7,7 +7,7 @@
import (
"fmt"
"io"
- "io/ioutil"
+ "io/fs"
"os"
"path/filepath"
)
@@ -38,10 +38,19 @@
// src to a new file created at dst with the same file mode
// as the old one.
//
+// If srcFS != nil, then src is assumed to be a path within
+// srcFS.
+//
// Returns a non-nil error if copying or acquiring the
// os.FileInfo for the file fails.
-func CopyFile(dst, src string, sfinfo os.FileInfo) error {
- sf, err := os.Open(src)
+func CopyFile(dst, src string, sfinfo fs.FileInfo, srcFS fs.FS) error {
+ var sf fs.File
+ var err error
+ if srcFS != nil {
+ sf, err = srcFS.Open(src)
+ } else {
+ sf, err = os.Open(src)
+ }
if err != nil {
return err
}
@@ -61,49 +70,18 @@
return err
}
-// CopySymlink takes the symlink at path src and installs a new
-// symlink at path dst which contains the same link path. As a result,
-// relative symlinks point to a new location, relative to dst.
-//
-// sfinfo should be the result of an Lstat on src, and should always
-// indicate a symlink. If not, or if sfinfo is nil, then the os.FileInfo
-// for the symlink at src is regenerated.
-//
-// In effect, sfinfo is just an optimization to avoid
-// querying the path for the os.FileInfo more than necessary.
-//
-// Returns a non-nil error if the path src doesn't point to a symlink
-// or if an error is encountered in reading the link or installing
-// a new link.
-func CopySymlink(dst, src string, sfinfo os.FileInfo) error {
- if sfinfo == nil || sfinfo.Mode()&os.ModeSymlink == 0 {
- var err error
- sfinfo, err = os.Lstat(src)
- if err != nil {
- return err
- }
- }
- if sfinfo.Mode()&os.ModeSymlink == 0 {
- return fmt.Errorf("source file is not a symlink")
- }
- // Handle a symlink by copying the
- // link verbatim.
- link, err := os.Readlink(src)
- if err != nil {
- return err
- }
- return os.Symlink(link, dst)
-}
-
// CopyDir recursively copies the directory at path src to
// a new directory at path dst. If a symlink is encountered
// along the way, its link is copied verbatim and installed
// in the destination directory heirarchy, as in CopySymlink.
//
+// If srcFS != nil, then src is assumed to be a path within
+// srcFS.
+//
// dst and directories under dst may not retain the permissions
// of src or the corresponding directories under src. Instead,
// we always set the permissions of the new directories to 0755.
-func CopyDir(dst, src string) error {
+func CopyDir(dst, src string, srcFS fs.FS) error {
// Ignore the permissions of src, since if dst
// isn't writable we can't actually copy files into it.
// Pick a safe default that allows us to modify the
@@ -112,22 +90,30 @@
if err := os.MkdirAll(dst, 0755); err != nil {
return err
}
- fs, err := ioutil.ReadDir(src)
+ var des []fs.DirEntry
+ var err error
+ if srcFS != nil {
+ des, err = fs.ReadDir(srcFS, src)
+ } else {
+ des, err = os.ReadDir(src)
+ }
if err != nil {
return err
}
- for _, fi := range fs {
+ for _, de := range des {
+ fi, err := de.Info()
+ if err != nil {
+ return err
+ }
d, s := filepath.Join(dst, fi.Name()), filepath.Join(src, fi.Name())
if fi.IsDir() {
- if err := CopyDir(d, s); err != nil {
+ if err := CopyDir(d, s, srcFS); err != nil {
return err
}
} else if fi.Mode()&os.ModeSymlink != 0 {
- if err := CopySymlink(d, s, fi); err != nil {
- return err
- }
+ return fmt.Errorf("symbolic links not supported")
} else {
- if err := CopyFile(d, s, fi); err != nil {
+ if err := CopyFile(d, s, fi, srcFS); err != nil {
return err
}
}
diff --git a/sweet/generators/copy.go b/sweet/generators/copy.go
index 2b051c2..9163015 100644
--- a/sweet/generators/copy.go
+++ b/sweet/generators/copy.go
@@ -34,7 +34,7 @@
for _, relPath := range relPaths {
outputPath := filepath.Join(dstPath, relPath)
inputPath := filepath.Join(srcPath, relPath)
- err := fileutil.CopyFile(outputPath, inputPath, nil)
+ err := fileutil.CopyFile(outputPath, inputPath, nil, nil)
if err != nil {
return err
}
diff --git a/sweet/generators/gvisor.go b/sweet/generators/gvisor.go
index 483f2f1..ed0b7db 100644
--- a/sweet/generators/gvisor.go
+++ b/sweet/generators/gvisor.go
@@ -165,5 +165,6 @@
return fileutil.CopyDir(
filepath.Join(cfg.OutputDir, "startup", "rootfs"),
filepath.Join(cfg.AssetsDir, "startup", "rootfs"),
+ nil,
)
}
diff --git a/sweet/generators/tile38.go b/sweet/generators/tile38.go
index 945bf98..d6bb2b2 100644
--- a/sweet/generators/tile38.go
+++ b/sweet/generators/tile38.go
@@ -145,6 +145,7 @@
err = fileutil.CopyDir(
filepath.Join(cfg.OutputDir, "data"),
tmpDataPath,
+ nil,
)
}
}()
diff --git a/sweet/harnesses/common.go b/sweet/harnesses/common.go
index 0ace974..38c7e11 100644
--- a/sweet/harnesses/common.go
+++ b/sweet/harnesses/common.go
@@ -32,7 +32,7 @@
func copyFile(dst, src string) error {
log.CommandPrintf("cp %s %s", src, dst)
- return fileutil.CopyFile(dst, src, nil)
+ return fileutil.CopyFile(dst, src, nil, nil)
}
func makeWriteable(dir string) error {
@@ -52,8 +52,3 @@
log.CommandPrintf("ln -s %s %s", src, dst)
return os.Symlink(src, dst)
}
-
-func copySymlink(dst, src string) error {
- log.CommandPrintf("cp %s %s", src, dst)
- return fileutil.CopySymlink(dst, src, nil)
-}
