| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package tar |
| |
| // TODO(dsymonds): |
| // - pax extensions |
| |
| import ( |
| "bytes" |
| "errors" |
| "io" |
| "io/ioutil" |
| "math" |
| "strconv" |
| "strings" |
| "time" |
| ) |
| |
| var ( |
| ErrHeader = errors.New("archive/tar: invalid tar header") |
| ) |
| |
| const maxNanoSecondIntSize = 9 |
| |
| // A Reader provides sequential access to the contents of a tar archive. |
| // A tar archive consists of a sequence of files. |
| // The Next method advances to the next file in the archive (including the first), |
| // and then it can be treated as an io.Reader to access the file's data. |
| type Reader struct { |
| r io.Reader |
| err error |
| pad int64 // amount of padding (ignored) after current file entry |
| curr numBytesReader // reader for current file entry |
| blk block // buffer to use as temporary local storage |
| } |
| |
| type parser struct { |
| err error // Last error seen |
| } |
| |
| // A numBytesReader is an io.Reader with a numBytes method, returning the number |
| // of bytes remaining in the underlying encoded data. |
| type numBytesReader interface { |
| io.Reader |
| numBytes() int64 |
| } |
| |
| // A regFileReader is a numBytesReader for reading file data from a tar archive. |
| type regFileReader struct { |
| r io.Reader // underlying reader |
| nb int64 // number of unread bytes for current file entry |
| } |
| |
| // A sparseFileReader is a numBytesReader for reading sparse file data from a |
| // tar archive. |
| type sparseFileReader struct { |
| rfr numBytesReader // Reads the sparse-encoded file data |
| sp []sparseEntry // The sparse map for the file |
| pos int64 // Keeps track of file position |
| total int64 // Total size of the file |
| } |
| |
| // A sparseEntry holds a single entry in a sparse file's sparse map. |
| // |
| // Sparse files are represented using a series of sparseEntrys. |
| // Despite the name, a sparseEntry represents an actual data fragment that |
| // references data found in the underlying archive stream. All regions not |
| // covered by a sparseEntry are logically filled with zeros. |
| // |
| // For example, if the underlying raw file contains the 10-byte data: |
| // var compactData = "abcdefgh" |
| // |
| // And the sparse map has the following entries: |
| // var sp = []sparseEntry{ |
| // {offset: 2, numBytes: 5} // Data fragment for [2..7] |
| // {offset: 18, numBytes: 3} // Data fragment for [18..21] |
| // } |
| // |
| // Then the content of the resulting sparse file with a "real" size of 25 is: |
| // var sparseData = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 |
| type sparseEntry struct { |
| offset int64 // Starting position of the fragment |
| numBytes int64 // Length of the fragment |
| } |
| |
| // Keywords for GNU sparse files in a PAX extended header |
| const ( |
| paxGNUSparseNumBlocks = "GNU.sparse.numblocks" |
| paxGNUSparseOffset = "GNU.sparse.offset" |
| paxGNUSparseNumBytes = "GNU.sparse.numbytes" |
| paxGNUSparseMap = "GNU.sparse.map" |
| paxGNUSparseName = "GNU.sparse.name" |
| paxGNUSparseMajor = "GNU.sparse.major" |
| paxGNUSparseMinor = "GNU.sparse.minor" |
| paxGNUSparseSize = "GNU.sparse.size" |
| paxGNUSparseRealSize = "GNU.sparse.realsize" |
| ) |
| |
| // NewReader creates a new Reader reading from r. |
| func NewReader(r io.Reader) *Reader { return &Reader{r: r} } |
| |
| // Next advances to the next entry in the tar archive. |
| // |
| // io.EOF is returned at the end of the input. |
| func (tr *Reader) Next() (*Header, error) { |
| if tr.err != nil { |
| return nil, tr.err |
| } |
| |
| var hdr *Header |
| var extHdrs map[string]string |
| |
| // Externally, Next iterates through the tar archive as if it is a series of |
| // files. Internally, the tar format often uses fake "files" to add meta |
| // data that describes the next file. These meta data "files" should not |
| // normally be visible to the outside. As such, this loop iterates through |
| // one or more "header files" until it finds a "normal file". |
| loop: |
| for { |
| tr.err = tr.skipUnread() |
| if tr.err != nil { |
| return nil, tr.err |
| } |
| |
| hdr = tr.readHeader() |
| if tr.err != nil { |
| return nil, tr.err |
| } |
| |
| // Check for PAX/GNU special headers and files. |
| switch hdr.Typeflag { |
| case TypeXHeader: |
| extHdrs, tr.err = parsePAX(tr) |
| if tr.err != nil { |
| return nil, tr.err |
| } |
| continue loop // This is a meta header affecting the next header |
| case TypeGNULongName, TypeGNULongLink: |
| var realname []byte |
| realname, tr.err = ioutil.ReadAll(tr) |
| if tr.err != nil { |
| return nil, tr.err |
| } |
| |
| // Convert GNU extensions to use PAX headers. |
| if extHdrs == nil { |
| extHdrs = make(map[string]string) |
| } |
| var p parser |
| switch hdr.Typeflag { |
| case TypeGNULongName: |
| extHdrs[paxPath] = p.parseString(realname) |
| case TypeGNULongLink: |
| extHdrs[paxLinkpath] = p.parseString(realname) |
| } |
| if p.err != nil { |
| tr.err = p.err |
| return nil, tr.err |
| } |
| continue loop // This is a meta header affecting the next header |
| default: |
| mergePAX(hdr, extHdrs) |
| |
| // Check for a PAX format sparse file |
| sp, err := tr.checkForGNUSparsePAXHeaders(hdr, extHdrs) |
| if err != nil { |
| tr.err = err |
| return nil, err |
| } |
| if sp != nil { |
| // Current file is a PAX format GNU sparse file. |
| // Set the current file reader to a sparse file reader. |
| tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size) |
| if tr.err != nil { |
| return nil, tr.err |
| } |
| } |
| break loop // This is a file, so stop |
| } |
| } |
| return hdr, nil |
| } |
| |
| // checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then |
| // this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to |
| // be treated as a regular file. |
| func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) { |
| var sparseFormat string |
| |
| // Check for sparse format indicators |
| major, majorOk := headers[paxGNUSparseMajor] |
| minor, minorOk := headers[paxGNUSparseMinor] |
| sparseName, sparseNameOk := headers[paxGNUSparseName] |
| _, sparseMapOk := headers[paxGNUSparseMap] |
| sparseSize, sparseSizeOk := headers[paxGNUSparseSize] |
| sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize] |
| |
| // Identify which, if any, sparse format applies from which PAX headers are set |
| if majorOk && minorOk { |
| sparseFormat = major + "." + minor |
| } else if sparseNameOk && sparseMapOk { |
| sparseFormat = "0.1" |
| } else if sparseSizeOk { |
| sparseFormat = "0.0" |
| } else { |
| // Not a PAX format GNU sparse file. |
| return nil, nil |
| } |
| |
| // Check for unknown sparse format |
| if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" { |
| return nil, nil |
| } |
| |
| // Update hdr from GNU sparse PAX headers |
| if sparseNameOk { |
| hdr.Name = sparseName |
| } |
| if sparseSizeOk { |
| realSize, err := strconv.ParseInt(sparseSize, 10, 0) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| hdr.Size = realSize |
| } else if sparseRealSizeOk { |
| realSize, err := strconv.ParseInt(sparseRealSize, 10, 0) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| hdr.Size = realSize |
| } |
| |
| // Set up the sparse map, according to the particular sparse format in use |
| var sp []sparseEntry |
| var err error |
| switch sparseFormat { |
| case "0.0", "0.1": |
| sp, err = readGNUSparseMap0x1(headers) |
| case "1.0": |
| sp, err = readGNUSparseMap1x0(tr.curr) |
| } |
| return sp, err |
| } |
| |
| // mergePAX merges well known headers according to PAX standard. |
| // In general headers with the same name as those found |
| // in the header struct overwrite those found in the header |
| // struct with higher precision or longer values. Esp. useful |
| // for name and linkname fields. |
| func mergePAX(hdr *Header, headers map[string]string) error { |
| for k, v := range headers { |
| switch k { |
| case paxPath: |
| hdr.Name = v |
| case paxLinkpath: |
| hdr.Linkname = v |
| case paxGname: |
| hdr.Gname = v |
| case paxUname: |
| hdr.Uname = v |
| case paxUid: |
| uid, err := strconv.ParseInt(v, 10, 0) |
| if err != nil { |
| return err |
| } |
| hdr.Uid = int(uid) |
| case paxGid: |
| gid, err := strconv.ParseInt(v, 10, 0) |
| if err != nil { |
| return err |
| } |
| hdr.Gid = int(gid) |
| case paxAtime: |
| t, err := parsePAXTime(v) |
| if err != nil { |
| return err |
| } |
| hdr.AccessTime = t |
| case paxMtime: |
| t, err := parsePAXTime(v) |
| if err != nil { |
| return err |
| } |
| hdr.ModTime = t |
| case paxCtime: |
| t, err := parsePAXTime(v) |
| if err != nil { |
| return err |
| } |
| hdr.ChangeTime = t |
| case paxSize: |
| size, err := strconv.ParseInt(v, 10, 0) |
| if err != nil { |
| return err |
| } |
| hdr.Size = size |
| default: |
| if strings.HasPrefix(k, paxXattr) { |
| if hdr.Xattrs == nil { |
| hdr.Xattrs = make(map[string]string) |
| } |
| hdr.Xattrs[k[len(paxXattr):]] = v |
| } |
| } |
| } |
| return nil |
| } |
| |
| // parsePAXTime takes a string of the form %d.%d as described in |
| // the PAX specification. |
| func parsePAXTime(t string) (time.Time, error) { |
| buf := []byte(t) |
| pos := bytes.IndexByte(buf, '.') |
| var seconds, nanoseconds int64 |
| var err error |
| if pos == -1 { |
| seconds, err = strconv.ParseInt(t, 10, 0) |
| if err != nil { |
| return time.Time{}, err |
| } |
| } else { |
| seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0) |
| if err != nil { |
| return time.Time{}, err |
| } |
| nanoBuf := string(buf[pos+1:]) |
| // Pad as needed before converting to a decimal. |
| // For example .030 -> .030000000 -> 30000000 nanoseconds |
| if len(nanoBuf) < maxNanoSecondIntSize { |
| // Right pad |
| nanoBuf += strings.Repeat("0", maxNanoSecondIntSize-len(nanoBuf)) |
| } else if len(nanoBuf) > maxNanoSecondIntSize { |
| // Right truncate |
| nanoBuf = nanoBuf[:maxNanoSecondIntSize] |
| } |
| nanoseconds, err = strconv.ParseInt(nanoBuf, 10, 0) |
| if err != nil { |
| return time.Time{}, err |
| } |
| } |
| ts := time.Unix(seconds, nanoseconds) |
| return ts, nil |
| } |
| |
| // parsePAX parses PAX headers. |
| // If an extended header (type 'x') is invalid, ErrHeader is returned |
| func parsePAX(r io.Reader) (map[string]string, error) { |
| buf, err := ioutil.ReadAll(r) |
| if err != nil { |
| return nil, err |
| } |
| sbuf := string(buf) |
| |
| // For GNU PAX sparse format 0.0 support. |
| // This function transforms the sparse format 0.0 headers into sparse format 0.1 headers. |
| var sparseMap bytes.Buffer |
| |
| headers := make(map[string]string) |
| // Each record is constructed as |
| // "%d %s=%s\n", length, keyword, value |
| for len(sbuf) > 0 { |
| key, value, residual, err := parsePAXRecord(sbuf) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| sbuf = residual |
| |
| keyStr := key |
| if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes { |
| // GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map. |
| sparseMap.WriteString(value) |
| sparseMap.Write([]byte{','}) |
| } else { |
| // Normal key. Set the value in the headers map. |
| headers[keyStr] = value |
| } |
| } |
| if sparseMap.Len() != 0 { |
| // Add sparse info to headers, chopping off the extra comma |
| sparseMap.Truncate(sparseMap.Len() - 1) |
| headers[paxGNUSparseMap] = sparseMap.String() |
| } |
| return headers, nil |
| } |
| |
| // parsePAXRecord parses the input PAX record string into a key-value pair. |
| // If parsing is successful, it will slice off the currently read record and |
| // return the remainder as r. |
| // |
| // A PAX record is of the following form: |
| // "%d %s=%s\n" % (size, key, value) |
| func parsePAXRecord(s string) (k, v, r string, err error) { |
| // The size field ends at the first space. |
| sp := strings.IndexByte(s, ' ') |
| if sp == -1 { |
| return "", "", s, ErrHeader |
| } |
| |
| // Parse the first token as a decimal integer. |
| n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int |
| if perr != nil || n < 5 || int64(len(s)) < n { |
| return "", "", s, ErrHeader |
| } |
| |
| // Extract everything between the space and the final newline. |
| rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:] |
| if nl != "\n" { |
| return "", "", s, ErrHeader |
| } |
| |
| // The first equals separates the key from the value. |
| eq := strings.IndexByte(rec, '=') |
| if eq == -1 { |
| return "", "", s, ErrHeader |
| } |
| return rec[:eq], rec[eq+1:], rem, nil |
| } |
| |
| // parseString parses bytes as a NUL-terminated C-style string. |
| // If a NUL byte is not found then the whole slice is returned as a string. |
| func (*parser) parseString(b []byte) string { |
| n := 0 |
| for n < len(b) && b[n] != 0 { |
| n++ |
| } |
| return string(b[0:n]) |
| } |
| |
| // parseNumeric parses the input as being encoded in either base-256 or octal. |
| // This function may return negative numbers. |
| // If parsing fails or an integer overflow occurs, err will be set. |
| func (p *parser) parseNumeric(b []byte) int64 { |
| // Check for base-256 (binary) format first. |
| // If the first bit is set, then all following bits constitute a two's |
| // complement encoded number in big-endian byte order. |
| if len(b) > 0 && b[0]&0x80 != 0 { |
| // Handling negative numbers relies on the following identity: |
| // -a-1 == ^a |
| // |
| // If the number is negative, we use an inversion mask to invert the |
| // data bytes and treat the value as an unsigned number. |
| var inv byte // 0x00 if positive or zero, 0xff if negative |
| if b[0]&0x40 != 0 { |
| inv = 0xff |
| } |
| |
| var x uint64 |
| for i, c := range b { |
| c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing |
| if i == 0 { |
| c &= 0x7f // Ignore signal bit in first byte |
| } |
| if (x >> 56) > 0 { |
| p.err = ErrHeader // Integer overflow |
| return 0 |
| } |
| x = x<<8 | uint64(c) |
| } |
| if (x >> 63) > 0 { |
| p.err = ErrHeader // Integer overflow |
| return 0 |
| } |
| if inv == 0xff { |
| return ^int64(x) |
| } |
| return int64(x) |
| } |
| |
| // Normal case is base-8 (octal) format. |
| return p.parseOctal(b) |
| } |
| |
| func (p *parser) parseOctal(b []byte) int64 { |
| // Because unused fields are filled with NULs, we need |
| // to skip leading NULs. Fields may also be padded with |
| // spaces or NULs. |
| // So we remove leading and trailing NULs and spaces to |
| // be sure. |
| b = bytes.Trim(b, " \x00") |
| |
| if len(b) == 0 { |
| return 0 |
| } |
| x, perr := strconv.ParseUint(p.parseString(b), 8, 64) |
| if perr != nil { |
| p.err = ErrHeader |
| } |
| return int64(x) |
| } |
| |
| // skipUnread skips any unread bytes in the existing file entry, as well as any |
| // alignment padding. It returns io.ErrUnexpectedEOF if any io.EOF is |
| // encountered in the data portion; it is okay to hit io.EOF in the padding. |
| // |
| // Note that this function still works properly even when sparse files are being |
| // used since numBytes returns the bytes remaining in the underlying io.Reader. |
| func (tr *Reader) skipUnread() error { |
| dataSkip := tr.numBytes() // Number of data bytes to skip |
| totalSkip := dataSkip + tr.pad // Total number of bytes to skip |
| tr.curr, tr.pad = nil, 0 |
| |
| // If possible, Seek to the last byte before the end of the data section. |
| // Do this because Seek is often lazy about reporting errors; this will mask |
| // the fact that the tar stream may be truncated. We can rely on the |
| // io.CopyN done shortly afterwards to trigger any IO errors. |
| var seekSkipped int64 // Number of bytes skipped via Seek |
| if sr, ok := tr.r.(io.Seeker); ok && dataSkip > 1 { |
| // Not all io.Seeker can actually Seek. For example, os.Stdin implements |
| // io.Seeker, but calling Seek always returns an error and performs |
| // no action. Thus, we try an innocent seek to the current position |
| // to see if Seek is really supported. |
| pos1, err := sr.Seek(0, io.SeekCurrent) |
| if err == nil { |
| // Seek seems supported, so perform the real Seek. |
| pos2, err := sr.Seek(dataSkip-1, io.SeekCurrent) |
| if err != nil { |
| tr.err = err |
| return tr.err |
| } |
| seekSkipped = pos2 - pos1 |
| } |
| } |
| |
| var copySkipped int64 // Number of bytes skipped via CopyN |
| copySkipped, tr.err = io.CopyN(ioutil.Discard, tr.r, totalSkip-seekSkipped) |
| if tr.err == io.EOF && seekSkipped+copySkipped < dataSkip { |
| tr.err = io.ErrUnexpectedEOF |
| } |
| return tr.err |
| } |
| |
| // readHeader reads the next block header and assumes that the underlying reader |
| // is already aligned to a block boundary. |
| // |
| // The err will be set to io.EOF only when one of the following occurs: |
| // * Exactly 0 bytes are read and EOF is hit. |
| // * Exactly 1 block of zeros is read and EOF is hit. |
| // * At least 2 blocks of zeros are read. |
| func (tr *Reader) readHeader() *Header { |
| if _, tr.err = io.ReadFull(tr.r, tr.blk[:]); tr.err != nil { |
| return nil // io.EOF is okay here |
| } |
| |
| // Two blocks of zero bytes marks the end of the archive. |
| if bytes.Equal(tr.blk[:], zeroBlock[:]) { |
| if _, tr.err = io.ReadFull(tr.r, tr.blk[:]); tr.err != nil { |
| return nil // io.EOF is okay here |
| } |
| if bytes.Equal(tr.blk[:], zeroBlock[:]) { |
| tr.err = io.EOF |
| } else { |
| tr.err = ErrHeader // zero block and then non-zero block |
| } |
| return nil |
| } |
| |
| // Verify the header matches a known format. |
| format := tr.blk.GetFormat() |
| if format == formatUnknown { |
| tr.err = ErrHeader |
| return nil |
| } |
| |
| var p parser |
| hdr := new(Header) |
| |
| // Unpack the V7 header. |
| v7 := tr.blk.V7() |
| hdr.Name = p.parseString(v7.Name()) |
| hdr.Mode = p.parseNumeric(v7.Mode()) |
| hdr.Uid = int(p.parseNumeric(v7.UID())) |
| hdr.Gid = int(p.parseNumeric(v7.GID())) |
| hdr.Size = p.parseNumeric(v7.Size()) |
| hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0) |
| hdr.Typeflag = v7.TypeFlag()[0] |
| hdr.Linkname = p.parseString(v7.LinkName()) |
| |
| // Unpack format specific fields. |
| if format > formatV7 { |
| ustar := tr.blk.USTAR() |
| hdr.Uname = p.parseString(ustar.UserName()) |
| hdr.Gname = p.parseString(ustar.GroupName()) |
| if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { |
| hdr.Devmajor = p.parseNumeric(ustar.DevMajor()) |
| hdr.Devminor = p.parseNumeric(ustar.DevMinor()) |
| } |
| |
| var prefix string |
| switch format { |
| case formatUSTAR, formatGNU: |
| // TODO(dsnet): Do not use the prefix field for the GNU format! |
| // See golang.org/issues/12594 |
| ustar := tr.blk.USTAR() |
| prefix = p.parseString(ustar.Prefix()) |
| case formatSTAR: |
| star := tr.blk.STAR() |
| prefix = p.parseString(star.Prefix()) |
| hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0) |
| hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0) |
| } |
| if len(prefix) > 0 { |
| hdr.Name = prefix + "/" + hdr.Name |
| } |
| } |
| |
| nb := hdr.Size |
| if isHeaderOnlyType(hdr.Typeflag) { |
| nb = 0 |
| } |
| if nb < 0 { |
| tr.err = ErrHeader |
| return nil |
| } |
| |
| // Set the current file reader. |
| tr.pad = -nb & (blockSize - 1) // blockSize is a power of two |
| tr.curr = ®FileReader{r: tr.r, nb: nb} |
| |
| // Check for old GNU sparse format entry. |
| if hdr.Typeflag == TypeGNUSparse { |
| // Get the real size of the file. |
| hdr.Size = p.parseNumeric(tr.blk.GNU().RealSize()) |
| if p.err != nil { |
| tr.err = p.err |
| return nil |
| } |
| |
| // Read the sparse map. |
| sp := tr.readOldGNUSparseMap(&tr.blk) |
| if tr.err != nil { |
| return nil |
| } |
| |
| // Current file is a GNU sparse file. Update the current file reader. |
| tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size) |
| if tr.err != nil { |
| return nil |
| } |
| } |
| |
| if p.err != nil { |
| tr.err = p.err |
| return nil |
| } |
| |
| return hdr |
| } |
| |
| // readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format. |
| // The sparse map is stored in the tar header if it's small enough. If it's larger than four entries, |
| // then one or more extension headers are used to store the rest of the sparse map. |
| func (tr *Reader) readOldGNUSparseMap(blk *block) []sparseEntry { |
| var p parser |
| var s sparseArray = blk.GNU().Sparse() |
| var sp = make([]sparseEntry, 0, s.MaxEntries()) |
| for i := 0; i < s.MaxEntries(); i++ { |
| offset := p.parseOctal(s.Entry(i).Offset()) |
| numBytes := p.parseOctal(s.Entry(i).NumBytes()) |
| if p.err != nil { |
| tr.err = p.err |
| return nil |
| } |
| if offset == 0 && numBytes == 0 { |
| break |
| } |
| sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) |
| } |
| |
| for s.IsExtended()[0] > 0 { |
| // There are more entries. Read an extension header and parse its entries. |
| var blk block |
| if _, tr.err = io.ReadFull(tr.r, blk[:]); tr.err != nil { |
| return nil |
| } |
| s = blk.Sparse() |
| |
| for i := 0; i < s.MaxEntries(); i++ { |
| offset := p.parseOctal(s.Entry(i).Offset()) |
| numBytes := p.parseOctal(s.Entry(i).NumBytes()) |
| if p.err != nil { |
| tr.err = p.err |
| return nil |
| } |
| if offset == 0 && numBytes == 0 { |
| break |
| } |
| sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) |
| } |
| } |
| return sp |
| } |
| |
| // readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format |
| // version 1.0. The format of the sparse map consists of a series of |
| // newline-terminated numeric fields. The first field is the number of entries |
| // and is always present. Following this are the entries, consisting of two |
| // fields (offset, numBytes). This function must stop reading at the end |
| // boundary of the block containing the last newline. |
| // |
| // Note that the GNU manual says that numeric values should be encoded in octal |
| // format. However, the GNU tar utility itself outputs these values in decimal. |
| // As such, this library treats values as being encoded in decimal. |
| func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { |
| var cntNewline int64 |
| var buf bytes.Buffer |
| var blk = make([]byte, blockSize) |
| |
| // feedTokens copies data in numBlock chunks from r into buf until there are |
| // at least cnt newlines in buf. It will not read more blocks than needed. |
| var feedTokens = func(cnt int64) error { |
| for cntNewline < cnt { |
| if _, err := io.ReadFull(r, blk); err != nil { |
| if err == io.EOF { |
| err = io.ErrUnexpectedEOF |
| } |
| return err |
| } |
| buf.Write(blk) |
| for _, c := range blk { |
| if c == '\n' { |
| cntNewline++ |
| } |
| } |
| } |
| return nil |
| } |
| |
| // nextToken gets the next token delimited by a newline. This assumes that |
| // at least one newline exists in the buffer. |
| var nextToken = func() string { |
| cntNewline-- |
| tok, _ := buf.ReadString('\n') |
| return tok[:len(tok)-1] // Cut off newline |
| } |
| |
| // Parse for the number of entries. |
| // Use integer overflow resistant math to check this. |
| if err := feedTokens(1); err != nil { |
| return nil, err |
| } |
| numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int |
| if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { |
| return nil, ErrHeader |
| } |
| |
| // Parse for all member entries. |
| // numEntries is trusted after this since a potential attacker must have |
| // committed resources proportional to what this library used. |
| if err := feedTokens(2 * numEntries); err != nil { |
| return nil, err |
| } |
| sp := make([]sparseEntry, 0, numEntries) |
| for i := int64(0); i < numEntries; i++ { |
| offset, err := strconv.ParseInt(nextToken(), 10, 64) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| numBytes, err := strconv.ParseInt(nextToken(), 10, 64) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) |
| } |
| return sp, nil |
| } |
| |
| // readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format |
| // version 0.1. The sparse map is stored in the PAX headers. |
| func readGNUSparseMap0x1(extHdrs map[string]string) ([]sparseEntry, error) { |
| // Get number of entries. |
| // Use integer overflow resistant math to check this. |
| numEntriesStr := extHdrs[paxGNUSparseNumBlocks] |
| numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int |
| if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { |
| return nil, ErrHeader |
| } |
| |
| // There should be two numbers in sparseMap for each entry. |
| sparseMap := strings.Split(extHdrs[paxGNUSparseMap], ",") |
| if int64(len(sparseMap)) != 2*numEntries { |
| return nil, ErrHeader |
| } |
| |
| // Loop through the entries in the sparse map. |
| // numEntries is trusted now. |
| sp := make([]sparseEntry, 0, numEntries) |
| for i := int64(0); i < numEntries; i++ { |
| offset, err := strconv.ParseInt(sparseMap[2*i], 10, 64) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 64) |
| if err != nil { |
| return nil, ErrHeader |
| } |
| sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) |
| } |
| return sp, nil |
| } |
| |
| // numBytes returns the number of bytes left to read in the current file's entry |
| // in the tar archive, or 0 if there is no current file. |
| func (tr *Reader) numBytes() int64 { |
| if tr.curr == nil { |
| // No current file, so no bytes |
| return 0 |
| } |
| return tr.curr.numBytes() |
| } |
| |
| // Read reads from the current entry in the tar archive. |
| // It returns 0, io.EOF when it reaches the end of that entry, |
| // until Next is called to advance to the next entry. |
| // |
| // Calling Read on special types like TypeLink, TypeSymLink, TypeChar, |
| // TypeBlock, TypeDir, and TypeFifo returns 0, io.EOF regardless of what |
| // the Header.Size claims. |
| func (tr *Reader) Read(b []byte) (n int, err error) { |
| if tr.err != nil { |
| return 0, tr.err |
| } |
| if tr.curr == nil { |
| return 0, io.EOF |
| } |
| |
| n, err = tr.curr.Read(b) |
| if err != nil && err != io.EOF { |
| tr.err = err |
| } |
| return |
| } |
| |
| func (rfr *regFileReader) Read(b []byte) (n int, err error) { |
| if rfr.nb == 0 { |
| // file consumed |
| return 0, io.EOF |
| } |
| if int64(len(b)) > rfr.nb { |
| b = b[0:rfr.nb] |
| } |
| n, err = rfr.r.Read(b) |
| rfr.nb -= int64(n) |
| |
| if err == io.EOF && rfr.nb > 0 { |
| err = io.ErrUnexpectedEOF |
| } |
| return |
| } |
| |
| // numBytes returns the number of bytes left to read in the file's data in the tar archive. |
| func (rfr *regFileReader) numBytes() int64 { |
| return rfr.nb |
| } |
| |
| // newSparseFileReader creates a new sparseFileReader, but validates all of the |
| // sparse entries before doing so. |
| func newSparseFileReader(rfr numBytesReader, sp []sparseEntry, total int64) (*sparseFileReader, error) { |
| if total < 0 { |
| return nil, ErrHeader // Total size cannot be negative |
| } |
| |
| // Validate all sparse entries. These are the same checks as performed by |
| // the BSD tar utility. |
| for i, s := range sp { |
| switch { |
| case s.offset < 0 || s.numBytes < 0: |
| return nil, ErrHeader // Negative values are never okay |
| case s.offset > math.MaxInt64-s.numBytes: |
| return nil, ErrHeader // Integer overflow with large length |
| case s.offset+s.numBytes > total: |
| return nil, ErrHeader // Region extends beyond the "real" size |
| case i > 0 && sp[i-1].offset+sp[i-1].numBytes > s.offset: |
| return nil, ErrHeader // Regions can't overlap and must be in order |
| } |
| } |
| return &sparseFileReader{rfr: rfr, sp: sp, total: total}, nil |
| } |
| |
| // readHole reads a sparse hole ending at endOffset. |
| func (sfr *sparseFileReader) readHole(b []byte, endOffset int64) int { |
| n64 := endOffset - sfr.pos |
| if n64 > int64(len(b)) { |
| n64 = int64(len(b)) |
| } |
| n := int(n64) |
| for i := 0; i < n; i++ { |
| b[i] = 0 |
| } |
| sfr.pos += n64 |
| return n |
| } |
| |
| // Read reads the sparse file data in expanded form. |
| func (sfr *sparseFileReader) Read(b []byte) (n int, err error) { |
| // Skip past all empty fragments. |
| for len(sfr.sp) > 0 && sfr.sp[0].numBytes == 0 { |
| sfr.sp = sfr.sp[1:] |
| } |
| |
| // If there are no more fragments, then it is possible that there |
| // is one last sparse hole. |
| if len(sfr.sp) == 0 { |
| // This behavior matches the BSD tar utility. |
| // However, GNU tar stops returning data even if sfr.total is unmet. |
| if sfr.pos < sfr.total { |
| return sfr.readHole(b, sfr.total), nil |
| } |
| return 0, io.EOF |
| } |
| |
| // In front of a data fragment, so read a hole. |
| if sfr.pos < sfr.sp[0].offset { |
| return sfr.readHole(b, sfr.sp[0].offset), nil |
| } |
| |
| // In a data fragment, so read from it. |
| // This math is overflow free since we verify that offset and numBytes can |
| // be safely added when creating the sparseFileReader. |
| endPos := sfr.sp[0].offset + sfr.sp[0].numBytes // End offset of fragment |
| bytesLeft := endPos - sfr.pos // Bytes left in fragment |
| if int64(len(b)) > bytesLeft { |
| b = b[:bytesLeft] |
| } |
| |
| n, err = sfr.rfr.Read(b) |
| sfr.pos += int64(n) |
| if err == io.EOF { |
| if sfr.pos < endPos { |
| err = io.ErrUnexpectedEOF // There was supposed to be more data |
| } else if sfr.pos < sfr.total { |
| err = nil // There is still an implicit sparse hole at the end |
| } |
| } |
| |
| if sfr.pos == endPos { |
| sfr.sp = sfr.sp[1:] // We are done with this fragment, so pop it |
| } |
| return n, err |
| } |
| |
| // numBytes returns the number of bytes left to read in the sparse file's |
| // sparse-encoded data in the tar archive. |
| func (sfr *sparseFileReader) numBytes() int64 { |
| return sfr.rfr.numBytes() |
| } |