diff --git a/archive/tar/common.go b/archive/tar/common.go index 5141bf9..0378401 100644 --- a/archive/tar/common.go +++ b/archive/tar/common.go @@ -44,22 +44,23 @@ const ( // A Header represents a single header in a tar archive. // Some fields may not be populated. type Header struct { - Name string // name of header file entry - Mode int64 // permission and mode bits - Uid int // user id of owner - Gid int // group id of owner - Size int64 // length in bytes - ModTime time.Time // modified time - Typeflag byte // type of header entry - Linkname string // target name of link - Uname string // user name of owner - Gname string // group name of owner - Devmajor int64 // major number of character or block device - Devminor int64 // minor number of character or block device - AccessTime time.Time // access time - ChangeTime time.Time // status change time - Xattrs map[string]string - Winheaders map[string]string + Name string // name of header file entry + Mode int64 // permission and mode bits + Uid int // user id of owner + Gid int // group id of owner + Size int64 // length in bytes + ModTime time.Time // modified time + Typeflag byte // type of header entry + Linkname string // target name of link + Uname string // user name of owner + Gname string // group name of owner + Devmajor int64 // major number of character or block device + Devminor int64 // minor number of character or block device + AccessTime time.Time // access time + ChangeTime time.Time // status change time + CreationTime time.Time // creation time + Xattrs map[string]string + Winheaders map[string]string } // File name constants from the tar spec. @@ -180,21 +181,22 @@ const ( // Keywords for the PAX Extended Header const ( - paxAtime = "atime" - paxCharset = "charset" - paxComment = "comment" - paxCtime = "ctime" // please note that ctime is not a valid pax header. - paxGid = "gid" - paxGname = "gname" - paxLinkpath = "linkpath" - paxMtime = "mtime" - paxPath = "path" - paxSize = "size" - paxUid = "uid" - paxUname = "uname" - paxXattr = "SCHILY.xattr." - paxWindows = "MSWINDOWS." - paxNone = "" + paxAtime = "atime" + paxCharset = "charset" + paxComment = "comment" + paxCtime = "ctime" // please note that ctime is not a valid pax header. + paxCreationTime = "LIBARCHIVE.creationtime" + paxGid = "gid" + paxGname = "gname" + paxLinkpath = "linkpath" + paxMtime = "mtime" + paxPath = "path" + paxSize = "size" + paxUid = "uid" + paxUname = "uname" + paxXattr = "SCHILY.xattr." + paxWindows = "MSWINDOWS." + paxNone = "" ) // FileInfoHeader creates a partially-populated Header from fi. diff --git a/archive/tar/reader.go b/archive/tar/reader.go index 6aee36c..e210c61 100644 --- a/archive/tar/reader.go +++ b/archive/tar/reader.go @@ -302,6 +302,12 @@ func mergePAX(hdr *Header, headers map[string]string) error { return err } hdr.ChangeTime = t + case paxCreationTime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.CreationTime = t case paxSize: size, err := strconv.ParseInt(v, 10, 0) if err != nil { diff --git a/archive/tar/testdata/gnu-multi-hdrs.tar b/archive/tar/testdata/gnu-multi-hdrs.tar new file mode 100644 index 0000000..8bcad55 Binary files /dev/null and b/archive/tar/testdata/gnu-multi-hdrs.tar differ diff --git a/archive/tar/testdata/gnu.tar b/archive/tar/testdata/gnu.tar new file mode 100644 index 0000000..fc899dc Binary files /dev/null and b/archive/tar/testdata/gnu.tar differ diff --git a/archive/tar/testdata/hardlink.tar b/archive/tar/testdata/hardlink.tar new file mode 100644 index 0000000..9cd1a26 Binary files /dev/null and b/archive/tar/testdata/hardlink.tar differ diff --git a/archive/tar/testdata/hdr-only.tar b/archive/tar/testdata/hdr-only.tar new file mode 100644 index 0000000..f250340 Binary files /dev/null and b/archive/tar/testdata/hdr-only.tar differ diff --git a/archive/tar/testdata/issue10968.tar b/archive/tar/testdata/issue10968.tar new file mode 100644 index 0000000..1cc837b Binary files /dev/null and b/archive/tar/testdata/issue10968.tar differ diff --git a/archive/tar/testdata/issue11169.tar b/archive/tar/testdata/issue11169.tar new file mode 100644 index 0000000..4d71fa1 Binary files /dev/null and b/archive/tar/testdata/issue11169.tar differ diff --git a/archive/tar/testdata/issue12435.tar b/archive/tar/testdata/issue12435.tar new file mode 100644 index 0000000..3542dd8 Binary files /dev/null and b/archive/tar/testdata/issue12435.tar differ diff --git a/archive/tar/testdata/neg-size.tar b/archive/tar/testdata/neg-size.tar new file mode 100644 index 0000000..21edf38 Binary files /dev/null and b/archive/tar/testdata/neg-size.tar differ diff --git a/archive/tar/testdata/nil-uid.tar b/archive/tar/testdata/nil-uid.tar new file mode 100644 index 0000000..cc9cfaa Binary files /dev/null and b/archive/tar/testdata/nil-uid.tar differ diff --git a/archive/tar/testdata/pax-multi-hdrs.tar b/archive/tar/testdata/pax-multi-hdrs.tar new file mode 100644 index 0000000..14bc759 Binary files /dev/null and b/archive/tar/testdata/pax-multi-hdrs.tar differ diff --git a/archive/tar/testdata/pax-path-hdr.tar b/archive/tar/testdata/pax-path-hdr.tar new file mode 100644 index 0000000..ab8fc32 Binary files /dev/null and b/archive/tar/testdata/pax-path-hdr.tar differ diff --git a/archive/tar/testdata/pax.tar b/archive/tar/testdata/pax.tar new file mode 100644 index 0000000..9bc24b6 Binary files /dev/null and b/archive/tar/testdata/pax.tar differ diff --git a/archive/tar/testdata/small.txt b/archive/tar/testdata/small.txt new file mode 100644 index 0000000..b249bfc --- /dev/null +++ b/archive/tar/testdata/small.txt @@ -0,0 +1 @@ +Kilts \ No newline at end of file diff --git a/archive/tar/testdata/small2.txt b/archive/tar/testdata/small2.txt new file mode 100644 index 0000000..394ee3e --- /dev/null +++ b/archive/tar/testdata/small2.txt @@ -0,0 +1 @@ +Google.com diff --git a/archive/tar/testdata/sparse-formats.tar b/archive/tar/testdata/sparse-formats.tar new file mode 100644 index 0000000..8bd4e74 Binary files /dev/null and b/archive/tar/testdata/sparse-formats.tar differ diff --git a/archive/tar/testdata/star.tar b/archive/tar/testdata/star.tar new file mode 100644 index 0000000..59e2d4e Binary files /dev/null and b/archive/tar/testdata/star.tar differ diff --git a/archive/tar/testdata/ustar-file-reg.tar b/archive/tar/testdata/ustar-file-reg.tar new file mode 100644 index 0000000..c84fa27 Binary files /dev/null and b/archive/tar/testdata/ustar-file-reg.tar differ diff --git a/archive/tar/testdata/ustar.tar b/archive/tar/testdata/ustar.tar new file mode 100644 index 0000000..29679d9 Binary files /dev/null and b/archive/tar/testdata/ustar.tar differ diff --git a/archive/tar/testdata/v7.tar b/archive/tar/testdata/v7.tar new file mode 100644 index 0000000..eb65fc9 Binary files /dev/null and b/archive/tar/testdata/v7.tar differ diff --git a/archive/tar/testdata/writer-big-long.tar b/archive/tar/testdata/writer-big-long.tar new file mode 100644 index 0000000..5960ee8 Binary files /dev/null and b/archive/tar/testdata/writer-big-long.tar differ diff --git a/archive/tar/testdata/writer-big.tar b/archive/tar/testdata/writer-big.tar new file mode 100644 index 0000000..753e883 Binary files /dev/null and b/archive/tar/testdata/writer-big.tar differ diff --git a/archive/tar/testdata/writer.tar b/archive/tar/testdata/writer.tar new file mode 100644 index 0000000..e6d816a Binary files /dev/null and b/archive/tar/testdata/writer.tar differ diff --git a/archive/tar/testdata/xattrs.tar b/archive/tar/testdata/xattrs.tar new file mode 100644 index 0000000..9701950 Binary files /dev/null and b/archive/tar/testdata/xattrs.tar differ diff --git a/archive/tar/writer.go b/archive/tar/writer.go index 05027a3..30d7e60 100644 --- a/archive/tar/writer.go +++ b/archive/tar/writer.go @@ -47,7 +47,7 @@ type formatter struct { } // NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } +func NewWriter(w io.Writer) *Writer { return &Writer{w: w, preferPax: true} } // Flush finishes writing the current file (optional). func (tw *Writer) Flush() error { @@ -201,23 +201,29 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { tw.usedBinary = true f.formatNumeric(b, x) } + var formatTime = func(b []byte, t time.Time, paxKeyword string) { + var unixTime int64 + if !t.Before(minTime) && !t.After(maxTime) { + unixTime = t.Unix() + } + formatNumeric(b, unixTime, paxNone) + + // Write a PAX header if the time didn't fit precisely. + if paxKeyword != "" && tw.preferPax && allowPax && (t.Nanosecond() != 0 || !t.Before(minTime) || !t.After(maxTime)) { + paxHeaders[paxKeyword] = formatPAXTime(t) + } + } // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax pathHeaderBytes := s.next(fileNameSize) formatString(pathHeaderBytes, hdr.Name, paxPath) - // Handle out of range ModTime carefully. - var modTime int64 - if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) { - modTime = hdr.ModTime.Unix() - } - f.formatOctal(s.next(8), hdr.Mode) // 100:108 formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116 formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124 formatNumeric(s.next(12), hdr.Size, paxSize) // 124:136 - formatNumeric(s.next(12), modTime, paxNone) // 136:148 --- consider using pax for finer granularity + formatTime(s.next(12), hdr.ModTime, paxMtime) // 136:148 s.next(8) // chksum (148:156) s.next(1)[0] = hdr.Typeflag // 156:157 @@ -265,6 +271,15 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { } if allowPax { + if !hdr.AccessTime.IsZero() { + paxHeaders[paxAtime] = formatPAXTime(hdr.AccessTime) + } + if !hdr.ChangeTime.IsZero() { + paxHeaders[paxCtime] = formatPAXTime(hdr.ChangeTime) + } + if !hdr.CreationTime.IsZero() { + paxHeaders[paxCreationTime] = formatPAXTime(hdr.CreationTime) + } for k, v := range hdr.Xattrs { paxHeaders[paxXattr+k] = v } @@ -288,6 +303,16 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { return tw.err } +func formatPAXTime(t time.Time) string { + sec := t.Unix() + usec := t.Nanosecond() + s := strconv.FormatInt(sec, 10) + if usec != 0 { + s = fmt.Sprintf("%s.%09d", s, usec) + } + return s +} + // splitUSTARPath splits a path according to USTAR prefix and suffix rules. // If the path is not splittable, then it will return ("", "", false). func splitUSTARPath(name string) (prefix, suffix string, ok bool) { diff --git a/archive/tar/writer_test.go b/archive/tar/writer_test.go index 6e91d90..a5c9382 100644 --- a/archive/tar/writer_test.go +++ b/archive/tar/writer_test.go @@ -720,3 +720,20 @@ func TestFormatNumeric(t *testing.T) { } } } + +func TestFormatPAXTime(t *testing.T) { + t1 := time.Date(2000, 1, 1, 11, 0, 0, 0, time.UTC) + t2 := time.Date(2000, 1, 1, 11, 0, 0, 100, time.UTC) + t3 := time.Date(1960, 1, 1, 11, 0, 0, 0, time.UTC) + t4 := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + verify := func(time time.Time, s string) { + p := formatPAXTime(time) + if p != s { + t.Errorf("for %v, expected %s, got %s", time, s, p) + } + } + verify(t1, "946724400") + verify(t2, "946724400.000000100") + verify(t3, "-315579600") + verify(t4, "0") +} diff --git a/backuptar/tar.go b/backuptar/tar.go index c988574..e578067 100644 --- a/backuptar/tar.go +++ b/backuptar/tar.go @@ -30,10 +30,6 @@ const ( const ( hdrFileAttributes = "fileattr" - hdrAccessTime = "accesstime" - hdrChangeTime = "changetime" - hdrCreateTime = "createtime" - hdrWriteTime = "writetime" hdrSecurityDescriptor = "sd" hdrMountPoint = "mountpoint" ) @@ -82,21 +78,7 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error { return nil } -func win32TimeFromTar(key string, hdrs map[string]string, unixTime time.Time) syscall.Filetime { - if s, ok := hdrs[key]; ok { - n, err := strconv.ParseUint(s, 10, 64) - if err == nil { - return syscall.Filetime{uint32(n & 0xffffffff), uint32(n >> 32)} - } - } - return syscall.NsecToFiletime(unixTime.UnixNano()) -} - -func win32TimeToTar(ft syscall.Filetime) (string, time.Time) { - return fmt.Sprintf("%d", uint64(ft.LowDateTime)+(uint64(ft.HighDateTime)<<32)), time.Unix(0, ft.Nanoseconds()) -} - -// Writes a file to a tar writer using data from a Win32 backup stream. +// WriteTarFileFromBackupStream writes a file to a tar writer using data from a Win32 backup stream. // // This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS. // @@ -104,30 +86,22 @@ func win32TimeToTar(ft syscall.Filetime) (string, time.Time) { // // MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value // -// MSWINDOWS.accesstime: The last access time, as a Filetime expressed as a 64-bit decimal value. -// -// MSWINDOWS.createtime: The creation time, as a Filetime expressed as a 64-bit decimal value. -// -// MSWINDOWS.changetime: The creation time, as a Filetime expressed as a 64-bit decimal value. -// -// MSWINDOWS.writetime: The creation time, as a Filetime expressed as a 64-bit decimal value. -// // MSWINDOWS.sd: The Win32 security descriptor, in SDDL (string) format // // MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink) func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error { name = filepath.ToSlash(name) hdr := &tar.Header{ - Name: name, - Size: size, - Typeflag: tar.TypeReg, - Winheaders: make(map[string]string), + Name: name, + Size: size, + Typeflag: tar.TypeReg, + ModTime: time.Unix(0, fileInfo.LastWriteTime.Nanoseconds()), + ChangeTime: time.Unix(0, fileInfo.ChangeTime.Nanoseconds()), + AccessTime: time.Unix(0, fileInfo.LastAccessTime.Nanoseconds()), + CreationTime: time.Unix(0, fileInfo.CreationTime.Nanoseconds()), + Winheaders: make(map[string]string), } hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes) - hdr.Winheaders[hdrAccessTime], hdr.AccessTime = win32TimeToTar(fileInfo.LastAccessTime) - hdr.Winheaders[hdrChangeTime], hdr.ChangeTime = win32TimeToTar(fileInfo.ChangeTime) - hdr.Winheaders[hdrCreateTime], _ = win32TimeToTar(fileInfo.CreationTime) - hdr.Winheaders[hdrWriteTime], hdr.ModTime = win32TimeToTar(fileInfo.LastWriteTime) if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { hdr.Mode |= c_ISDIR @@ -252,7 +226,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size return nil } -// Retrieves basic Win32 file information from a tar header, using the additional metadata written by +// FileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by // WriteTarFileFromBackupStream. func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) { name = hdr.Name @@ -260,10 +234,10 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win size = hdr.Size } fileInfo = &winio.FileBasicInfo{ - LastAccessTime: win32TimeFromTar(hdrAccessTime, hdr.Winheaders, hdr.AccessTime), - LastWriteTime: win32TimeFromTar(hdrWriteTime, hdr.Winheaders, hdr.ModTime), - ChangeTime: win32TimeFromTar(hdrChangeTime, hdr.Winheaders, hdr.ChangeTime), - CreationTime: win32TimeFromTar(hdrCreateTime, hdr.Winheaders, hdr.ModTime), + LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()), + LastWriteTime: syscall.NsecToFiletime(hdr.ModTime.UnixNano()), + ChangeTime: syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()), + CreationTime: syscall.NsecToFiletime(hdr.CreationTime.UnixNano()), } if attrStr, ok := hdr.Winheaders[hdrFileAttributes]; ok { attr, err := strconv.ParseUint(attrStr, 10, 32) @@ -279,7 +253,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win return } -// Writes a Win32 backup stream from the current tar file. Since this function may process multiple +// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple // tar file entries in order to collect all the alternate data streams for the file, it returns the next // tar file that was not processed, or io.EOF is there are no more. func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) { diff --git a/backuptar/tar_test.go b/backuptar/tar_test.go index f02b045..0e87a9e 100644 --- a/backuptar/tar_test.go +++ b/backuptar/tar_test.go @@ -4,6 +4,8 @@ import ( "bytes" "io/ioutil" "os" + "path/filepath" + "reflect" "testing" "github.com/Microsoft/go-winio" @@ -61,5 +63,22 @@ func TestRoundTrip(t *testing.T) { t.Fatal(err) } - ensurePresent(t, hdr.Winheaders, "fileattr", "sd", "accesstime", "changetime", "createtime", "writetime") + name, size, bi2, err := FileInfoFromHeader(hdr) + if err != nil { + t.Fatal(err) + } + + if name != filepath.ToSlash(f.Name()) { + t.Errorf("got name %s, expected %s", name, filepath.ToSlash(f.Name())) + } + + if size != fi.Size() { + t.Errorf("got size %d, expected %d", size, fi.Size()) + } + + if !reflect.DeepEqual(*bi, *bi2) { + t.Errorf("got %#v, expected %#v", *bi, *bi2) + } + + ensurePresent(t, hdr.Winheaders, "fileattr", "sd") }