Use existing PAX headers for time

This removes the MSWINDOWS.* time fields and uses PAX times directly. I
didn't previously realize that PAX times could be negative, which
eliminates the need to store Windows times in a different format.

This also uses the LIBARCHIVE.creationtime field for creation times to
match OS X behavior.

archive/tar/common.go

@@ -44,22 +44,23 @@ const (
 // A Header represents a single header in a tar archive.
 // Some fields may not be populated.
 type Header struct {
-	Name       string    // name of header file entry
-	Mode       int64     // permission and mode bits
-	Uid        int       // user id of owner
-	Gid        int       // group id of owner
-	Size       int64     // length in bytes
-	ModTime    time.Time // modified time
-	Typeflag   byte      // type of header entry
-	Linkname   string    // target name of link
-	Uname      string    // user name of owner
-	Gname      string    // group name of owner
-	Devmajor   int64     // major number of character or block device
-	Devminor   int64     // minor number of character or block device
-	AccessTime time.Time // access time
-	ChangeTime time.Time // status change time
-	Xattrs     map[string]string
-	Winheaders map[string]string
+	Name         string    // name of header file entry
+	Mode         int64     // permission and mode bits
+	Uid          int       // user id of owner
+	Gid          int       // group id of owner
+	Size         int64     // length in bytes
+	ModTime      time.Time // modified time
+	Typeflag     byte      // type of header entry
+	Linkname     string    // target name of link
+	Uname        string    // user name of owner
+	Gname        string    // group name of owner
+	Devmajor     int64     // major number of character or block device
+	Devminor     int64     // minor number of character or block device
+	AccessTime   time.Time // access time
+	ChangeTime   time.Time // status change time
+	CreationTime time.Time // creation time
+	Xattrs       map[string]string
+	Winheaders   map[string]string
 }
 
 // File name constants from the tar spec.
@@ -180,21 +181,22 @@ const (
 
 // Keywords for the PAX Extended Header
 const (
-	paxAtime    = "atime"
-	paxCharset  = "charset"
-	paxComment  = "comment"
-	paxCtime    = "ctime" // please note that ctime is not a valid pax header.
-	paxGid      = "gid"
-	paxGname    = "gname"
-	paxLinkpath = "linkpath"
-	paxMtime    = "mtime"
-	paxPath     = "path"
-	paxSize     = "size"
-	paxUid      = "uid"
-	paxUname    = "uname"
-	paxXattr    = "SCHILY.xattr."
-	paxWindows  = "MSWINDOWS."
-	paxNone     = ""
+	paxAtime        = "atime"
+	paxCharset      = "charset"
+	paxComment      = "comment"
+	paxCtime        = "ctime" // please note that ctime is not a valid pax header.
+	paxCreationTime = "LIBARCHIVE.creationtime"
+	paxGid          = "gid"
+	paxGname        = "gname"
+	paxLinkpath     = "linkpath"
+	paxMtime        = "mtime"
+	paxPath         = "path"
+	paxSize         = "size"
+	paxUid          = "uid"
+	paxUname        = "uname"
+	paxXattr        = "SCHILY.xattr."
+	paxWindows      = "MSWINDOWS."
+	paxNone         = ""
 )
 
 // FileInfoHeader creates a partially-populated Header from fi.

archive/tar/reader.go

@@ -302,6 +302,12 @@ func mergePAX(hdr *Header, headers map[string]string) error {
 				return err
 			}
 			hdr.ChangeTime = t
+		case paxCreationTime:
+			t, err := parsePAXTime(v)
+			if err != nil {
+				return err
+			}
+			hdr.CreationTime = t
 		case paxSize:
 			size, err := strconv.ParseInt(v, 10, 0)
 			if err != nil {

archive/tar/testdata/small.txt

@@ -0,0 +1 @@
+Kilts

archive/tar/testdata/small2.txt

@@ -0,0 +1 @@
+Google.com

archive/tar/writer.go

@@ -47,7 +47,7 @@ type formatter struct {
 }
 
 // NewWriter creates a new Writer writing to w.
-func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
+func NewWriter(w io.Writer) *Writer { return &Writer{w: w, preferPax: true} }
 
 // Flush finishes writing the current file (optional).
 func (tw *Writer) Flush() error {
@@ -201,23 +201,29 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 		tw.usedBinary = true
 		f.formatNumeric(b, x)
 	}
+	var formatTime = func(b []byte, t time.Time, paxKeyword string) {
+		var unixTime int64
+		if !t.Before(minTime) && !t.After(maxTime) {
+			unixTime = t.Unix()
+		}
+		formatNumeric(b, unixTime, paxNone)
+
+		// Write a PAX header if the time didn't fit precisely.
+		if paxKeyword != "" && tw.preferPax && allowPax && (t.Nanosecond() != 0 || !t.Before(minTime) || !t.After(maxTime)) {
+			paxHeaders[paxKeyword] = formatPAXTime(t)
+		}
+	}
 
 	// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
 	pathHeaderBytes := s.next(fileNameSize)
 
 	formatString(pathHeaderBytes, hdr.Name, paxPath)
 
-	// Handle out of range ModTime carefully.
-	var modTime int64
-	if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
-		modTime = hdr.ModTime.Unix()
-	}
-
 	f.formatOctal(s.next(8), hdr.Mode)               // 100:108
 	formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
 	formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
 	formatNumeric(s.next(12), hdr.Size, paxSize)     // 124:136
-	formatNumeric(s.next(12), modTime, paxNone)      // 136:148 --- consider using pax for finer granularity
+	formatTime(s.next(12), hdr.ModTime, paxMtime)    // 136:148
 	s.next(8)                                        // chksum (148:156)
 	s.next(1)[0] = hdr.Typeflag                      // 156:157
 
@@ -265,6 +271,15 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	}
 
 	if allowPax {
+		if !hdr.AccessTime.IsZero() {
+			paxHeaders[paxAtime] = formatPAXTime(hdr.AccessTime)
+		}
+		if !hdr.ChangeTime.IsZero() {
+			paxHeaders[paxCtime] = formatPAXTime(hdr.ChangeTime)
+		}
+		if !hdr.CreationTime.IsZero() {
+			paxHeaders[paxCreationTime] = formatPAXTime(hdr.CreationTime)
+		}
 		for k, v := range hdr.Xattrs {
 			paxHeaders[paxXattr+k] = v
 		}
@@ -288,6 +303,16 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	return tw.err
 }
 
+func formatPAXTime(t time.Time) string {
+	sec := t.Unix()
+	usec := t.Nanosecond()
+	s := strconv.FormatInt(sec, 10)
+	if usec != 0 {
+		s = fmt.Sprintf("%s.%09d", s, usec)
+	}
+	return s
+}
+
 // splitUSTARPath splits a path according to USTAR prefix and suffix rules.
 // If the path is not splittable, then it will return ("", "", false).
 func splitUSTARPath(name string) (prefix, suffix string, ok bool) {

archive/tar/writer_test.go

@@ -720,3 +720,20 @@ func TestFormatNumeric(t *testing.T) {
 		}
 	}
 }
+
+func TestFormatPAXTime(t *testing.T) {
+	t1 := time.Date(2000, 1, 1, 11, 0, 0, 0, time.UTC)
+	t2 := time.Date(2000, 1, 1, 11, 0, 0, 100, time.UTC)
+	t3 := time.Date(1960, 1, 1, 11, 0, 0, 0, time.UTC)
+	t4 := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+	verify := func(time time.Time, s string) {
+		p := formatPAXTime(time)
+		if p != s {
+			t.Errorf("for %v, expected %s, got %s", time, s, p)
+		}
+	}
+	verify(t1, "946724400")
+	verify(t2, "946724400.000000100")
+	verify(t3, "-315579600")
+	verify(t4, "0")
+}

backuptar/tar.go

@@ -30,10 +30,6 @@ const (
 
 const (
 	hdrFileAttributes     = "fileattr"
-	hdrAccessTime         = "accesstime"
-	hdrChangeTime         = "changetime"
-	hdrCreateTime         = "createtime"
-	hdrWriteTime          = "writetime"
 	hdrSecurityDescriptor = "sd"
 	hdrMountPoint         = "mountpoint"
 )
@@ -82,21 +78,7 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
 	return nil
 }
 
-func win32TimeFromTar(key string, hdrs map[string]string, unixTime time.Time) syscall.Filetime {
-	if s, ok := hdrs[key]; ok {
-		n, err := strconv.ParseUint(s, 10, 64)
-		if err == nil {
-			return syscall.Filetime{uint32(n & 0xffffffff), uint32(n >> 32)}
-		}
-	}
-	return syscall.NsecToFiletime(unixTime.UnixNano())
-}
-
-func win32TimeToTar(ft syscall.Filetime) (string, time.Time) {
-	return fmt.Sprintf("%d", uint64(ft.LowDateTime)+(uint64(ft.HighDateTime)<<32)), time.Unix(0, ft.Nanoseconds())
-}
-
-// Writes a file to a tar writer using data from a Win32 backup stream.
+// WriteTarFileFromBackupStream writes a file to a tar writer using data from a Win32 backup stream.
 //
 // This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS.
 //
@@ -104,30 +86,22 @@ func win32TimeToTar(ft syscall.Filetime) (string, time.Time) {
 //
 // MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
 //
-// MSWINDOWS.accesstime: The last access time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.createtime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.changetime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.writetime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
 // MSWINDOWS.sd: The Win32 security descriptor, in SDDL (string) format
 //
 // MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
 func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
 	name = filepath.ToSlash(name)
 	hdr := &tar.Header{
-		Name:       name,
-		Size:       size,
-		Typeflag:   tar.TypeReg,
-		Winheaders: make(map[string]string),
+		Name:         name,
+		Size:         size,
+		Typeflag:     tar.TypeReg,
+		ModTime:      time.Unix(0, fileInfo.LastWriteTime.Nanoseconds()),
+		ChangeTime:   time.Unix(0, fileInfo.ChangeTime.Nanoseconds()),
+		AccessTime:   time.Unix(0, fileInfo.LastAccessTime.Nanoseconds()),
+		CreationTime: time.Unix(0, fileInfo.CreationTime.Nanoseconds()),
+		Winheaders:   make(map[string]string),
 	}
 	hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
-	hdr.Winheaders[hdrAccessTime], hdr.AccessTime = win32TimeToTar(fileInfo.LastAccessTime)
-	hdr.Winheaders[hdrChangeTime], hdr.ChangeTime = win32TimeToTar(fileInfo.ChangeTime)
-	hdr.Winheaders[hdrCreateTime], _ = win32TimeToTar(fileInfo.CreationTime)
-	hdr.Winheaders[hdrWriteTime], hdr.ModTime = win32TimeToTar(fileInfo.LastWriteTime)
 
 	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
 		hdr.Mode |= c_ISDIR
@@ -252,7 +226,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
 	return nil
 }
 
-// Retrieves basic Win32 file information from a tar header, using the additional metadata written by
+// FileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by
 // WriteTarFileFromBackupStream.
 func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
 	name = hdr.Name
@@ -260,10 +234,10 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		size = hdr.Size
 	}
 	fileInfo = &winio.FileBasicInfo{
-		LastAccessTime: win32TimeFromTar(hdrAccessTime, hdr.Winheaders, hdr.AccessTime),
-		LastWriteTime:  win32TimeFromTar(hdrWriteTime, hdr.Winheaders, hdr.ModTime),
-		ChangeTime:     win32TimeFromTar(hdrChangeTime, hdr.Winheaders, hdr.ChangeTime),
-		CreationTime:   win32TimeFromTar(hdrCreateTime, hdr.Winheaders, hdr.ModTime),
+		LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()),
+		LastWriteTime:  syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
+		ChangeTime:     syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()),
+		CreationTime:   syscall.NsecToFiletime(hdr.CreationTime.UnixNano()),
 	}
 	if attrStr, ok := hdr.Winheaders[hdrFileAttributes]; ok {
 		attr, err := strconv.ParseUint(attrStr, 10, 32)
@@ -279,7 +253,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 	return
 }
 
-// Writes a Win32 backup stream from the current tar file. Since this function may process multiple
+// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
 // tar file entries in order to collect all the alternate data streams for the file, it returns the next
 // tar file that was not processed, or io.EOF is there are no more.
 func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {

backuptar/tar_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/Microsoft/go-winio"
@@ -61,5 +63,22 @@ func TestRoundTrip(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	ensurePresent(t, hdr.Winheaders, "fileattr", "sd", "accesstime", "changetime", "createtime", "writetime")
+	name, size, bi2, err := FileInfoFromHeader(hdr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if name != filepath.ToSlash(f.Name()) {
+		t.Errorf("got name %s, expected %s", name, filepath.ToSlash(f.Name()))
+	}
+
+	if size != fi.Size() {
+		t.Errorf("got size %d, expected %d", size, fi.Size())
+	}
+
+	if !reflect.DeepEqual(*bi, *bi2) {
+		t.Errorf("got %#v, expected %#v", *bi, *bi2)
+	}
+
+	ensurePresent(t, hdr.Winheaders, "fileattr", "sd")
 }