Merge pull request #14 from Microsoft/tar_changes

Backup-related improvements

archive/tar/common.go

@@ -44,22 +44,23 @@ const (
 // A Header represents a single header in a tar archive.
 // Some fields may not be populated.
 type Header struct {
-	Name       string    // name of header file entry
-	Mode       int64     // permission and mode bits
-	Uid        int       // user id of owner
-	Gid        int       // group id of owner
-	Size       int64     // length in bytes
-	ModTime    time.Time // modified time
-	Typeflag   byte      // type of header entry
-	Linkname   string    // target name of link
-	Uname      string    // user name of owner
-	Gname      string    // group name of owner
-	Devmajor   int64     // major number of character or block device
-	Devminor   int64     // minor number of character or block device
-	AccessTime time.Time // access time
-	ChangeTime time.Time // status change time
-	Xattrs     map[string]string
-	Winheaders map[string]string
+	Name         string    // name of header file entry
+	Mode         int64     // permission and mode bits
+	Uid          int       // user id of owner
+	Gid          int       // group id of owner
+	Size         int64     // length in bytes
+	ModTime      time.Time // modified time
+	Typeflag     byte      // type of header entry
+	Linkname     string    // target name of link
+	Uname        string    // user name of owner
+	Gname        string    // group name of owner
+	Devmajor     int64     // major number of character or block device
+	Devminor     int64     // minor number of character or block device
+	AccessTime   time.Time // access time
+	ChangeTime   time.Time // status change time
+	CreationTime time.Time // creation time
+	Xattrs       map[string]string
+	Winheaders   map[string]string
 }
 
 // File name constants from the tar spec.
@@ -180,21 +181,22 @@ const (
 
 // Keywords for the PAX Extended Header
 const (
-	paxAtime    = "atime"
-	paxCharset  = "charset"
-	paxComment  = "comment"
-	paxCtime    = "ctime" // please note that ctime is not a valid pax header.
-	paxGid      = "gid"
-	paxGname    = "gname"
-	paxLinkpath = "linkpath"
-	paxMtime    = "mtime"
-	paxPath     = "path"
-	paxSize     = "size"
-	paxUid      = "uid"
-	paxUname    = "uname"
-	paxXattr    = "SCHILY.xattr."
-	paxWindows  = "MSWINDOWS."
-	paxNone     = ""
+	paxAtime        = "atime"
+	paxCharset      = "charset"
+	paxComment      = "comment"
+	paxCtime        = "ctime" // please note that ctime is not a valid pax header.
+	paxCreationTime = "LIBARCHIVE.creationtime"
+	paxGid          = "gid"
+	paxGname        = "gname"
+	paxLinkpath     = "linkpath"
+	paxMtime        = "mtime"
+	paxPath         = "path"
+	paxSize         = "size"
+	paxUid          = "uid"
+	paxUname        = "uname"
+	paxXattr        = "SCHILY.xattr."
+	paxWindows      = "MSWINDOWS."
+	paxNone         = ""
 )
 
 // FileInfoHeader creates a partially-populated Header from fi.

archive/tar/reader.go

@@ -302,6 +302,12 @@ func mergePAX(hdr *Header, headers map[string]string) error {
 				return err
 			}
 			hdr.ChangeTime = t
+		case paxCreationTime:
+			t, err := parsePAXTime(v)
+			if err != nil {
+				return err
+			}
+			hdr.CreationTime = t
 		case paxSize:
 			size, err := strconv.ParseInt(v, 10, 0)
 			if err != nil {

archive/tar/testdata/small.txt

@@ -0,0 +1 @@
+Kilts

archive/tar/testdata/small2.txt

@@ -0,0 +1 @@
+Google.com

archive/tar/writer.go

@@ -47,7 +47,7 @@ type formatter struct {
 }
 
 // NewWriter creates a new Writer writing to w.
-func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
+func NewWriter(w io.Writer) *Writer { return &Writer{w: w, preferPax: true} }
 
 // Flush finishes writing the current file (optional).
 func (tw *Writer) Flush() error {
@@ -201,23 +201,29 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 		tw.usedBinary = true
 		f.formatNumeric(b, x)
 	}
+	var formatTime = func(b []byte, t time.Time, paxKeyword string) {
+		var unixTime int64
+		if !t.Before(minTime) && !t.After(maxTime) {
+			unixTime = t.Unix()
+		}
+		formatNumeric(b, unixTime, paxNone)
+
+		// Write a PAX header if the time didn't fit precisely.
+		if paxKeyword != "" && tw.preferPax && allowPax && (t.Nanosecond() != 0 || !t.Before(minTime) || !t.After(maxTime)) {
+			paxHeaders[paxKeyword] = formatPAXTime(t)
+		}
+	}
 
 	// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
 	pathHeaderBytes := s.next(fileNameSize)
 
 	formatString(pathHeaderBytes, hdr.Name, paxPath)
 
-	// Handle out of range ModTime carefully.
-	var modTime int64
-	if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
-		modTime = hdr.ModTime.Unix()
-	}
-
 	f.formatOctal(s.next(8), hdr.Mode)               // 100:108
 	formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
 	formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
 	formatNumeric(s.next(12), hdr.Size, paxSize)     // 124:136
-	formatNumeric(s.next(12), modTime, paxNone)      // 136:148 --- consider using pax for finer granularity
+	formatTime(s.next(12), hdr.ModTime, paxMtime)    // 136:148
 	s.next(8)                                        // chksum (148:156)
 	s.next(1)[0] = hdr.Typeflag                      // 156:157
 
@@ -265,6 +271,15 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	}
 
 	if allowPax {
+		if !hdr.AccessTime.IsZero() {
+			paxHeaders[paxAtime] = formatPAXTime(hdr.AccessTime)
+		}
+		if !hdr.ChangeTime.IsZero() {
+			paxHeaders[paxCtime] = formatPAXTime(hdr.ChangeTime)
+		}
+		if !hdr.CreationTime.IsZero() {
+			paxHeaders[paxCreationTime] = formatPAXTime(hdr.CreationTime)
+		}
 		for k, v := range hdr.Xattrs {
 			paxHeaders[paxXattr+k] = v
 		}
@@ -288,6 +303,16 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	return tw.err
 }
 
+func formatPAXTime(t time.Time) string {
+	sec := t.Unix()
+	usec := t.Nanosecond()
+	s := strconv.FormatInt(sec, 10)
+	if usec != 0 {
+		s = fmt.Sprintf("%s.%09d", s, usec)
+	}
+	return s
+}
+
 // splitUSTARPath splits a path according to USTAR prefix and suffix rules.
 // If the path is not splittable, then it will return ("", "", false).
 func splitUSTARPath(name string) (prefix, suffix string, ok bool) {

archive/tar/writer_test.go

@@ -720,3 +720,20 @@ func TestFormatNumeric(t *testing.T) {
 		}
 	}
 }
+
+func TestFormatPAXTime(t *testing.T) {
+	t1 := time.Date(2000, 1, 1, 11, 0, 0, 0, time.UTC)
+	t2 := time.Date(2000, 1, 1, 11, 0, 0, 100, time.UTC)
+	t3 := time.Date(1960, 1, 1, 11, 0, 0, 0, time.UTC)
+	t4 := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+	verify := func(time time.Time, s string) {
+		p := formatPAXTime(time)
+		if p != s {
+			t.Errorf("for %v, expected %s, got %s", time, s, p)
+		}
+	}
+	verify(t1, "946724400")
+	verify(t2, "946724400.000000100")
+	verify(t3, "-315579600")
+	verify(t4, "0")
+}

backup.go

@@ -26,10 +26,18 @@ const (
 	BackupReparseData
 	BackupSparseBlock
 	BackupTxfsData
+)
 
+const (
 	StreamSparseAttributes = uint32(8)
 )
 
+const (
+	WRITE_DAC              = 0x40000
+	WRITE_OWNER            = 0x80000
+	ACCESS_SYSTEM_SECURITY = 0x1000000
+)
+
 // BackupHeader represents a backup stream of a file.
 type BackupHeader struct {
 	Id         uint32 // The backup stream ID
@@ -239,3 +247,20 @@ func (w *BackupFileWriter) Close() error {
 	}
 	return nil
 }
+
+// OpenForBackup opens a file or directory, potentially skipping access checks if the backup
+// or restore privileges have been acquired.
+//
+// If the file opened was a directory, it cannot be used with Readdir().
+func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) {
+	winPath, err := syscall.UTF16FromString(path)
+	if err != nil {
+		return nil, err
+	}
+	h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
+	if err != nil {
+		err = &os.PathError{Op: "open", Path: path, Err: err}
+		return nil, err
+	}
+	return os.NewFile(uintptr(h), path), nil
+}

backuptar/tar.go

@@ -30,10 +30,6 @@ const (
 
 const (
 	hdrFileAttributes     = "fileattr"
-	hdrAccessTime         = "accesstime"
-	hdrChangeTime         = "changetime"
-	hdrCreateTime         = "createtime"
-	hdrWriteTime          = "writetime"
 	hdrSecurityDescriptor = "sd"
 	hdrMountPoint         = "mountpoint"
 )
@@ -82,21 +78,29 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
 	return nil
 }
 
-func win32TimeFromTar(key string, hdrs map[string]string, unixTime time.Time) syscall.Filetime {
-	if s, ok := hdrs[key]; ok {
-		n, err := strconv.ParseUint(s, 10, 64)
-		if err == nil {
-			return syscall.Filetime{uint32(n & 0xffffffff), uint32(n >> 32)}
-		}
+// BasicInfoHeader creates a tar header from basic file information.
+func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *tar.Header {
+	hdr := &tar.Header{
+		Name:         filepath.ToSlash(name),
+		Size:         size,
+		Typeflag:     tar.TypeReg,
+		ModTime:      time.Unix(0, fileInfo.LastWriteTime.Nanoseconds()),
+		ChangeTime:   time.Unix(0, fileInfo.ChangeTime.Nanoseconds()),
+		AccessTime:   time.Unix(0, fileInfo.LastAccessTime.Nanoseconds()),
+		CreationTime: time.Unix(0, fileInfo.CreationTime.Nanoseconds()),
+		Winheaders:   make(map[string]string),
 	}
-	return syscall.NsecToFiletime(unixTime.UnixNano())
+	hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
+
+	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
+		hdr.Mode |= c_ISDIR
+		hdr.Size = 0
+		hdr.Typeflag = tar.TypeDir
+	}
+	return hdr
 }
 
-func win32TimeToTar(ft syscall.Filetime) (string, time.Time) {
-	return fmt.Sprintf("%d", uint64(ft.LowDateTime)+(uint64(ft.HighDateTime)<<32)), time.Unix(0, ft.Nanoseconds())
-}
-
-// Writes a file to a tar writer using data from a Win32 backup stream.
+// WriteTarFileFromBackupStream writes a file to a tar writer using data from a Win32 backup stream.
 //
 // This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS.
 //
@@ -104,37 +108,12 @@ func win32TimeToTar(ft syscall.Filetime) (string, time.Time) {
 //
 // MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
 //
-// MSWINDOWS.accesstime: The last access time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.createtime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.changetime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
-// MSWINDOWS.writetime: The creation time, as a Filetime expressed as a 64-bit decimal value.
-//
 // MSWINDOWS.sd: The Win32 security descriptor, in SDDL (string) format
 //
 // MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
 func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
 	name = filepath.ToSlash(name)
-	hdr := &tar.Header{
-		Name:       name,
-		Size:       size,
-		Typeflag:   tar.TypeReg,
-		Winheaders: make(map[string]string),
-	}
-	hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
-	hdr.Winheaders[hdrAccessTime], hdr.AccessTime = win32TimeToTar(fileInfo.LastAccessTime)
-	hdr.Winheaders[hdrChangeTime], hdr.ChangeTime = win32TimeToTar(fileInfo.ChangeTime)
-	hdr.Winheaders[hdrCreateTime], _ = win32TimeToTar(fileInfo.CreationTime)
-	hdr.Winheaders[hdrWriteTime], hdr.ModTime = win32TimeToTar(fileInfo.LastWriteTime)
-
-	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
-		hdr.Mode |= c_ISDIR
-		hdr.Size = 0
-		hdr.Typeflag = tar.TypeDir
-	}
-
+	hdr := BasicInfoHeader(name, size, fileInfo)
 	br := winio.NewBackupStreamReader(r)
 	var dataHdr *winio.BackupHeader
 	for dataHdr == nil {
@@ -252,7 +231,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
 	return nil
 }
 
-// Retrieves basic Win32 file information from a tar header, using the additional metadata written by
+// FileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by
 // WriteTarFileFromBackupStream.
 func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
 	name = hdr.Name
@@ -260,10 +239,10 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		size = hdr.Size
 	}
 	fileInfo = &winio.FileBasicInfo{
-		LastAccessTime: win32TimeFromTar(hdrAccessTime, hdr.Winheaders, hdr.AccessTime),
-		LastWriteTime:  win32TimeFromTar(hdrWriteTime, hdr.Winheaders, hdr.ModTime),
-		ChangeTime:     win32TimeFromTar(hdrChangeTime, hdr.Winheaders, hdr.ChangeTime),
-		CreationTime:   win32TimeFromTar(hdrCreateTime, hdr.Winheaders, hdr.ModTime),
+		LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()),
+		LastWriteTime:  syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
+		ChangeTime:     syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()),
+		CreationTime:   syscall.NsecToFiletime(hdr.CreationTime.UnixNano()),
 	}
 	if attrStr, ok := hdr.Winheaders[hdrFileAttributes]; ok {
 		attr, err := strconv.ParseUint(attrStr, 10, 32)
@@ -279,7 +258,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 	return
 }
 
-// Writes a Win32 backup stream from the current tar file. Since this function may process multiple
+// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
 // tar file entries in order to collect all the alternate data streams for the file, it returns the next
 // tar file that was not processed, or io.EOF is there are no more.
 func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {

backuptar/tar_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/Microsoft/go-winio"
@@ -61,5 +63,22 @@ func TestRoundTrip(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	ensurePresent(t, hdr.Winheaders, "fileattr", "sd", "accesstime", "changetime", "createtime", "writetime")
+	name, size, bi2, err := FileInfoFromHeader(hdr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if name != filepath.ToSlash(f.Name()) {
+		t.Errorf("got name %s, expected %s", name, filepath.ToSlash(f.Name()))
+	}
+
+	if size != fi.Size() {
+		t.Errorf("got size %d, expected %d", size, fi.Size())
+	}
+
+	if !reflect.DeepEqual(*bi, *bi2) {
+		t.Errorf("got %#v, expected %#v", *bi, *bi2)
+	}
+
+	ensurePresent(t, hdr.Winheaders, "fileattr", "sd")
 }

fileinfo.go

@@ -9,22 +9,46 @@ import (
 //sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
 //sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
 
+const (
+	fileBasicInfo = 0
+	fileIDInfo    = 0x12
+)
+
+// FileBasicInfo contains file access time and file attributes information.
 type FileBasicInfo struct {
 	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
 	FileAttributes                                          uintptr // includes padding
 }
 
+// GetFileBasicInfo retrieves times and attributes for a file.
 func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
 	bi := &FileBasicInfo{}
-	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), 0, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
-		return nil, &os.PathError{"GetFileInformationByHandleEx", f.Name(), err}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
 	}
 	return bi, nil
 }
 
+// SetFileBasicInfo sets times and attributes for a file.
 func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
-	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), 0, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
-		return &os.PathError{"SetFileInformationByHandle", f.Name(), err}
+	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
 	}
 	return nil
 }
+
+// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
+// unique on a system.
+type FileIDInfo struct {
+	VolumeSerialNumber uint64
+	FileID             [16]byte
+}
+
+// GetFileID retrieves the unique (volume, file ID) pair for a file.
+func GetFileID(f *os.File) (*FileIDInfo, error) {
+	fileID := &FileIDInfo{}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	return fileID, nil
+}