From 42a2f45ea5db3e61161bcbe13478a148084b6a33 Mon Sep 17 00:00:00 2001 From: John Starks Date: Tue, 1 Aug 2017 17:24:15 -0700 Subject: [PATCH] Add support for EAs to Windows tars --- backup.go | 14 ++++- backuptar/tar.go | 92 +++++++++++++++++++++++++++- backuptar/tar_test.go | 2 +- ea.go | 137 ++++++++++++++++++++++++++++++++++++++++++ ea_test.go | 89 +++++++++++++++++++++++++++ 5 files changed, 328 insertions(+), 6 deletions(-) create mode 100644 ea.go create mode 100644 ea_test.go diff --git a/backup.go b/backup.go index 27d6ace..2be34af 100644 --- a/backup.go +++ b/backup.go @@ -68,10 +68,20 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader { return &BackupStreamReader{r, 0} } -// Next returns the next backup stream and prepares for calls to Write(). It skips the remainder of the current stream if +// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if // it was not completely read. func (r *BackupStreamReader) Next() (*BackupHeader, error) { if r.bytesLeft > 0 { + if s, ok := r.r.(io.Seeker); ok { + // Make sure Seek on io.SeekCurrent sometimes succeeds + // before trying the actual seek. + if _, err := s.Seek(0, io.SeekCurrent); err == nil { + if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil { + return nil, err + } + r.bytesLeft = 0 + } + } if _, err := io.Copy(ioutil.Discard, r); err != nil { return nil, err } @@ -220,7 +230,7 @@ type BackupFileWriter struct { ctx uintptr } -// NewBackupFileWrtier returns a new BackupFileWriter from a file handle. If includeSecurity is true, +// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true, // Write() will attempt to restore the security descriptor from the stream. func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter { w := &BackupFileWriter{f, includeSecurity, 0} diff --git a/backuptar/tar.go b/backuptar/tar.go index cfbcedf..53da908 100644 --- a/backuptar/tar.go +++ b/backuptar/tar.go @@ -36,6 +36,7 @@ const ( hdrSecurityDescriptor = "sd" hdrRawSecurityDescriptor = "rawsd" hdrMountPoint = "mountpoint" + hdrEaPrefix = "xattr." ) func writeZeroes(w io.Writer, count int64) error { @@ -118,6 +119,21 @@ func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *ta func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error { name = filepath.ToSlash(name) hdr := BasicInfoHeader(name, size, fileInfo) + + // If r can be seeked, then this function is two-pass: pass 1 collects the + // tar header data, and pass 2 copies the data stream. If r cannot be + // seeked, then some header data (in particular EAs) will be silently lost. + var ( + restartPos int64 + err error + ) + sr, readTwice := r.(io.Seeker) + if readTwice { + if restartPos, err = sr.Seek(0, io.SeekCurrent); err != nil { + readTwice = false + } + } + br := winio.NewBackupStreamReader(r) var dataHdr *winio.BackupHeader for dataHdr == nil { @@ -131,7 +147,9 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size switch bhdr.Id { case winio.BackupData: hdr.Mode |= c_ISREG - dataHdr = bhdr + if !readTwice { + dataHdr = bhdr + } case winio.BackupSecurity: sd, err := ioutil.ReadAll(br) if err != nil { @@ -151,18 +169,54 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size hdr.Winheaders[hdrMountPoint] = "1" } hdr.Linkname = rp.Target - case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData: + + case winio.BackupEaData: + eab, err := ioutil.ReadAll(br) + if err != nil { + return err + } + eas, err := winio.DecodeExtendedAttributes(eab) + if err != nil { + return err + } + for _, ea := range eas { + // Use base64 encoding for the binary value. Note that there + // is no way to encode the EA's flags, since their use doesn't + // make any sense for persisted EAs. + hdr.Winheaders[hdrEaPrefix+ea.Name] = base64.StdEncoding.EncodeToString(ea.Value) + } + + case winio.BackupAlternateData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData: // ignore these streams default: return fmt.Errorf("%s: unknown stream ID %d", name, bhdr.Id) } } - err := t.WriteHeader(hdr) + err = t.WriteHeader(hdr) if err != nil { return err } + if readTwice { + // Get back to the data stream. + if _, err = sr.Seek(restartPos, io.SeekStart); err != nil { + return err + } + for dataHdr == nil { + bhdr, err := br.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if bhdr.Id == winio.BackupData { + dataHdr = bhdr + } + } + } + if dataHdr != nil { // A data stream was found. Copy the data. if (dataHdr.Attributes & winio.StreamSparseAttributes) == 0 { @@ -293,6 +347,38 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) ( return nil, err } } + var eas []winio.ExtendedAttribute + for k, v := range hdr.Winheaders { + if !strings.HasPrefix(k, hdrEaPrefix) { + continue + } + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return nil, err + } + eas = append(eas, winio.ExtendedAttribute{ + Name: k[len(hdrEaPrefix):], + Value: data, + }) + } + if len(eas) != 0 { + eadata, err := winio.EncodeExtendedAttributes(eas) + if err != nil { + return nil, err + } + bhdr := winio.BackupHeader{ + Id: winio.BackupEaData, + Size: int64(len(eadata)), + } + err = bw.WriteHeader(&bhdr) + if err != nil { + return nil, err + } + _, err = bw.Write(eadata) + if err != nil { + return nil, err + } + } if hdr.Typeflag == tar.TypeSymlink { _, isMountPoint := hdr.Winheaders[hdrMountPoint] rp := winio.ReparsePoint{ diff --git a/backuptar/tar_test.go b/backuptar/tar_test.go index 0e87a9e..e04d47f 100644 --- a/backuptar/tar_test.go +++ b/backuptar/tar_test.go @@ -80,5 +80,5 @@ func TestRoundTrip(t *testing.T) { t.Errorf("got %#v, expected %#v", *bi, *bi2) } - ensurePresent(t, hdr.Winheaders, "fileattr", "sd") + ensurePresent(t, hdr.Winheaders, "fileattr", "rawsd") } diff --git a/ea.go b/ea.go new file mode 100644 index 0000000..b37e930 --- /dev/null +++ b/ea.go @@ -0,0 +1,137 @@ +package winio + +import ( + "bytes" + "encoding/binary" + "errors" +) + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} diff --git a/ea_test.go b/ea_test.go new file mode 100644 index 0000000..92d9d45 --- /dev/null +++ b/ea_test.go @@ -0,0 +1,89 @@ +package winio + +import ( + "io/ioutil" + "os" + "reflect" + "syscall" + "testing" + "unsafe" +) + +var ( + testEas = []ExtendedAttribute{ + {Name: "foo", Value: []byte("bar")}, + {Name: "fizz", Value: []byte("buzz")}, + } + + testEasEncoded = []byte{16, 0, 0, 0, 0, 3, 3, 0, 102, 111, 111, 0, 98, 97, 114, 0, 0, 0, 0, 0, 0, 4, 4, 0, 102, 105, 122, 122, 0, 98, 117, 122, 122, 0, 0, 0} + testEasNotPadded = testEasEncoded[0 : len(testEasEncoded)-3] + testEasTruncated = testEasEncoded[0:20] +) + +func Test_RoundTripEas(t *testing.T) { + b, err := EncodeExtendedAttributes(testEas) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEasEncoded, b) { + t.Fatalf("encoded mismatch %v %v", testEasEncoded, b) + } + eas, err := DecodeExtendedAttributes(b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEas, eas) { + t.Fatalf("mismatch %+v %+v", testEas, eas) + } +} + +func Test_EasDontNeedPaddingAtEnd(t *testing.T) { + eas, err := DecodeExtendedAttributes(testEasNotPadded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEas, eas) { + t.Fatalf("mismatch %+v %+v", testEas, eas) + } +} + +func Test_TruncatedEasFailCorrectly(t *testing.T) { + _, err := DecodeExtendedAttributes(testEasTruncated) + if err == nil { + t.Fatal("expected error") + } +} + +func Test_NilEasEncodeAndDecodeAsNil(t *testing.T) { + b, err := EncodeExtendedAttributes(nil) + if err != nil { + t.Fatal(err) + } + if len(b) != 0 { + t.Fatal("expected empty") + } + eas, err := DecodeExtendedAttributes(nil) + if err != nil { + t.Fatal(err) + } + if len(eas) != 0 { + t.Fatal("expected empty") + } +} + +// Test_SetFileEa makes sure that the test buffer is actually parsable by NtSetEaFile. +func Test_SetFileEa(t *testing.T) { + f, err := ioutil.TempFile("", "winio") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + defer f.Close() + ntdll := syscall.MustLoadDLL("ntdll.dll") + ntSetEaFile := ntdll.MustFindProc("NtSetEaFile") + var iosb [2]uintptr + r, _, _ := ntSetEaFile.Call(f.Fd(), uintptr(unsafe.Pointer(&iosb[0])), uintptr(unsafe.Pointer(&testEasEncoded[0])), uintptr(len(testEasEncoded))) + if r != 0 { + t.Fatalf("NtSetEaFile failed with %08x", r) + } +}