diff --git a/README.md b/README.md index 7244e33..5fb3669 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,10 @@ See the [developer guide](https://github.com/rwinkhart/libmutton/blob/main/wiki/ # Roadmap #### Release v0.5.0 -- [ ] Clipboard refactor +- Clipboard refactor +- Password aging support #### Release v0.6.0 -- [ ] Password aging support -#### Release v0.7.0 -- [ ] Implement "netpin" (quick-unlock) +- Implement "netpin" (quick-unlock) #### Release v1.0.0 - [ ] Create packaging scripts (libmuttonserver) - [ ] Stable source PKGBUILD diff --git a/age/age.go b/age/age.go new file mode 100644 index 0000000..b218e30 --- /dev/null +++ b/age/age.go @@ -0,0 +1,91 @@ +package age + +import ( + "crypto/rand" + "errors" + "math/big" + "os" + "strings" + "time" + + "github.com/rwinkhart/go-boilerplate/back" + "github.com/rwinkhart/libmutton/crypt" + "github.com/rwinkhart/libmutton/global" + "github.com/rwinkhart/libmutton/synccommon" +) + +// AgeEntry creates updates the age file for a vanity path. +func AgeEntry(vanityPath string, timestamp int64) error { + ageFilePath := global.AgeDir + global.PathSeparator + strings.ReplaceAll(vanityPath, "/", global.FSPath) + f, err := os.OpenFile(ageFilePath, os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return errors.New("unable to create age file for " + vanityPath + ": " + err.Error()) + } + _ = f.Close() // error ignored; if the file could be created, it can probably be closed + err = os.Chtimes(ageFilePath, time.Now(), time.Unix(timestamp, 0)) + if err != nil { + return errors.New("unable to set timestamp on age file for " + vanityPath + ": " + err.Error()) + } + return nil +} + +// AgeAllPasswordEntries adds aging data for all un-aged entries containing passwords. +// Each entry is aged with a random timestamp from within the last year to prevent +// all entries having their passwords expire at the same time. +func AgeAllPasswordEntries(forceReage bool) error { + allVanityPaths, _, err := synccommon.WalkEntryDir() + if err != nil { + return errors.New("unable to walk entry directory: " + err.Error()) + } + + for _, vanityPath := range allVanityPaths { + // ensure entry is not already aged (unless forcing re-aging) + if !forceReage { + // ignore error; we only care if we can access the path or not + isAccessible, _ := back.TargetIsFile(global.AgeDir+global.PathSeparator+strings.ReplaceAll(vanityPath, "/", global.FSPath), true) + if isAccessible { + // entry already aged, skip it + continue + } + } + + decSlice, err := crypt.DecryptFileToSlice(global.EntryRoot + vanityPath) + if err != nil { + return err + } + if decSlice != nil && decSlice[0] != "" { + // calculate random UNIX timestamp from within the last 365 days + offsetInt, _ := rand.Int(rand.Reader, big.NewInt(31557600)) + randomOffset := time.Duration(offsetInt.Int64()) * time.Second + err = AgeEntry(vanityPath, time.Now().Add(-randomOffset).Unix()) + if err != nil { + return err + } + } + } + + return nil +} + +// TranslateAgeTimestamp returns a uint8 value indicating +// the interpreted age status of an aging timestamp associated +// with an entry. +// Magic number legend: +// 0 -> no age, 1 -> fresh, 2 -> expiring soon (within a month), 3 -> expired +func TranslateAgeTimestamp(timestamp int64) uint8 { + if timestamp == 0 { + return 0 + } + + daysOld := time.Since(time.Unix(timestamp, 0)).Hours() / 24 + if daysOld >= 365 { + return 3 // expired + } else if daysOld >= 335 { + return 2 // expiring soon + } else { + return 1 // fresh + } +} + +// TODO +// 1. Add sync support for the aging directory diff --git a/clip/arguments.go b/clip/arguments.go index 2792e0b..c965389 100644 --- a/clip/arguments.go +++ b/clip/arguments.go @@ -13,15 +13,15 @@ import ( // CopyShortcut, given a path, decrypts an // entry and copies a field to the clipboard. -func CopyShortcut(targetLocation string, field int) error { - // ensure targetLocation exists and is a file - _, err := back.TargetIsFile(targetLocation, true) +func CopyShortcut(realPath string, field int) error { + // ensure realPath exists and is a file + _, err := back.TargetIsFile(realPath, true) if err != nil { return err } // decrypt entry - decSlice, err := crypt.DecryptFileToSlice(targetLocation) + decSlice, err := crypt.DecryptFileToSlice(realPath) if err != nil { return errors.New("unable to decrypt entry: " + err.Error()) } diff --git a/core/edit.go b/core/edit.go index e92866b..acd6d7c 100644 --- a/core/edit.go +++ b/core/edit.go @@ -10,15 +10,15 @@ import ( // GetOldEntryData decrypts and returns old entry data (with all required lines present). // This is a wrapper around the DecryptFileToSlice function that ensures all required lines are present in the returned slice. // This makes it ideal for editing entries, as it guarantees at least a baseline slice length. -func GetOldEntryData(targetLocation string, field int) ([]string, error) { - // ensure targetLocation exists and is a file - _, err := back.TargetIsFile(targetLocation, true) +func GetOldEntryData(realPath string, field int) ([]string, error) { + // ensure realPath exists and is a file + _, err := back.TargetIsFile(realPath, true) if err != nil { return nil, err } // read old entry data - decryptedEntry, err := crypt.DecryptFileToSlice(targetLocation) + decryptedEntry, err := crypt.DecryptFileToSlice(realPath) if err != nil { return nil, errors.New("unable to decrypt entry: " + err.Error()) } diff --git a/core/init.go b/core/init.go index 6ca756a..ecb8019 100644 --- a/core/init.go +++ b/core/init.go @@ -74,16 +74,17 @@ func LibmuttonInit(inputCB func(prompt string) string, clientSpecificIniData [][ {"LIBMUTTON", "sshKey", sshKeyPath}, {"LIBMUTTON", "sshKeyProtected", strconv.FormatBool(sshKeyProtected)}, {"LIBMUTTON", "sshEntryRoot", "null"}, + {"LIBMUTTON", "sshAgeDir", "null"}, {"LIBMUTTON", "sshIsWindows", "false"}}...), nil, false) if err != nil { return errors.New("unable to write config file: " + err.Error()) } // generate and register device ID - sshEntryRoot, sshIsWindows, err := synccycles.DeviceIDGen(oldDeviceID, "") + sshEntryRoot, sshAgeDir, sshIsWindows, err := synccycles.DeviceIDGen(oldDeviceID, "") if err != nil { return errors.New("unable to generate device ID: " + err.Error()) } - err = cfg.WriteConfig([][3]string{{"LIBMUTTON", "sshEntryRoot", sshEntryRoot}, {"LIBMUTTON", "sshIsWindows", sshIsWindows}}, nil, true) + err = cfg.WriteConfig([][3]string{{"LIBMUTTON", "sshEntryRoot", sshEntryRoot}, {"LIBMUTTON", "sshAgeDir", sshAgeDir}, {"LIBMUTTON", "sshIsWindows", sshIsWindows}}, nil, true) if err != nil { return errors.New("unable to write config file: " + err.Error()) } diff --git a/core/util.go b/core/util.go index 981a1ba..9e73f5f 100644 --- a/core/util.go +++ b/core/util.go @@ -9,18 +9,38 @@ import ( "github.com/pquerna/otp" "github.com/pquerna/otp/totp" "github.com/rwinkhart/go-boilerplate/back" + "github.com/rwinkhart/libmutton/age" "github.com/rwinkhart/libmutton/crypt" "github.com/rwinkhart/libmutton/global" + "github.com/rwinkhart/libmutton/syncclient" "github.com/rwinkhart/libmutton/synccommon" "github.com/rwinkhart/rcw/wrappers" ) -// WriteEntry writes entryData to an encrypted file at targetLocation. -func WriteEntry(targetLocation string, decBytes []byte) error { - err := os.WriteFile(targetLocation, crypt.EncryptBytes(decBytes), 0600) +// WriteEntry writes entryData to an encrypted file at realPath. +// If the entry contains an updated password, an aging file is also created. +func WriteEntry(realPath string, decSlice []string, passwordIsNew bool) error { + err := os.WriteFile(realPath, crypt.EncryptBytes([]byte(strings.Join(decSlice, "\n"))), 0600) if err != nil { return errors.New("unable to write to file: " + err.Error()) } + + if decSlice != nil { + if passwordIsNew { // update aging data when password changes + if decSlice[0] != "" { // if the password change was NOT a removal, update the aging file + err = age.AgeEntry(global.GetVanityPath(realPath), time.Now().Unix()) + if err != nil { + return errors.New("unable to update aging data: " + err.Error()) + } + } else { // if the password change was a removal, remove the associated aging file + err = syncclient.ShearRemoteFromClient(global.GetVanityPath(realPath), true) + if err != nil { + return errors.New("unable to remove aging data: " + err.Error()) + } + } + } + } + return nil } @@ -60,11 +80,11 @@ func EntryRefresh(oldRCWPassword, newRCWPassword []byte, removeOldDir bool) erro } // decrypt, optimize, and re-encrypt each entry - for _, entryName := range entries { - targetLocation := global.TargetLocationFormat(entryName) - encBytes, err := os.ReadFile(targetLocation) + for _, vanityPath := range entries { + realPath := global.GetRealPath(vanityPath) + encBytes, err := os.ReadFile(realPath) if err != nil { - return errors.New("unable to open \"" + targetLocation + "\" for decryption: " + err.Error()) + return errors.New("unable to open \"" + realPath + "\" for decryption: " + err.Error()) } decBytes, err := wrappers.Decrypt(encBytes, oldRCWPassword) decryptedEntry := strings.Split(string(decBytes), "\n") @@ -98,7 +118,7 @@ func EntryRefresh(oldRCWPassword, newRCWPassword []byte, removeOldDir bool) erro encBytes = wrappers.Encrypt([]byte(strings.Join(decryptedEntry, "\n")), newRCWPassword) // write the entry to the new directory - err = os.WriteFile(global.EntryRoot+"-new"+strings.ReplaceAll(entryName, "/", global.PathSeparator), encBytes, 0600) + err = os.WriteFile(global.EntryRoot+"-new"+strings.ReplaceAll(vanityPath, "/", global.PathSeparator), encBytes, 0600) if err != nil { return errors.New("unable to write to file: " + err.Error()) } @@ -155,16 +175,16 @@ func ClampTrailingWhitespace(note []string) { } // EntryAddPrecheck ensures the directory meant to contain a new -// entry exists and that the target entry location is not already used. -// Returns: statusCode (0 = success, 1 = target location already exists, 2 = containing directory is invalid). -func EntryAddPrecheck(targetLocation string) (uint8, error) { - // ensure target location does not already exist - isAccessible, _ := back.TargetIsFile(targetLocation, false) // error is ignored because dir/file status is irrelevant +// entry exists and that realPath is not already used. +// Returns: statusCode (0 = success, 1 = realPath already exists, 2 = containing directory is invalid). +func EntryAddPrecheck(realPath string) (uint8, error) { + // ensure realPath does not already exist + isAccessible, _ := back.TargetIsFile(realPath, false) // error is ignored because dir/file status is irrelevant if isAccessible { return 1, errors.New("target location already exists") } // ensure target containing directory exists and is not a file - containingDir := targetLocation[:strings.LastIndex(targetLocation, global.PathSeparator)] + containingDir := realPath[:strings.LastIndex(realPath, global.PathSeparator)] _, err := back.TargetIsFile(containingDir, false) if err != nil { return 2, errors.New("\"" + containingDir + "\" is not a valid containing directory: " + err.Error()) diff --git a/crypt/rcw.go b/crypt/rcw.go index 34d3397..8f02cd7 100644 --- a/crypt/rcw.go +++ b/crypt/rcw.go @@ -26,11 +26,11 @@ func RCWDArgument() { } // DecryptFileToSlice decrypts an RCW wrapped file and returns the contents as a slice of (trimmed) strings. -func DecryptFileToSlice(targetLocation string) ([]string, error) { +func DecryptFileToSlice(realPath string) ([]string, error) { // read encrypted file - encBytes, err := os.ReadFile(targetLocation) + encBytes, err := os.ReadFile(realPath) if err != nil { - return nil, errors.New("unable to open \"" + targetLocation + "\" for decryption: " + err.Error()) + return nil, errors.New("unable to open \"" + realPath + "\" for decryption: " + err.Error()) } // decrypt data using RCW daemon @@ -43,7 +43,7 @@ func DecryptFileToSlice(targetLocation string) ([]string, error) { // directly to avoid waiting for socket file creation decBytes, err := wrappers.Decrypt(encBytes, password) if err != nil { - return nil, errors.New("unable to decrypt \"" + targetLocation + "\": " + err.Error()) + return nil, errors.New("unable to decrypt \"" + realPath + "\": " + err.Error()) } return strings.Split(string(decBytes), "\n"), nil } diff --git a/global/1globals.go b/global/1globals.go index cf8c631..f5a2da9 100644 --- a/global/1globals.go +++ b/global/1globals.go @@ -4,6 +4,7 @@ type ByteInputFetcher func(prompt string) []byte var ( GetPassword ByteInputFetcher // Clients should set this to a function that fetches hidden input from the user + rootLength = len(EntryRoot) // length of global.EntryRoot string ) const ( diff --git a/global/2globals_UNIX.go b/global/2globals_UNIX.go index 18eb431..842435c 100644 --- a/global/2globals_UNIX.go +++ b/global/2globals_UNIX.go @@ -8,6 +8,7 @@ var ( EntryRoot = back.Home + "/.local/share/libmutton" // Path to libmutton entry directory ConfigDir = back.Home + "/.config/libmutton" // Path to libmutton configuration directory ConfigPath = ConfigDir + "/libmutton.ini" // Path to libmutton configuration file + AgeDir = ConfigDir + "/aging" // Path to libmutton password aging database ) const ( diff --git a/global/2globals_WIN.go b/global/2globals_WIN.go index de746b7..70b7233 100644 --- a/global/2globals_WIN.go +++ b/global/2globals_WIN.go @@ -8,6 +8,7 @@ var ( EntryRoot = back.Home + "\\AppData\\Local\\libmutton\\entries" // Path to libmutton entry directory ConfigDir = back.Home + "\\AppData\\Local\\libmutton\\config" // Path to libmutton configuration directory ConfigPath = ConfigDir + "\\libmutton.ini" // Path to libmutton configuration file + AgeDir = ConfigDir + "\\aging" // Path to libmutton password aging database ) const ( diff --git a/global/getPath.go b/global/getPath.go new file mode 100644 index 0000000..dc4e9eb --- /dev/null +++ b/global/getPath.go @@ -0,0 +1,8 @@ +package global + +import "strings" + +// GetRealAgePath returns the full path to an age file (given the vanity path) +func GetRealAgePath(vanityPath string) string { + return AgeDir + PathSeparator + strings.ReplaceAll(vanityPath, "/", FSPath) +} diff --git a/global/getPath_UNIX.go b/global/getPath_UNIX.go new file mode 100644 index 0000000..1342986 --- /dev/null +++ b/global/getPath_UNIX.go @@ -0,0 +1,13 @@ +//go:build !windows + +package global + +// GetRealPath returns the full path to an entry (given the vanity path). +func GetRealPath(vanityPath string) string { + return EntryRoot + vanityPath +} + +// GetVanityPath returns a vanityPath given a realPath +func GetVanityPath(realPath string) string { + return realPath[rootLength:] +} diff --git a/global/getPath_WIN.go b/global/getPath_WIN.go new file mode 100644 index 0000000..d66fcf9 --- /dev/null +++ b/global/getPath_WIN.go @@ -0,0 +1,17 @@ +//go:build windows + +package global + +import ( + "strings" +) + +// GetRealPath returns the full path to an entry (given the vanity path). +func GetRealPath(vanityPath string) string { + return EntryRoot + strings.ReplaceAll(vanityPath, "/", PathSeparator) +} + +// GetVanityPath returns a vanityPath given a realPath +func GetVanityPath(realPath string) string { + return strings.ReplaceAll(realPath[rootLength:], "\\", "/") +} diff --git a/global/init.go b/global/init.go index 624cb98..61b7116 100644 --- a/global/init.go +++ b/global/init.go @@ -39,5 +39,11 @@ func DirInit(preserveOldConfigDir bool) (string, error) { return "", errors.New("unable to create \"" + ConfigDir + "\": " + err.Error()) } + // create password aging directory + err = os.MkdirAll(AgeDir, 0700) + if err != nil { + return "", errors.New("unable to create \"" + AgeDir + "\": " + err.Error()) + } + return oldDeviceID, nil } diff --git a/global/targetLocationFormat_UNIX.go b/global/targetLocationFormat_UNIX.go deleted file mode 100644 index c04214d..0000000 --- a/global/targetLocationFormat_UNIX.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !windows - -package global - -// TargetLocationFormat returns the full location of an entry (given the name) formatted for the current platform. -func TargetLocationFormat(targetLocationIncomplete string) string { - return EntryRoot + targetLocationIncomplete -} diff --git a/global/targetLocationFormat_WIN.go b/global/targetLocationFormat_WIN.go deleted file mode 100644 index 1d5e4d0..0000000 --- a/global/targetLocationFormat_WIN.go +++ /dev/null @@ -1,12 +0,0 @@ -//go:build windows - -package global - -import ( - "strings" -) - -// TargetLocationFormat returns the full location of an entry (given the name) formatted for the current platform. -func TargetLocationFormat(targetLocationIncomplete string) string { - return EntryRoot + strings.ReplaceAll(targetLocationIncomplete, "/", PathSeparator) -} diff --git a/libmuttonserver.go b/libmuttonserver.go index b65c816..13e97a2 100644 --- a/libmuttonserver.go +++ b/libmuttonserver.go @@ -41,25 +41,30 @@ func main() { case "rename": // move an entry to a new location before using fallthrough to add its previous iteration to the deletions directory // stdin[0] is evaluated after fallthrough - // stdin[1] is expected to be the OLD incomplete target location with FSPath representing path separators - Always pass in UNIX format - // stdin[2] is expected to be the NEW incomplete target location with FSPath representing path separators - Always pass in UNIX format + // stdin[1] is expected to be the OLD vanityPath with FSPath representing path separators - Always pass in UNIX format + // stdin[2] is expected to be the NEW vanityPath with FSPath representing path separators - Always pass in UNIX format _ = synccommon.RenameLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), strings.ReplaceAll(stdin[2], global.FSPath, "/")) fallthrough // fallthrough to add the old entry to the deletions directory case "shear": // shear an entry from the server and add it to the deletions directory // stdin[0] is expected to be the device ID - // stdin[1] is expected to be the incomplete target location with FSPath representing path separators - Always pass in UNIX format - _, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0]) + // stdin[1] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format + _, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], false) + case "shear-age": + // shear ONLY the aging file associated with an entry from the server and add it to the deletions directory + // stdin[0] is expected to be the device ID + // stdin[1] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format + _, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], true) case "addfolder": // add a new folder to the server - // stdin[0] is expected to be the incomplete target location with FSPath representing path separators - Always pass in UNIX format + // stdin[0] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format _ = synccommon.AddFolderLocal(strings.ReplaceAll(stdin[0], global.FSPath, "/")) case "register": // register a new device ID // stdin[0] is expected to be the device ID // stdin[1] is expected to be the old device ID (for removal) - fileToClose, _ := os.OpenFile(global.ConfigDir+global.PathSeparator+"devices"+global.PathSeparator+stdin[0], os.O_CREATE|os.O_WRONLY, 0600) // errors ignored; failure unlikely to occur if init was successful; "register" is not a user-facing argument and thus the error would not be visible - _ = fileToClose.Close() + f, _ := os.OpenFile(global.ConfigDir+global.PathSeparator+"devices"+global.PathSeparator+stdin[0], os.O_CREATE|os.O_WRONLY, 0600) // errors ignored; failure unlikely to occur if init was successful; "register" is not a user-facing argument and thus the error would not be visible + _ = f.Close() if stdin[1] != global.FSMisc { // FSMisc is used to indicate that no device ID is being replaced // remove the old device ID file _ = os.RemoveAll(global.ConfigDir + global.PathSeparator + "devices" + global.PathSeparator + stdin[1]) @@ -67,14 +72,14 @@ func main() { deletionsDirRoot := global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator deletionsList, _ := os.ReadDir(deletionsDirRoot) for _, deletion := range deletionsList { - affectedIDTargetLocationIncomplete := strings.Split(deletion.Name(), global.FSSpace) - if affectedIDTargetLocationIncomplete[0] == stdin[1] { - _ = os.Rename(deletionsDirRoot+deletion.Name(), deletionsDirRoot+stdin[0]+global.FSSpace+affectedIDTargetLocationIncomplete[1]) + affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace) + if affectedIDVanityPath[0] == stdin[1] { + _ = os.Rename(deletionsDirRoot+deletion.Name(), deletionsDirRoot+stdin[0]+global.FSSpace+affectedIDVanityPath[1]) } } } - // print EntryRoot and bool indicating OS type to stdout for client to store in config - fmt.Print(global.EntryRoot + global.FSSpace + strconv.FormatBool(global.IsWindows)) + // print EntryRoot, AgeDir and bool indicating OS type to stdout for client to store in config + fmt.Print(global.EntryRoot + global.FSSpace + global.AgeDir + global.FSSpace + strconv.FormatBool(global.IsWindows)) case "init": // create the necessary directories for libmuttonserver to function _, err := global.DirInit(false) diff --git a/syncclient/client.go b/syncclient/client.go index 3a7f57a..6a231b6 100644 --- a/syncclient/client.go +++ b/syncclient/client.go @@ -23,19 +23,20 @@ import ( // offlineMode (whether the client is in offline mode). // sshIsWindows (whether the remote server is running Windows), // sshEntryRoot (the root directory for entries on the remote server), +// sshAgeDir (the directory housing age files on the remote server), // Only supports key-based authentication (passwords are supported for CLI-based implementations). -func GetSSHClient() (*ssh.Client, bool, bool, string, error) { +func GetSSHClient() (*ssh.Client, bool, bool, string, string, error) { // get SSH config info - sshUserConfig, err := cfg.ParseConfig([][2]string{{"LIBMUTTON", "offlineMode"}, {"LIBMUTTON", "sshUser"}, {"LIBMUTTON", "sshIP"}, {"LIBMUTTON", "sshPort"}, {"LIBMUTTON", "sshKey"}, {"LIBMUTTON", "sshKeyProtected"}, {"LIBMUTTON", "sshEntryRoot"}, {"LIBMUTTON", "sshIsWindows"}}) + sshUserConfig, err := cfg.ParseConfig([][2]string{{"LIBMUTTON", "offlineMode"}, {"LIBMUTTON", "sshUser"}, {"LIBMUTTON", "sshIP"}, {"LIBMUTTON", "sshPort"}, {"LIBMUTTON", "sshKey"}, {"LIBMUTTON", "sshKeyProtected"}, {"LIBMUTTON", "sshEntryRoot"}, {"LIBMUTTON", "sshAgeDir"}, {"LIBMUTTON", "sshIsWindows"}}) if len(sshUserConfig) == 1 { // offline mode is enabled - return nil, true, false, "", nil + return nil, true, false, "", "", nil } if err != nil { - return nil, false, false, "", errors.New("unable to parse SSH config: " + err.Error()) + return nil, false, false, "", "", errors.New("unable to parse SSH config: " + err.Error()) } - var user, ip, port, keyFile, keyFileProtected, entryRoot string + var user, ip, port, keyFile, keyFileProtected, entryRoot, ageDir string var isWindows bool for i, key := range sshUserConfig { switch i { @@ -52,9 +53,11 @@ func GetSSHClient() (*ssh.Client, bool, bool, string, error) { case 6: entryRoot = key case 7: + ageDir = key + case 8: isWindows, err = strconv.ParseBool(key) if err != nil { - return nil, false, false, "", errors.New("unable to parse server OS type: " + err.Error()) + return nil, false, false, "", "", errors.New("unable to parse server OS type: " + err.Error()) } } } @@ -62,7 +65,7 @@ func GetSSHClient() (*ssh.Client, bool, bool, string, error) { // read private key key, err := os.ReadFile(keyFile) if err != nil { - return nil, false, false, "", errors.New("unable to read private key: " + keyFile) + return nil, false, false, "", "", errors.New("unable to read private key: " + keyFile) } // parse private key @@ -73,14 +76,14 @@ func GetSSHClient() (*ssh.Client, bool, bool, string, error) { parsedKey, err = ssh.ParsePrivateKeyWithPassphrase(key, global.GetPassword("Enter password for your SSH keyfile:")) } if err != nil { - return nil, false, false, "", errors.New("unable to parse private key: " + keyFile) + return nil, false, false, "", "", errors.New("unable to parse private key: " + keyFile) } // read known hosts file var hostKeyCallback ssh.HostKeyCallback hostKeyCallback, err = knownhosts.New(back.Home + global.PathSeparator + ".ssh" + global.PathSeparator + "known_hosts") if err != nil { - return nil, false, false, "", errors.New("unable to read known hosts file: " + err.Error()) + return nil, false, false, "", "", errors.New("unable to read known hosts file: " + err.Error()) } // configure SSH client @@ -96,10 +99,10 @@ func GetSSHClient() (*ssh.Client, bool, bool, string, error) { // connect to SSH server sshClient, err := ssh.Dial("tcp", ip+":"+port, sshConfig) if err != nil { - return nil, false, false, "", errors.New("unable to connect to remote server: " + err.Error()) + return nil, false, false, "", "", errors.New("unable to connect to remote server: " + err.Error()) } - return sshClient, false, isWindows, entryRoot, nil + return sshClient, false, isWindows, entryRoot, ageDir, nil } // GetSSHOutput runs a command over SSH and returns the output as a string. @@ -127,48 +130,63 @@ func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) (string, error) { return outputString, nil } -// getRemoteDataFromClient returns a map of remote entries to their modification times, a list of remote folders, a list of queued deletions, and the current server&client times as UNIX timestamps. -func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, []string, []string, int64, int64, error) { +// getRemoteDataFromClient returns: +// a map of remote entries to their modification times, +// a list of remote age files to their timestamps, +// a list of remote folders, a list of queued deletions, +// and the current server&client times as UNIX timestamps. +func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[string]int64, []string, []string, int64, int64, error) { // get remote output over SSH deviceIDList, err := global.GenDeviceIDList() if err != nil { - return nil, nil, nil, 0, 0, err + return nil, nil, nil, nil, 0, 0, err } if len(deviceIDList) == 0 { - return nil, nil, nil, 0, 0, errors.New("no device ID found") + return nil, nil, nil, nil, 0, 0, errors.New("no device ID found") } clientTime := time.Now().Unix() // get client time now to avoid accuracy issues caused by unpredictable sync time output, err := GetSSHOutput(sshClient, "libmuttonserver fetch", (deviceIDList)[0].Name()) if err != nil { - return nil, nil, nil, 0, 0, errors.New("unable to run remote command: " + err.Error()) + return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command: " + err.Error()) } // split output into slice based on occurrences of FSSpace outputSlice := strings.Split(output, global.FSSpace) // parse output/re-form lists - if len(outputSlice) != 5 { // ensure information from server is complete - return nil, nil, nil, 0, 0, errors.New("unable to run remote command; server returned an unexpected response") + if len(outputSlice) != 7 { // ensure information from server is complete + return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command; server returned an unexpected response") } serverTime, err := strconv.ParseInt(outputSlice[0], 10, 64) if err != nil { - return nil, nil, nil, 0, 0, errors.New("unable to parse server time: " + err.Error()) + return nil, nil, nil, nil, 0, 0, errors.New("unable to parse server time: " + err.Error()) } entries := strings.Split(outputSlice[1], global.FSMisc)[1:] modsStrings := strings.Split(outputSlice[2], global.FSMisc)[1:] - folders := strings.Split(outputSlice[3], global.FSMisc)[1:] - deletions := strings.Split(outputSlice[4], global.FSMisc)[1:] + ageFiles := strings.Split(outputSlice[3], global.FSMisc)[1:] + ageFilesTimestampStrings := strings.Split(outputSlice[4], global.FSMisc)[1:] + folders := strings.Split(outputSlice[5], global.FSMisc)[1:] + deletions := strings.Split(outputSlice[6], global.FSMisc)[1:] - // convert the mod times to int64 + // convert the mod times+age timestamps to int64 var mods []int64 var mod int64 for _, modString := range modsStrings { mod, err = strconv.ParseInt(modString, 10, 64) if err != nil { - return nil, nil, nil, 0, 0, errors.New("unable to parse mod time: " + err.Error()) + return nil, nil, nil, nil, 0, 0, errors.New("unable to parse mod time: " + err.Error()) } mods = append(mods, mod) } + var timestamps []int64 + var timestamp int64 + for _, ageFilesTimestampString := range ageFilesTimestampStrings { + timestamp, err = strconv.ParseInt(ageFilesTimestampString, 10, 64) + if err != nil { + return nil, nil, nil, nil, 0, 0, errors.New("unable to parse age timestamp: " + err.Error()) + } + timestamps = append(timestamps, timestamp) + } // map remote entries to their modification times entryModMap := make(map[string]int64) @@ -176,7 +194,13 @@ func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, []string, entryModMap[entry] = mods[i] } - return entryModMap, folders, deletions, serverTime, clientTime, nil + // map remote age files to their timestamps + ageTimestampMap := make(map[string]int64) + for i, ageFile := range ageFiles { + ageTimestampMap[ageFile] = timestamps[i] + } + + return entryModMap, ageTimestampMap, folders, deletions, serverTime, clientTime, nil } // getLocalData returns a map of local entries to their modification times. @@ -200,17 +224,26 @@ func getLocalData() (map[string]int64, error) { return entryModMap, nil } -// targetLocationFormatSFTP formats the target location to match the remote server's entry directory and path separator. -func targetLocationFormatSFTP(targetName, serverEntryRoot string, serverIsWindows bool) string { +// getRealPathSFTP formats the vanityPath to match the remote server's entry/age file directory and path separator. +func getRealPathSFTP(vanityPath, serverEntryRoot string, serverIsWindows bool) string { if !serverIsWindows { - return serverEntryRoot + targetName + return serverEntryRoot + vanityPath } else { - return serverEntryRoot + strings.ReplaceAll(targetName, "/", "\\") + return serverEntryRoot + strings.ReplaceAll(vanityPath, "/", "\\") + } +} + +// getRealPathSFTP formats the vanityPath to match the remote server's entry/age file directory and path separator. +func getRealAgePathSFTP(vanityPath, serverAgeDir string, serverIsWindows bool) string { + if !serverIsWindows { + return serverAgeDir + "/" + strings.ReplaceAll(vanityPath, "/", global.FSPath) + } else { + return serverAgeDir + "\\" + strings.ReplaceAll(vanityPath, "/", global.FSPath) } } // sftpSync takes two slices of entries (one for downloads and one for uploads) and syncs them between the client and server using SFTP. -func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, downloadList, uploadList []string) error { +func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindows bool, downloadList, uploadList []string) error { // create an SFTP client from sshClient sftpClient, err := sftp.NewClient(sshClient) if err != nil { @@ -222,17 +255,28 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow // iterate over the download list var filesTransferred bool - for _, entryName := range downloadList { - filesTransferred = true // set a flag to indicate that files have been downloaded (used to determine whether to print a gap between download and upload messages) - - fmt.Println("Downloading " + back.AnsiGreen + entryName + back.AnsiReset) + for _, vanityPath := range downloadList { + // determine if remote file is an age file + var isAgeFile bool + if strings.HasPrefix(vanityPath, global.FSMisc) { + vanityPath = strings.TrimLeft(vanityPath, global.FSMisc) + isAgeFile = true + } else { + filesTransferred = true // set a flag to indicate that files have been downloaded (used to determine whether to print a gap between download and upload messages) + fmt.Println("Downloading " + back.AnsiGreen + vanityPath + back.AnsiReset) + } // store path to remote entry - remoteEntryFullPath := targetLocationFormatSFTP(entryName, sshEntryRoot, sshIsWindows) + var remoteFileRealPath string + if isAgeFile { + remoteFileRealPath = getRealAgePathSFTP(vanityPath, sshAgeDir, sshIsWindows) + } else { + remoteFileRealPath = getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows) + } // save modification time of remote file var fileInfo os.FileInfo - fileInfo, err = sftpClient.Stat(remoteEntryFullPath) + fileInfo, err = sftpClient.Stat(remoteFileRealPath) if err != nil { return errors.New("unable to get remote file info (mod time): " + err.Error()) } @@ -240,17 +284,22 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow // open remote file var remoteFile *sftp.File - remoteFile, err = sftpClient.Open(remoteEntryFullPath) + remoteFile, err = sftpClient.Open(remoteFileRealPath) if err != nil { return errors.New("unable to open remote file: " + err.Error()) } - // store path to local entry - localEntryFullPath := global.TargetLocationFormat(entryName) + // store path to local file + var localFileRealPath string + if isAgeFile { + localFileRealPath = global.GetRealAgePath(vanityPath) + } else { + localFileRealPath = global.GetRealPath(vanityPath) + } // create local file var localFile *os.File - localFile, err = os.OpenFile(localEntryFullPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + localFile, err = os.OpenFile(localFileRealPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if err != nil { return errors.New("unable to create local file: " + err.Error()) } @@ -266,7 +315,7 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow _ = localFile.Close() // set the modification time of the local file to match the value saved from the remote file (from before the download) - err = os.Chtimes(localEntryFullPath, time.Now(), modTime) + err = os.Chtimes(localFileRealPath, time.Now(), modTime) if err != nil { return errors.New("unable to set local file modification time: " + err.Error()) } @@ -278,17 +327,28 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow // iterate over the upload list filesTransferred = false - for _, entryName := range uploadList { - filesTransferred = true // set a flag to indicate that files have been uploaded (used to determine whether to print a gap between upload and sync complete messages) + for _, vanityPath := range uploadList { + // determine if local file is an age file + var isAgeFile bool + if strings.HasPrefix(vanityPath, global.FSMisc) { + vanityPath = strings.TrimLeft(vanityPath, global.FSMisc) + isAgeFile = true + } else { + filesTransferred = true // set a flag to indicate that files have been uploaded (used to determine whether to print a gap between upload and sync complete messages) + fmt.Println("Uploading " + back.AnsiBlue + vanityPath + back.AnsiReset) + } - fmt.Println("Uploading " + back.AnsiBlue + entryName + back.AnsiReset) - - // store path to local entry - localEntryFullPath := global.TargetLocationFormat(entryName) + // store path to local file + var localFileRealPath string + if isAgeFile { + localFileRealPath = global.GetRealAgePath(vanityPath) + } else { + localFileRealPath = global.GetRealPath(vanityPath) + } // save modification time of local file var fileInfo os.FileInfo - fileInfo, err = os.Stat(localEntryFullPath) + fileInfo, err = os.Stat(localFileRealPath) if err != nil { return errors.New("unable to get local file info (mod time): " + err.Error()) } @@ -296,17 +356,22 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow // open local file var localFile *os.File - localFile, err = os.Open(localEntryFullPath) + localFile, err = os.Open(localFileRealPath) if err != nil { return errors.New("unable to open local file: " + err.Error()) } // store path to remote entry - remoteEntryFullPath := targetLocationFormatSFTP(entryName, sshEntryRoot, sshIsWindows) + var remoteFileRealPath string + if isAgeFile { + remoteFileRealPath = getRealAgePathSFTP(vanityPath, sshAgeDir, sshIsWindows) + } else { + remoteFileRealPath = getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows) + } // create remote file var remoteFile *sftp.File - remoteFile, err = sftpClient.OpenFile(remoteEntryFullPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY) + remoteFile, err = sftpClient.OpenFile(remoteFileRealPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY) if err != nil { return errors.New("unable to create remote file: " + err.Error()) } @@ -322,13 +387,13 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow _ = remoteFile.Close() // set permissions on remote file - err = sftpClient.Chmod(remoteEntryFullPath, 0600) + err = sftpClient.Chmod(remoteFileRealPath, 0600) if err != nil { return errors.New("unable to set permissions on remote file: " + err.Error()) } // set the modification time of the remote file to match the value saved from the local file (from before the upload) - err = sftpClient.Chtimes(remoteEntryFullPath, time.Now(), modTime) + err = sftpClient.Chtimes(remoteFileRealPath, time.Now(), modTime) if err != nil { return errors.New("unable to set remote file modification time: " + err.Error()) } @@ -343,40 +408,59 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow // syncLists determines which entries need to be downloaded and uploaded for synchronizations and calls sftpSync with this information. // Using maps means that syncing will be done in an arbitrary order, but it is a worthy tradeoff for speed and simplicity. -func syncLists(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows, timeSynced, returnLists bool, localEntryModMap, remoteEntryModMap map[string]int64) ([3][]string, error) { +func syncLists(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindows, timeSynced, returnLists bool, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap map[string]int64) ([3][]string, error) { // initialize slices to store entries that need to be downloaded or uploaded var downloadList, uploadList []string // iterate over client entries - for entry, localModTime := range localEntryModMap { - // check if the entry is present in the server map - if remoteModTime, present := remoteEntryModMap[entry]; present { - // entry exists on both client and server, compare mod times - if remoteModTime > localModTime { - fmt.Println(back.AnsiGreen+entry+back.AnsiReset, "is newer on server, adding to download list") - downloadList = append(downloadList, entry) - } else if remoteModTime < localModTime { - fmt.Println(back.AnsiBlue+entry+back.AnsiReset, "is newer on client, adding to upload list") - uploadList = append(uploadList, entry) + localMapIter := func(localMap, remoteMap map[string]int64, forAging bool) { + for file, localTime := range localMap { + // check if the entry is present in the server map + if remoteTime, present := remoteMap[file]; present { + // entry exists on both client and server, compare mod times + if remoteTime > localTime { + if !forAging { + fmt.Println(back.AnsiGreen+file+back.AnsiReset, "is newer on server, adding to download list") + downloadList = append(downloadList, file) + } else { + downloadList = append(downloadList, global.FSMisc+file) + } + } else if remoteTime < localTime { + if !forAging { + fmt.Println(back.AnsiBlue+file+back.AnsiReset, "is newer on client, adding to upload list") + uploadList = append(uploadList, file) + } else { + uploadList = append(uploadList, global.FSMisc+file) + } + } + // remove entry from remoteMap (process of elimination) + delete(remoteMap, file) + } else { + if !forAging { + fmt.Println(back.AnsiBlue+file+back.AnsiReset, "does not exist on server, adding to upload list") + uploadList = append(uploadList, file) + } else { + uploadList = append(uploadList, global.FSMisc+file) + } } - // remove entry from remoteEntryModMap (process of elimination) - delete(remoteEntryModMap, entry) - } else { - fmt.Println(back.AnsiBlue+entry+back.AnsiReset, "does not exist on server, adding to upload list") - uploadList = append(uploadList, entry) } } + localMapIter(localEntryModMap, remoteEntryModMap, false) + localMapIter(localAgeTimestampMap, remoteAgeTimestampMap, true) - // iterate over remaining entries in remoteEntryModMap + // iterate over remaining entries in remote maps for entry := range remoteEntryModMap { fmt.Println(back.AnsiGreen+entry+back.AnsiReset, "does not exist on client, adding to download list") downloadList = append(downloadList, entry) } + for ageFile := range remoteAgeTimestampMap { + downloadList = append(downloadList, global.FSMisc+ageFile) + } // call sftpSync with the download and upload lists if timeSynced && (max(len(downloadList), len(uploadList)) > 0) { // only call sftpSync if there are entries to download or upload fmt.Println() // add a gap between list-add messages and the actual sync messages from sftpSync - err := sftpSync(sshClient, sshEntryRoot, sshIsWindows, downloadList, uploadList) + err := sftpSync(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, downloadList, uploadList) if err != nil { return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error()) } @@ -395,16 +479,22 @@ func syncLists(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows, timeSyn // deletionSync removes entries from the client that have been deleted on the server (multi-client deletion). func deletionSync(deletions []string) error { - var filesDeleted bool + var entryDeleted bool for _, deletion := range deletions { - filesDeleted = true // set a flag to indicate that files have been deleted (used to determine whether to print a gap between deletion and other messages) - fmt.Println(synccommon.AnsiDelete+deletion+back.AnsiReset, "has been sheared, removing locally (if it exists)") - err := os.RemoveAll(global.TargetLocationFormat(deletion)) + deletionSplit := strings.Split(deletion, global.FSSpace) + if deletionSplit[0] == "entry" { + entryDeleted = true // set a flag to indicate that at least one entry has been deleted (used to determine whether to print a gap between deletion and other messages) + fmt.Println(synccommon.AnsiDelete+deletionSplit[1]+back.AnsiReset, "has been sheared, removing locally (if it exists)") + } + err := os.RemoveAll(global.GetRealPath(deletionSplit[1])) if err != nil { - return errors.New("unable to shear " + deletion + " locally: " + err.Error()) + if deletionSplit[0] == "entry" { + return errors.New("unable to shear " + deletionSplit[1] + " locally: " + err.Error()) + } + return errors.New("unable to shear age file for " + deletionSplit[1] + " locally: " + err.Error()) } } - if filesDeleted { + if entryDeleted { fmt.Println() // add a gap between deletion and other messages } return nil @@ -414,7 +504,7 @@ func deletionSync(deletions []string) error { func folderSync(folders []string) error { for _, folder := range folders { // store the full local path of the folder - folderFullPath := global.TargetLocationFormat(folder) + folderFullPath := global.GetRealPath(folder) // check if target path already exists isAccessible, err := back.TargetIsFile(folderFullPath, false) @@ -435,7 +525,7 @@ func folderSync(folders []string) error { // Setting returnLists to true will return the deletions, downloads, and uploads lists for use by the client. func RunJob(returnLists bool) ([3][]string, error) { // get SSH client to re-use throughout the sync process - sshClient, offlineMode, sshIsWindows, sshEntryRoot, err := GetSSHClient() + sshClient, offlineMode, sshIsWindows, sshEntryRoot, sshAgeDir, err := GetSSHClient() if offlineMode { return [3][]string{nil, nil, nil}, nil } @@ -447,7 +537,7 @@ func RunJob(returnLists bool) ([3][]string, error) { }(sshClient) // fetch remote lists - remoteEntryModMap, remoteFolders, deletions, serverTime, clientTime, err := getRemoteDataFromClient(sshClient) + remoteEntryModMap, remoteAgeTimestampMap, remoteFolders, deletions, serverTime, clientTime, err := getRemoteDataFromClient(sshClient) if err != nil { return [3][]string{nil, nil, nil}, errors.New("unable to fetch remote data: " + err.Error()) } @@ -469,6 +559,11 @@ func RunJob(returnLists bool) ([3][]string, error) { if err != nil { return [3][]string{nil, nil, nil}, errors.New("unable to fetch local entry data: " + err.Error()) } + var localAgeTimestampMap map[string]int64 + localAgeTimestampMap, err = synccommon.GetEntryAges() + if err != nil { + return [3][]string{nil, nil, nil}, err + } // before syncing lists, ensure the client and server clocks are synced within 45 seconds var timeSynced = true @@ -481,14 +576,14 @@ func RunJob(returnLists bool) ([3][]string, error) { // sync new and updated entries var lists [3][]string if returnLists { - lists, err = syncLists(sshClient, sshEntryRoot, sshIsWindows, timeSynced, true, localEntryModMap, remoteEntryModMap) + lists, err = syncLists(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, timeSynced, true, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap) if err != nil { return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error()) } lists[0] = deletions return lists, nil } - _, err = syncLists(sshClient, sshEntryRoot, sshIsWindows, timeSynced, false, localEntryModMap, remoteEntryModMap) + _, err = syncLists(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, timeSynced, false, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap) if err != nil { return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error()) } diff --git a/syncclient/oneOff.go b/syncclient/oneOff.go index d1abeef..8d22249 100644 --- a/syncclient/oneOff.go +++ b/syncclient/oneOff.go @@ -15,13 +15,14 @@ import ( // It can safely be called in offline mode, as well, so this is // the intended interface for shearing (ShearLocal should only // be used directly by the server binary). -func ShearRemoteFromClient(targetLocationIncomplete string) error { - deviceID, isDir, err := synccommon.ShearLocal(targetLocationIncomplete, "") // remove the target from the local system and get the device ID of the client +func ShearRemoteFromClient(vanityPath string, onlyShearAgingFile bool) error { + deviceID, isDir, err := synccommon.ShearLocal(vanityPath, "", onlyShearAgingFile) // remove the target from the local system and get the device ID of the client if err != nil { return errors.New("unable to shear target locally: " + err.Error()) } - sshClient, offlineMode, _, _, err := GetSSHClient() + var modifier string + sshClient, offlineMode, _, _, _, err := GetSSHClient() if offlineMode { goto end } @@ -32,13 +33,16 @@ func ShearRemoteFromClient(targetLocationIncomplete string) error { return errors.New("unable to shear target remotely: no device ID found") } - // ensure targetLocationIncomplete ends with a slash if it is a directory (for clarity in shear message) - if isDir && !strings.HasSuffix(targetLocationIncomplete, "/") { - targetLocationIncomplete += "/" + // ensure vanityPath ends with a slash if it is a directory (for clarity in shear message) + if isDir && !strings.HasSuffix(vanityPath, "/") { + vanityPath += "/" } // call the server to remotely shear the target and add it to the deletions list - _, err = GetSSHOutput(sshClient, "libmuttonserver shear", deviceID+"\n"+strings.ReplaceAll(targetLocationIncomplete, global.PathSeparator, global.FSPath)) + if onlyShearAgingFile { + modifier = "-age" + } + _, err = GetSSHOutput(sshClient, "libmuttonserver shear"+modifier, deviceID+"\n"+strings.ReplaceAll(vanityPath, global.PathSeparator, global.FSPath)) if err != nil { return errors.New("unable to shear target remotely: " + err.Error()) } @@ -54,13 +58,13 @@ end: return nil } -// RenameRemoteFromClient renames oldLocationIncomplete to newLocationIncomplete on +// RenameRemoteFromClient renames oldVanityPath to newVanityPath on // the local system and calls the server to perform the rename remotely and add the // old target to the deletions list. // It can safely be called in offline mode, as well, so this is the intended // interface for renaming (RenameLocal should only be used directly by the server binary). -func RenameRemoteFromClient(oldLocationIncomplete, newLocationIncomplete string) error { - err := synccommon.RenameLocal(oldLocationIncomplete, newLocationIncomplete) // move the target on the local system +func RenameRemoteFromClient(oldVanityPath, newVanityPath string) error { + err := synccommon.RenameLocal(oldVanityPath, newVanityPath) // move the target on the local system if err != nil { return errors.New("unable to rename target locally: " + err.Error()) } @@ -69,23 +73,24 @@ func RenameRemoteFromClient(oldLocationIncomplete, newLocationIncomplete string) if err != nil { return errors.New("unable to generate device ID list: " + err.Error()) } + if deviceIDList[0].Name() == "" { + return errors.New("unable to rename target remotely: no device ID found") + } + // create an SSH client - sshClient, offlineMode, _, _, err := GetSSHClient() + sshClient, offlineMode, _, _, _, err := GetSSHClient() if offlineMode { goto end } if err != nil { return errors.New("unable to connect to SSH client: " + err.Error()) } - if deviceIDList[0].Name() == "" { - return errors.New("unable to rename target remotely: no device ID found") - } // call the server to move the target on the remote system and add the old target to the deletions list _, err = GetSSHOutput(sshClient, "libmuttonserver rename", (deviceIDList)[0].Name()+"\n"+ - strings.ReplaceAll(oldLocationIncomplete, global.PathSeparator, global.FSPath)+"\n"+ - strings.ReplaceAll(newLocationIncomplete, global.PathSeparator, global.FSPath)) + strings.ReplaceAll(oldVanityPath, global.PathSeparator, global.FSPath)+"\n"+ + strings.ReplaceAll(newVanityPath, global.PathSeparator, global.FSPath)) if err != nil { return errors.New("unable to rename target remotely: " + err.Error()) } @@ -106,14 +111,14 @@ end: // It can safely be called in offline mode, as well, so this is the // intended interface for adding folders (AddFolderLocal should only be // used directly by the server binary). -func AddFolderRemoteFromClient(targetLocationIncomplete string) error { - err := synccommon.AddFolderLocal(targetLocationIncomplete) // add the folder on the local system +func AddFolderRemoteFromClient(vanityPath string) error { + err := synccommon.AddFolderLocal(vanityPath) // add the folder on the local system if err != nil { return errors.New("unable to add folder locally: " + err.Error()) } // create an SSH client - sshClient, offlineMode, _, _, err := GetSSHClient() + sshClient, offlineMode, _, _, _, err := GetSSHClient() if offlineMode { goto end } @@ -122,7 +127,7 @@ func AddFolderRemoteFromClient(targetLocationIncomplete string) error { } // call the server to create the folder remotely - _, err = GetSSHOutput(sshClient, "libmuttonserver addfolder", strings.ReplaceAll(targetLocationIncomplete, global.PathSeparator, global.FSPath)) // call the server to create the folder remotely + _, err = GetSSHOutput(sshClient, "libmuttonserver addfolder", strings.ReplaceAll(vanityPath, global.PathSeparator, global.FSPath)) // call the server to create the folder remotely if err != nil { return errors.New("unable to add folder remotely: " + err.Error()) } diff --git a/synccommon/common.go b/synccommon/common.go index 2515149..a0b73e3 100644 --- a/synccommon/common.go +++ b/synccommon/common.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/rwinkhart/go-boilerplate/back" "github.com/rwinkhart/libmutton/global" @@ -15,13 +16,11 @@ const ( AnsiDelete = "\033[38;5;1m" ) -var RootLength = len(global.EntryRoot) // length of global.EntryRoot string - // GetModTimes returns a list of all entry modification times. func GetModTimes(entryList []string) []int64 { var modList []int64 for _, file := range entryList { - modTime, _ := os.Stat(global.TargetLocationFormat(file)) + modTime, _ := os.Stat(global.GetRealPath(file)) modList = append(modList, modTime.ModTime().Unix()) } @@ -33,7 +32,7 @@ func GetModTimes(entryList []string) []int64 { // isDir (only on client; for use in ShearRemoteFromClient). // If the local system is a server, it will also add the target to the deletions list for all clients (except the requesting client). // This function should only be used directly by the server binary. -func ShearLocal(targetLocationIncomplete, clientDeviceID string) (string, bool, error) { +func ShearLocal(vanityPath, clientDeviceID string, onlyShearAgingFile bool) (string, bool, error) { // determine if running on a server var onServer bool if clientDeviceID != "" { @@ -45,26 +44,35 @@ func ShearLocal(targetLocationIncomplete, clientDeviceID string) (string, bool, return "", false, errors.New("unable to generate device ID list: " + err.Error()) } - // add the sheared target (incomplete, vanity) to the deletions list (if running on a server) + // add the sheared vanityPath to the deletions list (if running on a server) if onServer { for _, device := range deviceIDList { if device.Name() != clientDeviceID { - fileToClose, err := os.OpenFile(global.ConfigDir+global.PathSeparator+"deletions"+global.PathSeparator+device.Name()+global.FSSpace+strings.ReplaceAll(targetLocationIncomplete, "/", global.FSPath), os.O_CREATE|os.O_WRONLY, 0600) + if !onlyShearAgingFile { + f, err := os.OpenFile(global.ConfigDir+global.PathSeparator+"deletions"+global.PathSeparator+device.Name()+global.FSSpace+"entry"+global.FSSpace+strings.ReplaceAll(vanityPath, "/", global.FSPath), os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + // do not print error as there is currently no way of seeing server-side errors + // failure to add the target to the deletions list will exit the program and result in a client re-uploading the target (non-critical) + os.Exit(back.ErrorWrite) + } + _ = f.Close() // error ignored; if the file could be created, it can probably be closed + } + f, err := os.OpenFile(global.ConfigDir+global.PathSeparator+"deletions"+global.PathSeparator+device.Name()+global.FSSpace+"age"+global.FSSpace+strings.ReplaceAll(vanityPath, "/", global.FSPath), os.O_CREATE|os.O_WRONLY, 0600) if err != nil { // do not print error as there is currently no way of seeing server-side errors // failure to add the target to the deletions list will exit the program and result in a client re-uploading the target (non-critical) os.Exit(back.ErrorWrite) } - _ = fileToClose.Close() // error ignored; if the file could be created, it can probably be closed + _ = f.Close() // error ignored; if the file could be created, it can probably be closed } } } // remove the target locally - targetLocationComplete := global.TargetLocationFormat(targetLocationIncomplete) + realPath := global.GetRealPath(vanityPath) var isFile bool if !onServer { // error if target does not exist on client, needed because os.RemoveAll does not return an error if target does not exist - isAccessible, err := back.TargetIsFile(targetLocationComplete, true) + isAccessible, err := back.TargetIsFile(realPath, true) if !isAccessible { return "", false, err } @@ -72,9 +80,15 @@ func ShearLocal(targetLocationIncomplete, clientDeviceID string) (string, bool, isFile = true } } - err = os.RemoveAll(targetLocationComplete) + if !onlyShearAgingFile { + err = os.RemoveAll(realPath) + if err != nil { + return "", false, errors.New("unable to remove local entry (" + vanityPath + "): " + err.Error()) + } + } + err = ShearAgeFileLocal(vanityPath) if err != nil { - return "", false, errors.New("unable to remove local target: " + err.Error()) + return "", false, err } if !onServer && len(deviceIDList) > 0 { // return the device ID if running on the client and a device ID exists (online mode) @@ -85,36 +99,84 @@ func ShearLocal(targetLocationIncomplete, clientDeviceID string) (string, bool, // do not exit program, as this function is used as part of ShearRemoteFromClient } +// ShearAgeFileLocal removes the age file for a vanity path. +// This function should only be used directly by the server binary. +func ShearAgeFileLocal(vanityPath string) error { + err := os.RemoveAll(global.AgeDir + global.PathSeparator + strings.ReplaceAll(vanityPath, "/", global.FSPath)) + if err != nil { + return errors.New("unable to remove age file for " + vanityPath + ": " + err.Error()) + } + return nil +} + +// GetEntryAges reads the aging directory and returns a +// map of vanity paths to their corresponding age timestamps. +func GetEntryAges() (map[string]int64, error) { + contents, err := os.ReadDir(global.AgeDir) + if err != nil { + return nil, errors.New("unable to read aging directory contents: " + err.Error()) + } + + var vanityPathsToTimestamps = make(map[string]int64) + for _, dirEntry := range contents { + if !dirEntry.IsDir() { + vanityPath := strings.ReplaceAll(dirEntry.Name(), global.FSPath, "/") + info, err := dirEntry.Info() + if err != nil { + return nil, errors.New("unable to read aging file modtime for " + vanityPath + ": " + err.Error()) + } + vanityPathsToTimestamps[vanityPath] = info.ModTime().Unix() + } + } + + return vanityPathsToTimestamps, nil +} + // RenameLocal renames oldLocationIncomplete to newLocationIncomplete on the local system. // This function should only be used directly by the server binary. -func RenameLocal(oldLocationIncomplete, newLocationIncomplete string) error { +func RenameLocal(oldVanityPath, newVanityPath string) error { // get full paths for both locations - oldLocation := global.TargetLocationFormat(oldLocationIncomplete) - newLocation := global.TargetLocationFormat(newLocationIncomplete) + oldRealPath := global.GetRealPath(oldVanityPath) + oldRealAgePath := global.GetRealAgePath(oldVanityPath) + newRealPath := global.GetRealPath(newVanityPath) + newRealAgePath := global.GetRealAgePath(newVanityPath) // ensure newLocation does not exist - isAccessible, _ := back.TargetIsFile(newLocation, true) // error is ignored because dir/file status is irrelevant + isAccessible, _ := back.TargetIsFile(newRealPath, true) // error is ignored because dir/file status is irrelevant if isAccessible { - return errors.New("new target (" + newLocation + ") already exists") + return errors.New("new target (" + newRealPath + ") already exists") } // rename oldLocation to newLocation - err := os.Rename(oldLocation, newLocation) + err := os.Rename(oldRealPath, newRealPath) if err != nil { return errors.New("unable to rename: " + err.Error()) } - return nil + // do the same for the age file (if one exists) - also back up timestamp first + var fileInfo os.FileInfo + fileInfo, err = os.Stat(oldRealAgePath) + if err == nil { // assume age file does not exist if os.Stat errors + err = os.Rename(oldRealAgePath, newRealAgePath) + if err != nil { + return errors.New("unable to rename: " + err.Error()) + } + err = os.Chtimes(newRealAgePath, time.Now(), fileInfo.ModTime()) + if err != nil { + return errors.New("unable to set timestamp on age file for " + newVanityPath + ": " + err.Error()) + } + } + return nil // do not exit program, as this function is used as part of RenameRemoteFromClient } // AddFolderLocal creates a new entry-containing directory on the local system. // This function should only be used directly by the server binary. -func AddFolderLocal(targetLocationIncomplete string) error { +func AddFolderLocal(vanityPath string) error { // create the target locally - targetLocationComplete := global.TargetLocationFormat(targetLocationIncomplete) - err := os.Mkdir(targetLocationComplete, 0700) + realPath := global.GetRealPath(vanityPath) + err := os.Mkdir(realPath, 0700) if err != nil { if os.IsExist(err) { fmt.Println(back.AnsiBlue + "Directory already exists - libmutton will still ensure it exists on the server") diff --git a/synccommon/walk_WIN.go b/synccommon/walk.go similarity index 75% rename from synccommon/walk_WIN.go rename to synccommon/walk.go index 52b6667..6d96dd6 100644 --- a/synccommon/walk_WIN.go +++ b/synccommon/walk.go @@ -1,5 +1,3 @@ -//go:build windows - package synccommon import ( @@ -7,21 +5,20 @@ import ( "io/fs" "os" "path/filepath" - "strings" "github.com/rwinkhart/libmutton/global" ) -// WalkEntryDir walks the entry directory and returns lists of all files and directories found (two separate lists). +// WalkEntryDir walks the entry directory and returns lists of all files and +// directories found (two separate lists) relative to the libmutton entry root. // Regardless of platform, all paths are stored with forward slashes (UNIX-style). func WalkEntryDir() ([]string, []string, error) { // define file/directory containing slices so that they may be accessed by the anonymous WalkDir function - var fileList []string - var dirList []string + var fileList, dirList []string // walk entry directory err := filepath.WalkDir(global.EntryRoot, - func(fullPath string, entry fs.DirEntry, err error) error { + func(realPath string, entry fs.DirEntry, err error) error { // check for errors encountered while walking directory if err != nil { @@ -33,13 +30,13 @@ func WalkEntryDir() ([]string, []string, error) { } // trim root path from each path before storing and replace backslashes with forward slashes - trimmedPath := strings.ReplaceAll(fullPath[RootLength:], "\\", "/") + vanityPath := global.GetVanityPath(realPath) // append the path to the appropriate slice if !entry.IsDir() { - fileList = append(fileList, trimmedPath) + fileList = append(fileList, vanityPath) } else { - dirList = append(dirList, trimmedPath) + dirList = append(dirList, vanityPath) } return nil diff --git a/synccommon/walk_UNIX.go b/synccommon/walk_UNIX.go deleted file mode 100644 index 7937738..0000000 --- a/synccommon/walk_UNIX.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build !windows - -package synccommon - -import ( - "errors" - "io/fs" - "os" - "path/filepath" - - "github.com/rwinkhart/libmutton/global" -) - -// WalkEntryDir walks the entry directory and returns lists of all files and directories found (two separate lists). -// Regardless of platform, all paths are stored with forward slashes (UNIX-style). -func WalkEntryDir() ([]string, []string, error) { - // define file/directory containing slices so that they may be accessed by the anonymous WalkDir function - var fileList []string - var dirList []string - - // walk entry directory - err := filepath.WalkDir(global.EntryRoot, - func(fullPath string, entry fs.DirEntry, err error) error { - - // check for errors encountered while walking directory - if err != nil { - if os.IsNotExist(err) { - return errors.New("entry directory does not exist; initialize libmutton to create it") - } else { - return errors.New("an unexpected error occurred while generating the entry list: " + err.Error()) - } - } - - // trim root path from each path before storing - trimmedPath := fullPath[RootLength:] - - // append the path to the appropriate slice - if !entry.IsDir() { - fileList = append(fileList, trimmedPath) - } else { - dirList = append(dirList, trimmedPath) - } - - return nil - }) - if err != nil { - return nil, nil, err - } - return fileList, dirList, nil -} diff --git a/synccycles/init.go b/synccycles/init.go index 4c1087f..e49977c 100644 --- a/synccycles/init.go +++ b/synccycles/init.go @@ -17,8 +17,8 @@ import ( // Device IDs are only needed for online synchronization. // Device IDs are guaranteed unique as the current UNIX time is appended to them. // Leave prefix empty to use the current hostname as the prefix. -// Returns: the remote EntryRoot and OS type indicator. -func DeviceIDGen(oldDeviceID, prefix string) (string, string, error) { +// Returns: the remote EntryRoot, the remote AgeDir, and OS type indicator. +func DeviceIDGen(oldDeviceID, prefix string) (string, string, string, error) { // generate new device ID if prefix == "" { prefix, _ = os.Hostname() @@ -30,7 +30,7 @@ func DeviceIDGen(oldDeviceID, prefix string) (string, string, error) { oldDeviceIDPath := global.ConfigDir + global.PathSeparator + "devices" + global.PathSeparator + oldDeviceID f, err := os.OpenFile(newDeviceIDPath, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { - return "", "", errors.New("unable to create local device ID file: " + err.Error()) + return "", "", "", errors.New("unable to create local device ID file: " + err.Error()) } _ = f.Close() // error ignored; if the file could be created, it can probably be closed @@ -49,24 +49,24 @@ func DeviceIDGen(oldDeviceID, prefix string) (string, string, error) { // register new device ID with server and fetch remote EntryRoot and OS type // also removes the old device ID file (remotely) // if registration fails, remove the new device ID file locally and return before removing the old one - sshClient, _, _, _, err := syncclient.GetSSHClient() + sshClient, _, _, _, _, err := syncclient.GetSSHClient() if err != nil { cleanupOnFail() - return "", "", errors.New("unable to connect to SSH client: " + err.Error()) + return "", "", "", errors.New("unable to connect to SSH client: " + err.Error()) } output, err := syncclient.GetSSHOutput(sshClient, "libmuttonserver register", newDeviceID+"\n"+oldDeviceID) if err != nil { cleanupOnFail() - return "", "", errors.New("unable to register device ID with server: " + err.Error()) + return "", "", "", errors.New("unable to register device ID with server: " + err.Error()) } - sshEntryRootSSHIsWindows := strings.Split(output, global.FSSpace) + sshEntryRootSSHAgeDirSSHIsWindows := strings.Split(output, global.FSSpace) _ = sshClient.Close() // ignore error; non-critical/unlikely/not much could be done about it // remove old device ID file (locally; may not exist) err = os.RemoveAll(oldDeviceIDPath) if err != nil { - return "", "", errors.New("unable to remove old device ID file (locally): " + err.Error()) + return "", "", "", errors.New("unable to remove old device ID file (locally): " + err.Error()) } - return sshEntryRootSSHIsWindows[0], sshEntryRootSSHIsWindows[1], nil + return sshEntryRootSSHAgeDirSSHIsWindows[0], sshEntryRootSSHAgeDirSSHIsWindows[1], sshEntryRootSSHAgeDirSSHIsWindows[2], nil } diff --git a/syncserver/server.go b/syncserver/server.go index 8bc3e3b..828e29e 100644 --- a/syncserver/server.go +++ b/syncserver/server.go @@ -17,6 +17,13 @@ func GetRemoteDataFromServer(clientDeviceID string) { entryList, dirList, _ := synccommon.WalkEntryDir() modList := synccommon.GetModTimes(entryList) deletionsList, _ := os.ReadDir(global.ConfigDir + global.PathSeparator + "deletions") + vanityPathsToTimestamps, _ := synccommon.GetEntryAges() + var ageVanityPaths []string + var ageTimestamps []int64 + for vanityPath, timestamp := range vanityPathsToTimestamps { + ageVanityPaths = append(ageVanityPaths, vanityPath) + ageTimestamps = append(ageTimestamps, timestamp) + } // print the current UNIX timestamp to stdout fmt.Print(time.Now().Unix()) @@ -35,6 +42,19 @@ func GetRemoteDataFromServer(clientDeviceID string) { fmt.Print(mod) } + // age file list + fmt.Print(global.FSSpace) + for _, file := range ageVanityPaths { + fmt.Print(global.FSMisc + file) + } + + // age file timestamp list + fmt.Print(global.FSSpace) + for _, timestamp := range ageTimestamps { + fmt.Print(global.FSMisc) + fmt.Print(timestamp) + } + // directory/folder list fmt.Print(global.FSSpace) for _, dir := range dirList { @@ -44,13 +64,13 @@ func GetRemoteDataFromServer(clientDeviceID string) { // deletions list fmt.Print(global.FSSpace) for _, deletion := range deletionsList { - // print deletion if it is relevant to the current client device - affectedIDTargetLocationIncomplete := strings.Split(deletion.Name(), global.FSSpace) - if affectedIDTargetLocationIncomplete[0] == clientDeviceID { - fmt.Print(global.FSMisc + strings.ReplaceAll(affectedIDTargetLocationIncomplete[1], global.FSPath, "/")) + // perform deletion if it is relevant to the current client device + affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace) + if affectedIDVanityPath[0] == clientDeviceID { + fmt.Print(global.FSMisc + strings.ReplaceAll(affectedIDVanityPath[1]+global.FSSpace+affectedIDVanityPath[2], global.FSPath, "/")) // assume successful client deletion and remove deletions file (if assumption is somehow false, worst case scenario is that the client will re-upload the deleted entry) - _ = os.Remove(global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator + deletion.Name()) // error ignored; function not run from a user-facing argument and thus the error would not be visible + _ = os.RemoveAll(global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator + deletion.Name()) // error ignored; function not run from a user-facing argument and thus the error would not be visible } } }