Major sync refactor (do not use SFTP for age files; allow using core.LibmuttonInit with custom device ID prefixes)

age/age.go

@@ -14,7 +14,7 @@ import (
 	"github.com/rwinkhart/libmutton/synccommon"
 )
 
-// Entry creates updates the age file for a vanity path.
+// Entry creates/updates the age file for a vanity path.
 func Entry(vanityPath string, timestamp int64) error {
 	ageFilePath := global.AgeDir + global.PathSeparator + strings.ReplaceAll(vanityPath, "/", global.FSPath)
 	f, err := os.OpenFile(ageFilePath, os.O_CREATE|os.O_WRONLY, 0600)
@@ -72,11 +72,11 @@ func AllPasswordEntries(forceReage bool) error {
 // with an entry.
 // Magic number legend:
 // 0 -> no age, 1 -> fresh, 2 -> expiring soon (within a month), 3 -> expired
-func TranslateAgeTimestamp(timestamp int64) uint8 {
+func TranslateAgeTimestamp(timestamp *int64) uint8 {
-	if timestamp == 0 {
+	if timestamp == nil {
 		return 0
 	}
-	daysOld := time.Since(time.Unix(timestamp, 0)).Hours() / 24
+	daysOld := time.Since(time.Unix(*timestamp, 0)).Hours() / 24
 	if daysOld >= 365 {
 		return 3 // expired
 	} else if daysOld >= 335 {

config/cfg.go

@@ -16,7 +16,6 @@ type CfgT struct {
 		SSHIP            *string `json:"sshIP"`
 		SSHPort          *string `json:"sshPort"`
 		SSHEntryRootPath *string `json:"sshEntryRootPath"`
-		SSHAgeDirPath    *string `json:"sshAgeDirPath"`
 		SSHKeyPath       *string `json:"sshKeyPath"`
 		SSHKeyProtected  *bool   `json:"sshKeyProtected"`
 		SSHIsWindows     *bool   `json:"sshIsWindows"`

core/init.go

@@ -14,8 +14,9 @@ import (
 )
 
 // LibmuttonInit creates the libmutton config structure based on user input.
-// rcwPassword and clientSpecificCfg can be left blank/nil if not needed.
+// deviceIDPrefix can be left blank to use the system hostname.
-func LibmuttonInit(inputCB func(prompt string) string, clientSpecificCfg map[string]any, rcwPassword []byte, appendMode, forceOfflineMode bool) error {
+// clientSpecificCfg can be left nil if not needed.
+func LibmuttonInit(inputCB func(prompt string) string, rcwPassword []byte, appendMode, forceOfflineMode bool, deviceIDPrefix string, clientSpecificCfg map[string]any) error {
 	// handle clientSpecificCfg
 	newCfg := &config.CfgT{}
 	if clientSpecificCfg != nil {
@@ -40,8 +41,7 @@ func LibmuttonInit(inputCB func(prompt string) string, clientSpecificCfg map[str
 		// write config file
 		offlineMode := true
 		newCfg.Libmutton.OfflineMode = &offlineMode
-		err = config.Write(newCfg, false)
+		if err = config.Write(newCfg, false); err != nil {
-		if err != nil {
 			return err
 		}
 	} else {
@@ -77,30 +77,24 @@ func LibmuttonInit(inputCB func(prompt string) string, clientSpecificCfg map[str
 		newCfg.Libmutton.SSHPort = &sshPort
 		newCfg.Libmutton.SSHKeyPath = &sshKeyPath
 		newCfg.Libmutton.SSHKeyProtected = &sshKeyProtected
-		err = config.Write(newCfg, appendMode) // pass appendMode to allow not completely destroying existing (client-specific) config
+		if err = config.Write(newCfg, appendMode); err != nil { // pass appendMode to allow not completely destroying existing (client-specific) config
-		if err != nil {
 			return err
 		}
 		// generate and register device ID
-		sshEntryRoot, sshAgeDir, sshIsWindows, err := syncclient.GenDeviceID(oldDeviceID, "")
+		sshEntryRoot, sshIsWindows, err := syncclient.GenDeviceID(oldDeviceID, deviceIDPrefix)
 		if err != nil {
 			return errors.New("unable to generate device ID: " + err.Error())
 		}
 		// update config file
 		newCfg.Libmutton.SSHEntryRootPath = &sshEntryRoot
-		newCfg.Libmutton.SSHAgeDirPath = &sshAgeDir
 		newCfg.Libmutton.SSHIsWindows = &sshIsWindows
-		err = config.Write(newCfg, true)
+		if err = config.Write(newCfg, true); err != nil {
-		if err != nil {
 			return err
 		}
 	}
-	// generate rcw sanity check file (if requested)
+	// generate rcw sanity check file
-	if rcwPassword != nil {
+	if err := RCWSanityCheckGen(rcwPassword); err != nil {
-		err := RCWSanityCheckGen(rcwPassword)
+		return err
-		if err != nil {
-			return err
-		}
 	}
 	return nil
 }

libmuttonserver.go

@@ -129,8 +129,8 @@ func main() {
 			}
 		}
 
-		// print EntryRoot, AgeDir and bool indicating OS type to stdout for client to store in config
+		// print EntryRoot and bool indicating OS type to stdout for client to store in config
-		registerResp := synccommon.RegisterResp{EntryRoot: global.EntryRoot, AgeDir: global.AgeDir, IsWindows: global.IsWindows}
+		registerResp := synccommon.RegisterResp{EntryRoot: global.EntryRoot, IsWindows: global.IsWindows}
 		registerRespBytes, err := json.Marshal(registerResp)
 		if err != nil {
 			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())

syncclient/client.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/pkg/sftp"
 	"github.com/rwinkhart/go-boilerplate/back"
+	"github.com/rwinkhart/libmutton/age"
 	"github.com/rwinkhart/libmutton/config"
 	"github.com/rwinkhart/libmutton/global"
 	"github.com/rwinkhart/libmutton/synccommon"
@@ -23,22 +24,21 @@ import (
 // offlineMode (whether the client is in offline mode).
 // sshIsWindows (whether the remote server is running Windows),
 // sshEntryRoot (the root directory for entries on the remote server),
-// sshAgeDir (the directory housing age files on the remote server),
 // Only supports key-based authentication (passwords are supported for CLI-based implementations).
-func GetSSHClient() (*ssh.Client, bool, *bool, *string, *string, error) {
+func GetSSHClient() (*ssh.Client, bool, *bool, *string, error) {
 	// get SSH config info
 	cfg, err := config.Load()
 	if err != nil {
-		return nil, false, nil, nil, nil, errors.New("unable to parse SSH config: " + err.Error())
+		return nil, false, nil, nil, errors.New("unable to parse SSH config: " + err.Error())
 	}
 	if *cfg.Libmutton.OfflineMode {
-		return nil, true, nil, nil, nil, nil
+		return nil, true, nil, nil, nil
 	}
 
 	// read private key
 	key, err := os.ReadFile(*cfg.Libmutton.SSHKeyPath)
 	if err != nil {
-		return nil, false, nil, nil, nil, errors.New("unable to read private key: " + *cfg.Libmutton.SSHKeyPath)
+		return nil, false, nil, nil, errors.New("unable to read private key: " + *cfg.Libmutton.SSHKeyPath)
 	}
 
 	// parse private key
@@ -49,14 +49,14 @@ func GetSSHClient() (*ssh.Client, bool, *bool, *string, *string, error) {
 		parsedKey, err = ssh.ParsePrivateKeyWithPassphrase(key, global.GetPassword("Enter password for your SSH keyfile:"))
 	}
 	if err != nil {
-		return nil, false, nil, nil, nil, errors.New("unable to parse private key: " + *cfg.Libmutton.SSHKeyPath)
+		return nil, false, nil, nil, errors.New("unable to parse private key: " + *cfg.Libmutton.SSHKeyPath)
 	}
 
 	// read known hosts file
 	var hostKeyCallback ssh.HostKeyCallback
 	hostKeyCallback, err = knownhosts.New(back.Home + global.PathSeparator + ".ssh" + global.PathSeparator + "known_hosts")
 	if err != nil {
-		return nil, false, nil, nil, nil, errors.New("unable to read known hosts file: " + err.Error())
+		return nil, false, nil, nil, errors.New("unable to read known hosts file: " + err.Error())
 	}
 
 	// configure SSH client
@@ -72,10 +72,10 @@ func GetSSHClient() (*ssh.Client, bool, *bool, *string, *string, error) {
 	// connect to SSH server
 	sshClient, err := ssh.Dial("tcp", *cfg.Libmutton.SSHIP+":"+*cfg.Libmutton.SSHPort, sshCfg)
 	if err != nil {
-		return nil, false, nil, nil, nil, errors.New("unable to connect to remote server: " + err.Error())
+		return nil, false, nil, nil, errors.New("unable to connect to remote server: " + err.Error())
 	}
 
-	return sshClient, false, cfg.Libmutton.SSHIsWindows, cfg.Libmutton.SSHEntryRootPath, cfg.Libmutton.SSHAgeDirPath, nil
+	return sshClient, false, cfg.Libmutton.SSHIsWindows, cfg.Libmutton.SSHEntryRootPath, nil
 }
 
 // GetSSHOutput runs a command over SSH and returns the output as a string.
@@ -100,71 +100,35 @@ func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) ([]byte, error) {
 }
 
 // getRemoteDataFromClient returns:
-// a map of remote entries to their modification times,
+// a map of remote vanityPaths to their containing folders and mod+age timestamps,
-// a map of remote entries to their timestamps,
-// a list of remote folders,
 // a list of queued deletions,
 // and the current server&client times as UNIX timestamps.
-func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[string]int64, []string, []synccommon.Deletion, int64, int64, error) {
+func getRemoteDataFromClient(sshClient *ssh.Client) (synccommon.EntriesMap, []synccommon.Deletion, int64, int64, error) {
 	// get remote output over SSH
 	deviceIDList, err := global.GenDeviceIDList()
 	if err != nil {
-		return nil, nil, nil, nil, 0, 0, err
+		return nil, nil, 0, 0, err
 	}
 	if len(deviceIDList) == 0 {
-		return nil, nil, nil, nil, 0, 0, errors.New("no device ID found")
+		return nil, nil, 0, 0, errors.New("no device ID found")
 	}
 	clientTime := time.Now().Unix() // get client time now to avoid accuracy issues caused by unpredictable sync time
 	output, err := GetSSHOutput(sshClient, "libmuttonserver fetch", (deviceIDList)[0].Name())
 	if err != nil {
-		return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command: " + err.Error())
+		return nil, nil, 0, 0, errors.New("unable to run remote command: " + err.Error())
 	}
 
 	var fetchResp synccommon.FetchResp
 	err = json.Unmarshal(output, &fetchResp)
 	if err != nil {
 		fmt.Println(string(output))
-		return nil, nil, nil, nil, 0, 0, errors.New("unable to unmarshal server fetch response: " + err.Error())
+		return nil, nil, 0, 0, errors.New("unable to unmarshal server fetch response: " + err.Error())
 	}
 	if fetchResp.ErrMsg != nil {
-		return nil, nil, nil, nil, 0, 0, errors.New("unable to complete fetch; server-side error occurred: " + strings.ReplaceAll(*fetchResp.ErrMsg, global.FSSpace, "\n"))
+		return nil, nil, 0, 0, errors.New("unable to complete fetch; server-side error occurred: " + strings.ReplaceAll(*fetchResp.ErrMsg, global.FSSpace, "\n"))
 	}
 
-	entryModMap := make(map[string]int64)
+	return fetchResp.Entries, fetchResp.Deletions, fetchResp.ServerTime, clientTime, nil
-	ageTimestampMap := make(map[string]int64)
-	var folders []string
-	for folderName, containedEntries := range fetchResp.FoldersToEntries {
-		folders = append(folders, folderName)
-		for _, entry := range containedEntries {
-			entryModMap[entry.VanityPath] = entry.ModTime
-			if entry.AgeTimestamp != nil {
-				ageTimestampMap[entry.VanityPath] = *entry.AgeTimestamp
-			}
-		}
-	}
-
-	return entryModMap, ageTimestampMap, folders, fetchResp.Deletions, fetchResp.ServerTime, clientTime, nil
-}
-
-// getLocalData returns a map of local entries to their modification times.
-func getLocalData() (map[string]int64, error) {
-	// get a list of all entries
-	entries, _, err := synccommon.WalkEntryDir()
-	if err != nil {
-		return nil, err
-	}
-
-	// get a list of all entry modification times
-	modList := synccommon.GetModTimes(entries)
-
-	// map the entries to their modification times
-	entryModMap := make(map[string]int64)
-	for i, entry := range entries {
-		entryModMap[entry] = modList[i]
-	}
-
-	// return the lists
-	return entryModMap, nil
 }
 
 // getRealPathSFTP formats the vanityPath to match the remote server's entry/age file directory and path separator.
@@ -175,16 +139,8 @@ func getRealPathSFTP(vanityPath, serverEntryRoot string, serverIsWindows bool) s
 	return serverEntryRoot + strings.ReplaceAll(vanityPath, "/", "\\")
 }
 
-// getRealPathSFTP formats the vanityPath to match the remote server's entry/age file directory and path separator.
-func getRealAgePathSFTP(vanityPath, serverAgeDir string, serverIsWindows bool) string {
-	if !serverIsWindows {
-		return serverAgeDir + "/" + strings.ReplaceAll(vanityPath, "/", global.FSPath)
-	}
-	return serverAgeDir + "\\" + strings.ReplaceAll(vanityPath, "/", global.FSPath)
-}
-
 // sftpSync takes two slices of entries (one for downloads and one for uploads) and syncs them between the client and server using SFTP.
-func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindows bool, downloadList, uploadList []string) error {
+func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, downloadList, uploadList []string) error {
 	// create an SFTP client from sshClient
 	sftpClient, err := sftp.NewClient(sshClient)
 	if err != nil {
@@ -197,23 +153,12 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindow
 	// iterate over the download list
 	var filesTransferred bool
 	for _, vanityPath := range downloadList {
-		// determine if remote file is an age file
+		filesTransferred = true // set a flag to indicate that files have been downloaded (used to determine whether to print a gap between download and upload messages)
-		var isAgeFile bool
+		fmt.Println("Downloading " + back.AnsiGreen + vanityPath + back.AnsiReset)
-		if strings.HasPrefix(vanityPath, global.FSMisc) {
-			vanityPath = strings.TrimLeft(vanityPath, global.FSMisc)
-			isAgeFile = true
-		} else {
-			filesTransferred = true // set a flag to indicate that files have been downloaded (used to determine whether to print a gap between download and upload messages)
-			fmt.Println("Downloading " + back.AnsiGreen + vanityPath + back.AnsiReset)
-		}
 
 		// store path to remote entry
 		var remoteFileRealPath string
-		if isAgeFile {
+		remoteFileRealPath = getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows)
-			remoteFileRealPath = getRealAgePathSFTP(vanityPath, sshAgeDir, sshIsWindows)
-		} else {
-			remoteFileRealPath = getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows)
-		}
 
 		// save modification time of remote file
 		var fileInfo os.FileInfo
@@ -232,11 +177,7 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindow
 
 		// store path to local file
 		var localFileRealPath string
-		if isAgeFile {
+		localFileRealPath = global.GetRealPath(vanityPath)
-			localFileRealPath = global.GetRealAgePath(vanityPath)
-		} else {
-			localFileRealPath = global.GetRealPath(vanityPath)
-		}
 
 		// create local file
 		var localFile *os.File
@@ -303,12 +244,7 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindow
 		}
 
 		// store path to remote entry
-		var remoteFileRealPath string
+		remoteFileRealPath := getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows)
-		if isAgeFile {
-			remoteFileRealPath = getRealAgePathSFTP(vanityPath, sshAgeDir, sshIsWindows)
-		} else {
-			remoteFileRealPath = getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows)
-		}
 
 		// create remote file
 		var remoteFile *sftp.File
@@ -347,75 +283,59 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindow
 	return nil
 }
 
-// syncLists determines which entries need to be downloaded and uploaded for synchronizations and calls sftpSync with this information.
+// syncLists determines which entries need to be downloaded and uploaded
-// Using maps means that syncing will be done in an arbitrary order, but it is a worthy tradeoff for speed and simplicity.
+// for synchronization and calls sftpSync with this information.
-func syncLists(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindows, timeSynced, returnLists bool, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap map[string]int64) ([3][]string, error) {
+func syncLists(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, timeSyncedErr error, localEntryMap, remoteEntryMap synccommon.EntriesMap) ([3][]string, error) {
 	// initialize slices to store entries that need to be downloaded or uploaded
 	var downloadList, uploadList []string
 
-	// iterate over client entries
+	// iterate over client entries in local map
-	localMapIter := func(localMap, remoteMap map[string]int64, forAging bool) {
+	for vanityPath, localInfo := range localEntryMap {
-		for file, localTime := range localMap {
+		// check if the entry is present on the server
-			// check if the entry is present in the server map
+		if _, exists := remoteEntryMap[vanityPath]; exists {
-			if remoteTime, present := remoteMap[file]; present {
+			// entry exists on both client and server, compare mod times
-				// entry exists on both client and server, compare mod times
+			remoteInfo := remoteEntryMap[vanityPath]
-				if remoteTime > localTime {
+			if remoteInfo.ModTime > localInfo.ModTime {
-					if !forAging {
+				fmt.Println(back.AnsiGreen+vanityPath+back.AnsiReset, "is newer on server, adding to download list")
-						fmt.Println(back.AnsiGreen+file+back.AnsiReset, "is newer on server, adding to download list")
+				downloadList = append(downloadList, vanityPath)
-						downloadList = append(downloadList, file)
+				if remoteInfo.AgeTimestamp != nil {
-					} else {
+					age.Entry(vanityPath, *remoteInfo.AgeTimestamp)
-						downloadList = append(downloadList, global.FSMisc+file)
-					}
-				} else if remoteTime < localTime {
-					if !forAging {
-						fmt.Println(back.AnsiBlue+file+back.AnsiReset, "is newer on client, adding to upload list")
-						uploadList = append(uploadList, file)
-					} else {
-						uploadList = append(uploadList, global.FSMisc+file)
-					}
-				}
-				// remove entry from remoteMap (process of elimination)
-				delete(remoteMap, file)
-			} else {
-				if !forAging {
-					fmt.Println(back.AnsiBlue+file+back.AnsiReset, "does not exist on server, adding to upload list")
-					uploadList = append(uploadList, file)
-				} else {
-					uploadList = append(uploadList, global.FSMisc+file)
 				}
+			} else if remoteInfo.ModTime < localInfo.ModTime {
+				fmt.Println(back.AnsiBlue+vanityPath+back.AnsiReset, "is newer on client, adding to upload list")
+				uploadList = append(uploadList, vanityPath)
 			}
+			// remove entry from remote map (process of elimination)
+			delete(remoteEntryMap, vanityPath)
+		} else {
+			fmt.Println(back.AnsiBlue+vanityPath+back.AnsiReset, "does not exist on server, adding to upload list")
+			uploadList = append(uploadList, vanityPath)
 		}
 	}
-	localMapIter(localEntryModMap, remoteEntryModMap, false)
-	localMapIter(localAgeTimestampMap, remoteAgeTimestampMap, true)
 
-	// iterate over remaining entries in remote maps
+	// iterate over remaining entries in remote map
-	for entry := range remoteEntryModMap {
+	for vanityPath, remoteInfo := range remoteEntryMap {
-		fmt.Println(back.AnsiGreen+entry+back.AnsiReset, "does not exist on client, adding to download list")
+		fmt.Println(back.AnsiGreen+vanityPath+back.AnsiReset, "does not exist on client, adding to download list")
-		downloadList = append(downloadList, entry)
+		downloadList = append(downloadList, vanityPath)
-	}
+		if err := os.MkdirAll(global.GetRealPath(remoteInfo.ContainingFolder), 0700); err != nil {
-	for ageFile := range remoteAgeTimestampMap {
+			return [3][]string{nil, nil, nil}, errors.New("unable to create containing folder for " + vanityPath + ": " + err.Error())
-		downloadList = append(downloadList, global.FSMisc+ageFile)
+		}
+		if remoteInfo.AgeTimestamp != nil {
+			age.Entry(vanityPath, *remoteInfo.AgeTimestamp)
+		}
 	}
 
 	// call sftpSync with the download and upload lists
-	if timeSynced && (max(len(downloadList), len(uploadList)) > 0) { // only call sftpSync if there are entries to download or upload
+	if timeSyncedErr == nil && (max(len(downloadList), len(uploadList)) > 0) { // only call sftpSync if there are entries to download or upload
 		fmt.Println() // add a gap between list-add messages and the actual sync messages from sftpSync
-		err := sftpSync(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, downloadList, uploadList)
+		err := sftpSync(sshClient, sshEntryRoot, sshIsWindows, downloadList, uploadList)
 		if err != nil {
 			return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error())
 		}
-	} else if !timeSynced {
+		fmt.Println("Client is synchronized with server")
-		// do not call sftpSync if the client and server times are out of sync
-		back.Exit(global.ErrorSyncProcess)
 	}
 
-	fmt.Println("Client is synchronized with server")
+	return [3][]string{nil, downloadList, uploadList}, timeSyncedErr
-
-	if returnLists {
-		return [3][]string{nil, downloadList, uploadList}, nil
-	}
-	return [3][]string{nil, nil, nil}, nil
 }
 
 // deletionSync removes entries from the client that have been deleted on the server (multi-client deletion).
@@ -440,32 +360,11 @@ func deletionSync(deletions []synccommon.Deletion) error {
 	return nil
 }
 
-// folderSync creates folders on the client (from the given list of folder names).
+// RunJob runs the SSH sync job and returns deletions, downloads,
-func folderSync(folders []string) error {
+// and uploads lists for the client to report to the user.
-	for _, folder := range folders {
+func RunJob() ([3][]string, error) {
-		// store the full local path of the folder
-		folderFullPath := global.GetRealPath(folder)
-
-		// check if target path already exists
-		isAccessible, err := back.TargetIsFile(folderFullPath, false)
-
-		if !isAccessible {
-			err := os.MkdirAll(folderFullPath, 0700)
-			if err != nil {
-				return errors.New("unable to create folder (" + folder + "): " + err.Error())
-			}
-		} else if err != nil {
-			return errors.New("unable to create folder (" + folder + "): " + err.Error())
-		}
-	}
-	return nil
-}
-
-// RunJob runs the SSH sync job.
-// Setting returnLists to true will return the deletions, downloads, and uploads lists for use by the client.
-func RunJob(returnLists bool) ([3][]string, error) {
 	// get SSH client to re-use throughout the sync process
-	sshClient, offlineMode, sshIsWindows, sshEntryRoot, sshAgeDir, err := GetSSHClient()
+	sshClient, offlineMode, sshIsWindows, sshEntryRoot, err := GetSSHClient()
 	if offlineMode {
 		return [3][]string{nil, nil, nil}, nil
 	}
@@ -477,7 +376,7 @@ func RunJob(returnLists bool) ([3][]string, error) {
 	}(sshClient)
 
 	// fetch remote lists
-	remoteEntryModMap, remoteAgeTimestampMap, remoteFolders, deletions, serverTime, clientTime, err := getRemoteDataFromClient(sshClient)
+	remoteEntryMap, deletions, serverTime, clientTime, err := getRemoteDataFromClient(sshClient)
 	if err != nil {
 		return [3][]string{nil, nil, nil}, errors.New("unable to fetch remote data: " + err.Error())
 	}
@@ -488,50 +387,33 @@ func RunJob(returnLists bool) ([3][]string, error) {
 		return [3][]string{nil, nil, nil}, errors.New("unable to sync deletions: " + err.Error())
 	}
 
-	// sync folders
-	err = folderSync(remoteFolders)
-	if err != nil {
-		return [3][]string{nil, nil, nil}, errors.New("unable to sync folders: " + err.Error())
-	}
-
 	// fetch local lists
-	localEntryModMap, err := getLocalData()
+	localEntryMap, err := synccommon.GetAllEntryData()
 	if err != nil {
 		return [3][]string{nil, nil, nil}, errors.New("unable to fetch local entry data: " + err.Error())
 	}
-	var localAgeTimestampMap map[string]int64
-	localAgeTimestampMap, err = synccommon.GetEntryAges()
-	if err != nil {
-		return [3][]string{nil, nil, nil}, err
-	}
 
 	// before syncing lists, ensure the client and server clocks are synced within 45 seconds
-	var timeSynced = true
+	var timeSyncedErr error
 	timeDiff := serverTime - clientTime
 	if timeDiff < -45 || timeDiff > 45 {
-		timeSynced = false
+		timeSyncedErr = errors.New("client and server clocks are out of sync\n\nplease ensure both clocks are correct before attempting to sync again\n\na dry sync has been performed; it is strongly recommended to review it and manually update the modification times as applicable to ensure the correct version of each entry is kept\n\nif the client's clock is at fault, update the modification times of any entries pending upload, even if the correct (upload) operation is being performed on them; failure to do so could result in entries being uploaded to the server with the incorrect modification times (could result in data loss)" + back.AnsiReset)
-		fmt.Print(back.AnsiError + "Client and server clocks are out of sync.\n\nPlease ensure both clocks are correct before attempting to sync again.\n\nA dry sync output will be printed below (if any operations would have been performed). It is strongly recommended to review it and manually update the modification times as applicable to ensure the correct version of each entry is kept.\n\nIf the client's clock is at fault, update the modification times of any entries pending upload, even if the correct (upload) operation is being performed on them. Failure to do so could result in entries being uploaded to the server with the incorrect modification times (could result in data loss).\n\n" + back.AnsiReset)
 	}
 
 	// sync new and updated entries
-	var lists [3][]string
+	// if time is not synced, the time sync error and upload/download lists will be returned here
-	if returnLists {
+	lists, err := syncLists(sshClient, *sshEntryRoot, *sshIsWindows, timeSyncedErr, localEntryMap, remoteEntryMap)
-		lists, err = syncLists(sshClient, *sshEntryRoot, *sshAgeDir, *sshIsWindows, timeSynced, true, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap)
-		if err != nil {
-			return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error())
-		}
-		lists[0] = []string{} // initialize deletions list
-		for _, deletion := range deletions {
-			if !deletion.IsAgeFile {
-				lists[0] = append(lists[0], deletion.VanityPath)
-			}
-		}
-		return lists, nil
-	}
-	_, err = syncLists(sshClient, *sshEntryRoot, *sshAgeDir, *sshIsWindows, timeSynced, false, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap)
 	if err != nil {
-		return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error())
+		return lists, errors.New("unable to sync entries: " + err.Error())
 	}
-	_ = sshClient.Close() // ignore error; non-critical/unlikely/not much could be done about it
+
-	return lists, nil     // dummy return for when not returning lists
+	// add deletions info to sync lists
+	lists[0] = []string{}
+	for _, deletion := range deletions {
+		if !deletion.IsAgeFile {
+			lists[0] = append(lists[0], deletion.VanityPath)
+		}
+	}
+
+	return lists, nil
 }

syncclient/oneOff.go

@@ -29,7 +29,7 @@ func ShearRemote(vanityPath string, onlyShearAgeFile bool) error {
 
 	var modifier string
 	var output []byte
-	sshClient, offlineMode, _, _, _, err := GetSSHClient()
+	sshClient, offlineMode, _, _, err := GetSSHClient()
 	if offlineMode {
 		goto end
 	}
@@ -89,7 +89,7 @@ func RenameRemote(oldVanityPath, newVanityPath string) error {
 
 	// create an SSH client
 	var output []byte
-	sshClient, offlineMode, _, _, _, err := GetSSHClient()
+	sshClient, offlineMode, _, _, err := GetSSHClient()
 	if offlineMode {
 		goto end
 	}
@@ -133,7 +133,7 @@ func AddFolderRemote(vanityPath string) error {
 
 	// create an SSH client
 	var output []byte
-	sshClient, offlineMode, _, _, _, err := GetSSHClient()
+	sshClient, offlineMode, _, _, err := GetSSHClient()
 	if offlineMode {
 		goto end
 	}
@@ -166,8 +166,8 @@ end:
 // Device IDs are only needed for online synchronization.
 // Device IDs are guaranteed unique as the current UNIX time is appended to them.
 // Leave prefix empty to use the current hostname as the prefix.
-// Returns: the remote EntryRoot, the remote AgeDir, and OS type indicator.
+// Returns: the remote EntryRoot and OS type indicator.
-func GenDeviceID(oldDeviceID, prefix string) (string, string, bool, error) {
+func GenDeviceID(oldDeviceID, prefix string) (string, bool, error) {
 	// generate new device ID
 	if prefix == "" {
 		prefix, _ = os.Hostname()
@@ -179,7 +179,7 @@ func GenDeviceID(oldDeviceID, prefix string) (string, string, bool, error) {
 	oldDeviceIDPath := global.CfgDir + global.PathSeparator + "devices" + global.PathSeparator + oldDeviceID
 	f, err := os.OpenFile(newDeviceIDPath, os.O_CREATE|os.O_WRONLY, 0600)
 	if err != nil {
-		return "", "", false, errors.New("unable to create local device ID file: " + err.Error())
+		return "", false, errors.New("unable to create local device ID file: " + err.Error())
 	}
 	_ = f.Close() // error ignored; if the file could be created, it can probably be closed
 
@@ -198,25 +198,25 @@ func GenDeviceID(oldDeviceID, prefix string) (string, string, bool, error) {
 	// register new device ID with server and fetch remote EntryRoot and OS type
 	// also removes the old device ID file (remotely)
 	// if registration fails, remove the new device ID file locally and return before removing the old one
-	sshClient, _, _, _, _, err := GetSSHClient()
+	sshClient, _, _, _, err := GetSSHClient()
 	if err != nil {
 		cleanupOnFail()
-		return "", "", false, errors.New("unable to connect to SSH client: " + err.Error())
+		return "", false, errors.New("unable to connect to SSH client: " + err.Error())
 	}
 	output, err := GetSSHOutput(sshClient, "libmuttonserver register", newDeviceID+"\n"+oldDeviceID)
 	if err != nil {
 		cleanupOnFail()
-		return "", "", false, errors.New("unable to register device ID with server: " + err.Error())
+		return "", false, errors.New("unable to register device ID with server: " + err.Error())
 	}
 	var registerResp synccommon.RegisterResp
 	err = json.Unmarshal(output, &registerResp)
 	if err != nil {
 		cleanupOnFail()
-		return "", "", false, errors.New("unable to unmarshal server register response: " + err.Error())
+		return "", false, errors.New("unable to unmarshal server register response: " + err.Error())
 	}
 	if registerResp.ErrMsg != nil {
 		cleanupOnFail()
-		return "", "", false, errors.New("unable to complete register; server-side error occurred: " + strings.ReplaceAll(*registerResp.ErrMsg, global.FSSpace, "\n"))
+		return "", false, errors.New("unable to complete register; server-side error occurred: " + strings.ReplaceAll(*registerResp.ErrMsg, global.FSSpace, "\n"))
 	}
 	_ = sshClient.Close() // ignore error; non-critical/unlikely/not much could be done about it
 
@@ -224,8 +224,8 @@ func GenDeviceID(oldDeviceID, prefix string) (string, string, bool, error) {
 	err = os.RemoveAll(oldDeviceIDPath)
 	if err != nil {
 		cleanupOnFail()
-		return "", "", false, errors.New("unable to remove old device ID file (locally): " + err.Error())
+		return "", false, errors.New("unable to remove old device ID file (locally): " + err.Error())
 	}
 
-	return registerResp.EntryRoot, registerResp.AgeDir, registerResp.IsWindows, nil
+	return registerResp.EntryRoot, registerResp.IsWindows, nil
 }

synccommon/common.go

@@ -18,38 +18,57 @@ const (
 
 // FetchResp defines the structure of responses from `libmuttonserver fetch`.
 type FetchResp struct {
-	ErrMsg           *string            `json:"errMsg"` // nil if no error occurred
+	ErrMsg     *string    `json:"errMsg"` // nil if no error occurred
-	ServerTime       int64              `json:"serverTime"`
+	ServerTime int64      `json:"serverTime"`
-	Deletions        []Deletion         `json:"deletions"`
+	Deletions  []Deletion `json:"deletions"`
-	FoldersToEntries map[string][]Entry `json:"folders"`
+	Entries    EntriesMap `json:"entries"`
 }
 type Deletion struct {
 	VanityPath string `json:"vanityPath"`
 	IsAgeFile  bool   `json:"isAgeFile"`
 }
+type EntriesMap map[string]Entry // map vanity paths to containing folders and mod+age timestamps
 type Entry struct {
-	VanityPath   string `json:"vanityPath"`
+	ContainingFolder string `json:"containingFolder"`
-	ModTime      int64  `json:"modTime"`
+	ModTime          int64  `json:"modTime"`
-	AgeTimestamp *int64 `json:"ageTimestamp"` // nil if no age file is present (non-password entry)
+	AgeTimestamp     *int64 `json:"ageTimestamp"` // nil if no age file is present (non-password entry)
 }
 
 // RegisterResp defines the structure of responses from `libmuttonserver register`
 type RegisterResp struct {
 	ErrMsg    *string `json:"errMsg"` // nil if no error occurred
 	EntryRoot string  `json:"entryRoot"`
-	AgeDir    string  `json:"ageDir"`
 	IsWindows bool    `json:"isWindows"`
 }
 
-// GetModTimes returns a list of all entry modification times.
+// GetAllEntryData returns a map of all vanity paths to
-func GetModTimes(entryList []string) []int64 {
+// their respective containing folders and mod+age timestamps.
-	var modList []int64
+func GetAllEntryData() (EntriesMap, error) {
-	for _, file := range entryList {
+	var err error
-		modTime, _ := os.Stat(global.GetRealPath(file))
+	entryList, _, err := WalkEntryDir()
-		modList = append(modList, modTime.ModTime().Unix())
+	if err != nil {
+		return nil, errors.New("unable to walk entry directory: " + err.Error())
 	}
-
+	// initialize vanityPath keys in map
-	return modList
+	outputEntries := make(EntriesMap)
+	var modInfo, ageInfo os.FileInfo
+	for _, vanityPath := range entryList {
+		containingFolder := vanityPath[:strings.LastIndex(vanityPath, "/")]
+		modInfo, err = os.Stat(global.GetRealPath(vanityPath))
+		if err != nil {
+			return nil, errors.New("unable to read mod time for " + vanityPath + ": " + err.Error())
+		}
+		ageInfo, err = os.Stat(global.GetRealAgePath(vanityPath))
+		var ageTimestamp *int64
+		if err == nil {
+			ageTime := ageInfo.ModTime().Unix()
+			ageTimestamp = &ageTime
+		} else if !os.IsNotExist(err) {
+			return nil, errors.New("unable to read age time for " + vanityPath + ": " + err.Error())
+		}
+		outputEntries[vanityPath] = Entry{ContainingFolder: containingFolder, ModTime: modInfo.ModTime().Unix(), AgeTimestamp: ageTimestamp}
+	}
+	return outputEntries, nil
 }
 
 // ShearLocal removes the target file or directory from the local system.
@@ -132,29 +151,6 @@ func ShearAgeFileLocal(vanityPath string) error {
 	return nil
 }
 
-// GetEntryAges reads the age directory and returns a
-// map of vanity paths to their corresponding age timestamps.
-func GetEntryAges() (map[string]int64, error) {
-	contents, err := os.ReadDir(global.AgeDir)
-	if err != nil {
-		return nil, errors.New("unable to read age directory contents: " + err.Error())
-	}
-
-	var vanityPathsToTimestamps = make(map[string]int64)
-	for _, dirEntry := range contents {
-		if !dirEntry.IsDir() {
-			vanityPath := strings.ReplaceAll(dirEntry.Name(), global.FSPath, "/")
-			info, err := dirEntry.Info()
-			if err != nil {
-				return nil, errors.New("unable to read age file modtime for " + vanityPath + ": " + err.Error())
-			}
-			vanityPathsToTimestamps[vanityPath] = info.ModTime().Unix()
-		}
-	}
-
-	return vanityPathsToTimestamps, nil
-}
-
 // RenameLocal renames oldLocationIncomplete to newLocationIncomplete on the local system.
 // This function should only be used directly by the server binary.
 func RenameLocal(oldVanityPath, newVanityPath string) error {

syncserver/server.go

@@ -12,50 +12,21 @@ import (
 )
 
 // GetRemoteDataFromServer prints to stdout the remote entries, mod times, folders, and deletions.
-// Lists in output are separated by FSSpace.
 // Output is meant to be captured over SSH for interpretation by the client.
 func GetRemoteDataFromServer(clientDeviceID string) {
-	entryList, dirList, err := synccommon.WalkEntryDir()
+	// collect info
-	if err != nil {
+	entryMap, err := synccommon.GetAllEntryData()
-		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
-		return
-	}
-	modList := synccommon.GetModTimes(entryList)
 	deletionsList, err := os.ReadDir(global.CfgDir + global.PathSeparator + "deletions")
 	if err != nil {
 		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
 		return
 	}
-	vanityPathsToTimestamps, err := synccommon.GetEntryAges()
+
-	if err != nil {
+	// form response
-		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
-		return
-	}
 	var fetchResp synccommon.FetchResp
-
+	//// server time
-	// server time
 	fetchResp.ServerTime = time.Now().Unix()
-
+	//// deletions
-	// folders (initialize keys in map)
-	fetchResp.FoldersToEntries = make(map[string][]synccommon.Entry)
-	for _, folder := range dirList {
-		if _, exists := fetchResp.FoldersToEntries[folder]; !exists {
-			fetchResp.FoldersToEntries[folder] = []synccommon.Entry{}
-		}
-	}
-
-	// entries
-	var folder string
-	for i := range entryList {
-		var ageTimestamp *int64
-		if timestamp, exists := vanityPathsToTimestamps[entryList[i]]; exists {
-			ageTimestamp = &timestamp
-		}
-		folder = entryList[i][:strings.LastIndex(entryList[i], "/")]
-		fetchResp.FoldersToEntries[folder] = append(fetchResp.FoldersToEntries[folder], synccommon.Entry{VanityPath: entryList[i], ModTime: modList[i], AgeTimestamp: ageTimestamp})
-	}
-
-	// deletions
 	for _, deletion := range deletionsList {
 		// perform deletion if it is relevant to the current client device
 		affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace)
@@ -74,6 +45,8 @@ func GetRemoteDataFromServer(clientDeviceID string) {
 			}
 		}
 	}
+	//// entries
+	fetchResp.Entries = entryMap
 
 	// marshal and print response to send to client
 	fetchRespBytes, err := json.Marshal(fetchResp)