Sync (fetch) refactor; return info from server using new JSON type

global/1globals.go

@@ -4,7 +4,7 @@ type ByteInputFetcher func(prompt string) []byte
 
 var (
 	GetPassword ByteInputFetcher // Clients should set this to a function that fetches hidden input from the user
-	RootLength  = len(EntryRoot) // length of global.EntryRoot string
+	RootLength  = len(EntryRoot)
 )
 
 const (

libmuttonserver.go

@@ -43,38 +43,70 @@ func main() {
 		// stdin[0] is evaluated after fallthrough
 		// stdin[1] is expected to be the OLD vanityPath with FSPath representing path separators - Always pass in UNIX format
 		// stdin[2] is expected to be the NEW vanityPath with FSPath representing path separators - Always pass in UNIX format
-		_ = synccommon.RenameLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), strings.ReplaceAll(stdin[2], global.FSPath, "/"))
+		err := synccommon.RenameLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), strings.ReplaceAll(stdin[2], global.FSPath, "/"))
+		if err != nil {
+			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+			return
+		}
 		fallthrough // fallthrough to add the old entry to the deletions directory
 	case "shear":
 		// shear an entry from the server and add it to the deletions directory
 		// stdin[0] is expected to be the device ID
 		// stdin[1] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format
-		_, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], false)
+		_, _, err := synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], false)
+		if err != nil {
+			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+			return
+		}
 	case "shear-age":
 		// shear ONLY the age file associated with an entry from the server and add it to the deletions directory
 		// stdin[0] is expected to be the device ID
 		// stdin[1] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format
-		_, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], true)
+		_, _, err := synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], true)
+		if err != nil {
+			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+			return
+		}
 	case "addfolder":
 		// add a new folder to the server
 		// stdin[0] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format
-		_ = synccommon.AddFolderLocal(strings.ReplaceAll(stdin[0], global.FSPath, "/"))
+		err := synccommon.AddFolderLocal(strings.ReplaceAll(stdin[0], global.FSPath, "/"))
+		if err != nil {
+			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+			return
+		}
 	case "register":
 		// register a new device ID
 		// stdin[0] is expected to be the device ID
 		// stdin[1] is expected to be the old device ID (for removal)
-		f, _ := os.OpenFile(global.ConfigDir+global.PathSeparator+"devices"+global.PathSeparator+stdin[0], os.O_CREATE|os.O_WRONLY, 0600) // errors ignored; failure unlikely to occur if init was successful; "register" is not a user-facing argument and thus the error would not be visible
+		f, err := os.OpenFile(global.ConfigDir+global.PathSeparator+"devices"+global.PathSeparator+stdin[0], os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+			return
+		}
 		_ = f.Close()
 		if stdin[1] != global.FSMisc { // FSMisc is used to indicate that no device ID is being replaced
 			// remove the old device ID file
-			_ = os.RemoveAll(global.ConfigDir + global.PathSeparator + "devices" + global.PathSeparator + stdin[1])
+			err = os.RemoveAll(global.ConfigDir + global.PathSeparator + "devices" + global.PathSeparator + stdin[1])
+			if err != nil {
+				fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+				return
+			}
 			// carry over deletions from the old device ID to the new one
 			deletionsDirRoot := global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator
-			deletionsList, _ := os.ReadDir(deletionsDirRoot)
+			deletionsList, err := os.ReadDir(deletionsDirRoot)
+			if err != nil {
+				fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+				return
+			}
 			for _, deletion := range deletionsList {
 				affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace)
 				if affectedIDVanityPath[0] == stdin[1] {
-					_ = os.Rename(deletionsDirRoot+deletion.Name(), deletionsDirRoot+stdin[0]+global.FSSpace+affectedIDVanityPath[1])
+					err = os.Rename(deletionsDirRoot+deletion.Name(), deletionsDirRoot+stdin[0]+global.FSSpace+affectedIDVanityPath[1])
+					if err != nil {
+						fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+						return
+					}
 				}
 			}
 		}

syncclient/client.go

@@ -1,6 +1,7 @@
 package syncclient
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -106,11 +107,11 @@ func GetSSHClient() (*ssh.Client, bool, bool, string, string, error) {
 }
 
 // GetSSHOutput runs a command over SSH and returns the output as a string.
-func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) (string, error) {
+func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) ([]byte, error) {
 	// create a session
 	sshSession, err := sshClient.NewSession()
 	if err != nil {
-		return "", errors.New("unable to establish SSH session: " + err.Error())
+		return nil, errors.New("unable to establish SSH session: " + err.Error())
 	}
 
 	// provide stdin data for session
@@ -120,22 +121,19 @@ func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) (string, error) {
 	var output []byte
 	output, err = sshSession.CombinedOutput(cmd)
 	if err != nil {
-		return "", errors.New("unable to run SSH command: " + err.Error())
+		return nil, errors.New("unable to run SSH command: " + err.Error())
 	}
 
-	// convert the output to a string and remove leading/trailing whitespace
+	return output, nil
-	outputString := string(output)
-	outputString = strings.TrimSpace(outputString)
-
-	return outputString, nil
 }
 
 // getRemoteDataFromClient returns:
 // a map of remote entries to their modification times,
-// a list of remote age files to their timestamps,
+// a map of remote entries to their timestamps,
-// a list of remote folders, a list of queued deletions,
+// a list of remote folders,
+// a list of queued deletions,
 // and the current server&client times as UNIX timestamps.
-func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[string]int64, []string, []string, int64, int64, error) {
+func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[string]int64, []string, []synccommon.Deletion, int64, int64, error) {
 	// get remote output over SSH
 	deviceIDList, err := global.GenDeviceIDList()
 	if err != nil {
@@ -150,57 +148,30 @@ func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[strin
 		return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command: " + err.Error())
 	}
 
-	// split output into slice based on occurrences of FSSpace
+	var fetchResp synccommon.FetchResp
-	outputSlice := strings.Split(output, global.FSSpace)
+	err = json.Unmarshal(output, &fetchResp)
-
-	// parse output/re-form lists
-	if len(outputSlice) != 7 { // ensure information from server is complete
-		return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command; server returned an unexpected response")
-	}
-	serverTime, err := strconv.ParseInt(outputSlice[0], 10, 64)
 	if err != nil {
-		return nil, nil, nil, nil, 0, 0, errors.New("unable to parse server time: " + err.Error())
+		fmt.Println(string(output))
+		return nil, nil, nil, nil, 0, 0, errors.New("unable to unmarshal server fetch response: " + err.Error())
 	}
-	entries := strings.Split(outputSlice[1], global.FSMisc)[1:]
+	if fetchResp.ErrMsg != nil {
-	modsStrings := strings.Split(outputSlice[2], global.FSMisc)[1:]
+		return nil, nil, nil, nil, 0, 0, errors.New("unable to complete fetch; server-side error occurred: " + *fetchResp.ErrMsg)
-	ageFiles := strings.Split(outputSlice[3], global.FSMisc)[1:]
-	ageFilesTimestampStrings := strings.Split(outputSlice[4], global.FSMisc)[1:]
-	folders := strings.Split(outputSlice[5], global.FSMisc)[1:]
-	deletions := strings.Split(outputSlice[6], global.FSMisc)[1:]
-
-	// convert the mod times+age timestamps to int64
-	var mods []int64
-	var mod int64
-	for _, modString := range modsStrings {
-		mod, err = strconv.ParseInt(modString, 10, 64)
-		if err != nil {
-			return nil, nil, nil, nil, 0, 0, errors.New("unable to parse mod time: " + err.Error())
-		}
-		mods = append(mods, mod)
-	}
-	var timestamps []int64
-	var timestamp int64
-	for _, ageFilesTimestampString := range ageFilesTimestampStrings {
-		timestamp, err = strconv.ParseInt(ageFilesTimestampString, 10, 64)
-		if err != nil {
-			return nil, nil, nil, nil, 0, 0, errors.New("unable to parse age timestamp: " + err.Error())
-		}
-		timestamps = append(timestamps, timestamp)
 	}
 
-	// map remote entries to their modification times
 	entryModMap := make(map[string]int64)
-	for i, entry := range entries {
-		entryModMap[entry] = mods[i]
-	}
-
-	// map remote age files to their timestamps
 	ageTimestampMap := make(map[string]int64)
-	for i, ageFile := range ageFiles {
+	var folders []string
-		ageTimestampMap[ageFile] = timestamps[i]
+	for folderName, containedEntries := range fetchResp.FoldersToEntries {
+		folders = append(folders, folderName)
+		for _, entry := range containedEntries {
+			entryModMap[entry.VanityPath] = entry.ModTime
+			if entry.AgeTimestamp != nil {
+				ageTimestampMap[entry.VanityPath] = *entry.AgeTimestamp
+			}
+		}
 	}
 
-	return entryModMap, ageTimestampMap, folders, deletions, serverTime, clientTime, nil
+	return entryModMap, ageTimestampMap, folders, fetchResp.Deletions, fetchResp.ServerTime, clientTime, nil
 }
 
 // getLocalData returns a map of local entries to their modification times.
@@ -478,20 +449,19 @@ func syncLists(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindo
 }
 
 // deletionSync removes entries from the client that have been deleted on the server (multi-client deletion).
-func deletionSync(deletions []string) error {
+func deletionSync(deletions []synccommon.Deletion) error {
 	var entryDeleted bool
 	for _, deletion := range deletions {
-		deletionSplit := strings.Split(deletion, global.FSSpace)
+		if !deletion.IsAgeFile {
-		if deletionSplit[0] == "entry" {
 			entryDeleted = true // set a flag to indicate that at least one entry has been deleted (used to determine whether to print a gap between deletion and other messages)
-			fmt.Println(synccommon.AnsiDelete+deletionSplit[1]+back.AnsiReset, "has been sheared, removing locally (if it exists)")
+			fmt.Println(synccommon.AnsiDelete+deletion.VanityPath+back.AnsiReset, "has been sheared, removing locally (if it exists)")
 		}
-		err := os.RemoveAll(global.GetRealPath(deletionSplit[1]))
+		err := os.RemoveAll(global.GetRealPath(deletion.VanityPath))
 		if err != nil {
-			if deletionSplit[0] == "entry" {
+			if !deletion.IsAgeFile {
-				return errors.New("unable to shear " + deletionSplit[1] + " locally: " + err.Error())
+				return errors.New("unable to shear " + deletion.VanityPath + " locally: " + err.Error())
 			}
-			return errors.New("unable to shear age file for " + deletionSplit[1] + " locally: " + err.Error())
+			return errors.New("unable to shear age file for " + deletion.VanityPath + " locally: " + err.Error())
 		}
 	}
 	if entryDeleted {
@@ -580,7 +550,11 @@ func RunJob(returnLists bool) ([3][]string, error) {
 		if err != nil {
 			return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error())
 		}
-		lists[0] = deletions
+		for _, deletion := range deletions {
+			if !deletion.IsAgeFile {
+				lists[0] = append(lists[0], deletion.VanityPath)
+			}
+		}
 		return lists, nil
 	}
 	_, err = syncLists(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, timeSynced, false, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap)

synccommon/common.go

@@ -16,6 +16,25 @@ const (
 	AnsiDelete = "\033[38;5;1m"
 )
 
+// FetchResponse defines the structure of responses from `libmuttonserver fetch`.
+type FetchResp struct {
+	ErrMsg           *string            `json:"errMsg"` // nil if no error occurred
+	ServerTime       int64              `json:"serverTime"`
+	Deletions        []Deletion         `json:"deletions"`
+	FoldersToEntries map[string][]Entry `json:"folders"`
+}
+
+type Deletion struct {
+	VanityPath string `json:"vanityPath"`
+	IsAgeFile  bool   `json:"isAgeFile"`
+}
+
+type Entry struct {
+	VanityPath   string `json:"vanityPath"`
+	ModTime      int64  `json:"modTime"`
+	AgeTimestamp *int64 `json:"ageTimestamp"` // nil if no age file is present (non-password entry)
+}
+
 // GetModTimes returns a list of all entry modification times.
 func GetModTimes(entryList []string) []int64 {
 	var modList []int64

synccycles/init.go

@@ -59,7 +59,7 @@ func DeviceIDGen(oldDeviceID, prefix string) (string, string, string, error) {
 		cleanupOnFail()
 		return "", "", "", errors.New("unable to register device ID with server: " + err.Error())
 	}
-	sshEntryRootSSHAgeDirSSHIsWindows := strings.Split(output, global.FSSpace)
+	sshEntryRootSSHAgeDirSSHIsWindows := strings.Split(string(output), global.FSSpace)
 	_ = sshClient.Close() // ignore error; non-critical/unlikely/not much could be done about it
 
 	// remove old device ID file (locally; may not exist)

syncserver/server.go

@@ -1,6 +1,7 @@
 package syncserver
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
@@ -14,63 +15,71 @@ import (
 // Lists in output are separated by FSSpace.
 // Output is meant to be captured over SSH for interpretation by the client.
 func GetRemoteDataFromServer(clientDeviceID string) {
-	entryList, dirList, _ := synccommon.WalkEntryDir()
+	entryList, dirList, err := synccommon.WalkEntryDir()
+	if err != nil {
+		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+		return
+	}
 	modList := synccommon.GetModTimes(entryList)
-	deletionsList, _ := os.ReadDir(global.ConfigDir + global.PathSeparator + "deletions")
+	deletionsList, err := os.ReadDir(global.ConfigDir + global.PathSeparator + "deletions")
-	vanityPathsToTimestamps, _ := synccommon.GetEntryAges()
+	if err != nil {
-	var ageVanityPaths []string
+		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
-	var ageTimestamps []int64
+		return
-	for vanityPath, timestamp := range vanityPathsToTimestamps {
+	}
-		ageVanityPaths = append(ageVanityPaths, vanityPath)
+	vanityPathsToTimestamps, err := synccommon.GetEntryAges()
-		ageTimestamps = append(ageTimestamps, timestamp)
+	if err != nil {
+		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+		return
+	}
+	var fetchResp synccommon.FetchResp
+
+	// server time
+	fetchResp.ServerTime = time.Now().Unix()
+
+	// folders (initialize keys in map)
+	fetchResp.FoldersToEntries = make(map[string][]synccommon.Entry)
+	for _, folder := range dirList {
+		if _, exists := fetchResp.FoldersToEntries[folder]; !exists {
+			fetchResp.FoldersToEntries[folder] = []synccommon.Entry{}
+		}
 	}
 
-	// print the current UNIX timestamp to stdout
+	// entries
-	fmt.Print(time.Now().Unix())
+	var folder string
-
+	for i := range entryList {
-	// print the lists to stdout
+		var ageTimestamp *int64
-	// entry list
+		if timestamp, exists := vanityPathsToTimestamps[entryList[i]]; exists {
-	fmt.Print(global.FSSpace)
+			ageTimestamp = &timestamp
-	for _, entry := range entryList {
+		}
-		fmt.Print(global.FSMisc + entry)
+		folder = entryList[i][:strings.LastIndex(entryList[i], "/")]
+		fetchResp.FoldersToEntries[folder] = append(fetchResp.FoldersToEntries[folder], synccommon.Entry{VanityPath: entryList[i], ModTime: modList[i], AgeTimestamp: ageTimestamp})
 	}
 
-	// modification time list
+	// deletions
-	fmt.Print(global.FSSpace)
-	for _, mod := range modList {
-		fmt.Print(global.FSMisc)
-		fmt.Print(mod)
-	}
-
-	// age file list
-	fmt.Print(global.FSSpace)
-	for _, file := range ageVanityPaths {
-		fmt.Print(global.FSMisc + file)
-	}
-
-	// age file timestamp list
-	fmt.Print(global.FSSpace)
-	for _, timestamp := range ageTimestamps {
-		fmt.Print(global.FSMisc)
-		fmt.Print(timestamp)
-	}
-
-	// directory/folder list
-	fmt.Print(global.FSSpace)
-	for _, dir := range dirList {
-		fmt.Print(global.FSMisc + dir)
-	}
-
-	// deletions list
-	fmt.Print(global.FSSpace)
 	for _, deletion := range deletionsList {
 		// perform deletion if it is relevant to the current client device
 		affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace)
 		if affectedIDVanityPath[0] == clientDeviceID {
-			fmt.Print(global.FSMisc + strings.ReplaceAll(affectedIDVanityPath[1]+global.FSSpace+affectedIDVanityPath[2], global.FSPath, "/"))
+			var isAgeFile bool
+			if affectedIDVanityPath[1] == "age" {
+				isAgeFile = true
+			}
+			fetchResp.Deletions = append(fetchResp.Deletions, synccommon.Deletion{VanityPath: strings.ReplaceAll(affectedIDVanityPath[2], global.FSPath, "/"), IsAgeFile: isAgeFile})
 
 			// assume successful client deletion and remove deletions file (if assumption is somehow false, worst case scenario is that the client will re-upload the deleted entry)
-			_ = os.RemoveAll(global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator + deletion.Name()) // error ignored; function not run from a user-facing argument and thus the error would not be visible
+			err = os.RemoveAll(global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator + deletion.Name()) // error ignored; function not run from a user-facing argument and thus the error would not be visible
+			if err != nil {
+				fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+				return
+			}
 		}
 	}
+
+	// marshal and print response to send to client
+	fetchRespBytes, err := json.Marshal(fetchResp)
+	if err != nil {
+		fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())
+		return
+	}
+	fmt.Print(string(fetchRespBytes))
 }