From e355c810da06d794fd5a6ed10b8a886afa7f3492 Mon Sep 17 00:00:00 2001 From: Randall Winkhart Date: Tue, 9 Dec 2025 23:17:59 -0500 Subject: [PATCH] Sync (fetch) refactor; return info from server using new JSON type --- global/1globals.go | 2 +- libmuttonserver.go | 48 ++++++++++++++++---- syncclient/client.go | 100 ++++++++++++++++-------------------------- synccommon/common.go | 19 ++++++++ synccycles/init.go | 2 +- syncserver/server.go | 101 +++++++++++++++++++++++-------------------- 6 files changed, 153 insertions(+), 119 deletions(-) diff --git a/global/1globals.go b/global/1globals.go index ca28a9c..d3fcca5 100644 --- a/global/1globals.go +++ b/global/1globals.go @@ -4,7 +4,7 @@ type ByteInputFetcher func(prompt string) []byte var ( GetPassword ByteInputFetcher // Clients should set this to a function that fetches hidden input from the user - RootLength = len(EntryRoot) // length of global.EntryRoot string + RootLength = len(EntryRoot) ) const ( diff --git a/libmuttonserver.go b/libmuttonserver.go index ae7efa3..74b1f16 100644 --- a/libmuttonserver.go +++ b/libmuttonserver.go @@ -43,38 +43,70 @@ func main() { // stdin[0] is evaluated after fallthrough // stdin[1] is expected to be the OLD vanityPath with FSPath representing path separators - Always pass in UNIX format // stdin[2] is expected to be the NEW vanityPath with FSPath representing path separators - Always pass in UNIX format - _ = synccommon.RenameLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), strings.ReplaceAll(stdin[2], global.FSPath, "/")) + err := synccommon.RenameLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), strings.ReplaceAll(stdin[2], global.FSPath, "/")) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } fallthrough // fallthrough to add the old entry to the deletions directory case "shear": // shear an entry from the server and add it to the deletions directory // stdin[0] is expected to be the device ID // stdin[1] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format - _, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], false) + _, _, err := synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], false) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } case "shear-age": // shear ONLY the age file associated with an entry from the server and add it to the deletions directory // stdin[0] is expected to be the device ID // stdin[1] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format - _, _, _ = synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], true) + _, _, err := synccommon.ShearLocal(strings.ReplaceAll(stdin[1], global.FSPath, "/"), stdin[0], true) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } case "addfolder": // add a new folder to the server // stdin[0] is expected to be the vanityPath with FSPath representing path separators - Always pass in UNIX format - _ = synccommon.AddFolderLocal(strings.ReplaceAll(stdin[0], global.FSPath, "/")) + err := synccommon.AddFolderLocal(strings.ReplaceAll(stdin[0], global.FSPath, "/")) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } case "register": // register a new device ID // stdin[0] is expected to be the device ID // stdin[1] is expected to be the old device ID (for removal) - f, _ := os.OpenFile(global.ConfigDir+global.PathSeparator+"devices"+global.PathSeparator+stdin[0], os.O_CREATE|os.O_WRONLY, 0600) // errors ignored; failure unlikely to occur if init was successful; "register" is not a user-facing argument and thus the error would not be visible + f, err := os.OpenFile(global.ConfigDir+global.PathSeparator+"devices"+global.PathSeparator+stdin[0], os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } _ = f.Close() if stdin[1] != global.FSMisc { // FSMisc is used to indicate that no device ID is being replaced // remove the old device ID file - _ = os.RemoveAll(global.ConfigDir + global.PathSeparator + "devices" + global.PathSeparator + stdin[1]) + err = os.RemoveAll(global.ConfigDir + global.PathSeparator + "devices" + global.PathSeparator + stdin[1]) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } // carry over deletions from the old device ID to the new one deletionsDirRoot := global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator - deletionsList, _ := os.ReadDir(deletionsDirRoot) + deletionsList, err := os.ReadDir(deletionsDirRoot) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } for _, deletion := range deletionsList { affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace) if affectedIDVanityPath[0] == stdin[1] { - _ = os.Rename(deletionsDirRoot+deletion.Name(), deletionsDirRoot+stdin[0]+global.FSSpace+affectedIDVanityPath[1]) + err = os.Rename(deletionsDirRoot+deletion.Name(), deletionsDirRoot+stdin[0]+global.FSSpace+affectedIDVanityPath[1]) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } } } } diff --git a/syncclient/client.go b/syncclient/client.go index 8a08697..c5146cf 100644 --- a/syncclient/client.go +++ b/syncclient/client.go @@ -1,6 +1,7 @@ package syncclient import ( + "encoding/json" "errors" "fmt" "os" @@ -106,11 +107,11 @@ func GetSSHClient() (*ssh.Client, bool, bool, string, string, error) { } // GetSSHOutput runs a command over SSH and returns the output as a string. -func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) (string, error) { +func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) ([]byte, error) { // create a session sshSession, err := sshClient.NewSession() if err != nil { - return "", errors.New("unable to establish SSH session: " + err.Error()) + return nil, errors.New("unable to establish SSH session: " + err.Error()) } // provide stdin data for session @@ -120,22 +121,19 @@ func GetSSHOutput(sshClient *ssh.Client, cmd, stdin string) (string, error) { var output []byte output, err = sshSession.CombinedOutput(cmd) if err != nil { - return "", errors.New("unable to run SSH command: " + err.Error()) + return nil, errors.New("unable to run SSH command: " + err.Error()) } - // convert the output to a string and remove leading/trailing whitespace - outputString := string(output) - outputString = strings.TrimSpace(outputString) - - return outputString, nil + return output, nil } // getRemoteDataFromClient returns: // a map of remote entries to their modification times, -// a list of remote age files to their timestamps, -// a list of remote folders, a list of queued deletions, +// a map of remote entries to their timestamps, +// a list of remote folders, +// a list of queued deletions, // and the current server&client times as UNIX timestamps. -func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[string]int64, []string, []string, int64, int64, error) { +func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[string]int64, []string, []synccommon.Deletion, int64, int64, error) { // get remote output over SSH deviceIDList, err := global.GenDeviceIDList() if err != nil { @@ -150,57 +148,30 @@ func getRemoteDataFromClient(sshClient *ssh.Client) (map[string]int64, map[strin return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command: " + err.Error()) } - // split output into slice based on occurrences of FSSpace - outputSlice := strings.Split(output, global.FSSpace) - - // parse output/re-form lists - if len(outputSlice) != 7 { // ensure information from server is complete - return nil, nil, nil, nil, 0, 0, errors.New("unable to run remote command; server returned an unexpected response") - } - serverTime, err := strconv.ParseInt(outputSlice[0], 10, 64) + var fetchResp synccommon.FetchResp + err = json.Unmarshal(output, &fetchResp) if err != nil { - return nil, nil, nil, nil, 0, 0, errors.New("unable to parse server time: " + err.Error()) + fmt.Println(string(output)) + return nil, nil, nil, nil, 0, 0, errors.New("unable to unmarshal server fetch response: " + err.Error()) } - entries := strings.Split(outputSlice[1], global.FSMisc)[1:] - modsStrings := strings.Split(outputSlice[2], global.FSMisc)[1:] - ageFiles := strings.Split(outputSlice[3], global.FSMisc)[1:] - ageFilesTimestampStrings := strings.Split(outputSlice[4], global.FSMisc)[1:] - folders := strings.Split(outputSlice[5], global.FSMisc)[1:] - deletions := strings.Split(outputSlice[6], global.FSMisc)[1:] - - // convert the mod times+age timestamps to int64 - var mods []int64 - var mod int64 - for _, modString := range modsStrings { - mod, err = strconv.ParseInt(modString, 10, 64) - if err != nil { - return nil, nil, nil, nil, 0, 0, errors.New("unable to parse mod time: " + err.Error()) - } - mods = append(mods, mod) - } - var timestamps []int64 - var timestamp int64 - for _, ageFilesTimestampString := range ageFilesTimestampStrings { - timestamp, err = strconv.ParseInt(ageFilesTimestampString, 10, 64) - if err != nil { - return nil, nil, nil, nil, 0, 0, errors.New("unable to parse age timestamp: " + err.Error()) - } - timestamps = append(timestamps, timestamp) + if fetchResp.ErrMsg != nil { + return nil, nil, nil, nil, 0, 0, errors.New("unable to complete fetch; server-side error occurred: " + *fetchResp.ErrMsg) } - // map remote entries to their modification times entryModMap := make(map[string]int64) - for i, entry := range entries { - entryModMap[entry] = mods[i] - } - - // map remote age files to their timestamps ageTimestampMap := make(map[string]int64) - for i, ageFile := range ageFiles { - ageTimestampMap[ageFile] = timestamps[i] + var folders []string + for folderName, containedEntries := range fetchResp.FoldersToEntries { + folders = append(folders, folderName) + for _, entry := range containedEntries { + entryModMap[entry.VanityPath] = entry.ModTime + if entry.AgeTimestamp != nil { + ageTimestampMap[entry.VanityPath] = *entry.AgeTimestamp + } + } } - return entryModMap, ageTimestampMap, folders, deletions, serverTime, clientTime, nil + return entryModMap, ageTimestampMap, folders, fetchResp.Deletions, fetchResp.ServerTime, clientTime, nil } // getLocalData returns a map of local entries to their modification times. @@ -478,20 +449,19 @@ func syncLists(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindo } // deletionSync removes entries from the client that have been deleted on the server (multi-client deletion). -func deletionSync(deletions []string) error { +func deletionSync(deletions []synccommon.Deletion) error { var entryDeleted bool for _, deletion := range deletions { - deletionSplit := strings.Split(deletion, global.FSSpace) - if deletionSplit[0] == "entry" { + if !deletion.IsAgeFile { entryDeleted = true // set a flag to indicate that at least one entry has been deleted (used to determine whether to print a gap between deletion and other messages) - fmt.Println(synccommon.AnsiDelete+deletionSplit[1]+back.AnsiReset, "has been sheared, removing locally (if it exists)") + fmt.Println(synccommon.AnsiDelete+deletion.VanityPath+back.AnsiReset, "has been sheared, removing locally (if it exists)") } - err := os.RemoveAll(global.GetRealPath(deletionSplit[1])) + err := os.RemoveAll(global.GetRealPath(deletion.VanityPath)) if err != nil { - if deletionSplit[0] == "entry" { - return errors.New("unable to shear " + deletionSplit[1] + " locally: " + err.Error()) + if !deletion.IsAgeFile { + return errors.New("unable to shear " + deletion.VanityPath + " locally: " + err.Error()) } - return errors.New("unable to shear age file for " + deletionSplit[1] + " locally: " + err.Error()) + return errors.New("unable to shear age file for " + deletion.VanityPath + " locally: " + err.Error()) } } if entryDeleted { @@ -580,7 +550,11 @@ func RunJob(returnLists bool) ([3][]string, error) { if err != nil { return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error()) } - lists[0] = deletions + for _, deletion := range deletions { + if !deletion.IsAgeFile { + lists[0] = append(lists[0], deletion.VanityPath) + } + } return lists, nil } _, err = syncLists(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, timeSynced, false, localEntryModMap, remoteEntryModMap, localAgeTimestampMap, remoteAgeTimestampMap) diff --git a/synccommon/common.go b/synccommon/common.go index e86851b..ba496c7 100644 --- a/synccommon/common.go +++ b/synccommon/common.go @@ -16,6 +16,25 @@ const ( AnsiDelete = "\033[38;5;1m" ) +// FetchResponse defines the structure of responses from `libmuttonserver fetch`. +type FetchResp struct { + ErrMsg *string `json:"errMsg"` // nil if no error occurred + ServerTime int64 `json:"serverTime"` + Deletions []Deletion `json:"deletions"` + FoldersToEntries map[string][]Entry `json:"folders"` +} + +type Deletion struct { + VanityPath string `json:"vanityPath"` + IsAgeFile bool `json:"isAgeFile"` +} + +type Entry struct { + VanityPath string `json:"vanityPath"` + ModTime int64 `json:"modTime"` + AgeTimestamp *int64 `json:"ageTimestamp"` // nil if no age file is present (non-password entry) +} + // GetModTimes returns a list of all entry modification times. func GetModTimes(entryList []string) []int64 { var modList []int64 diff --git a/synccycles/init.go b/synccycles/init.go index e49977c..6dd8389 100644 --- a/synccycles/init.go +++ b/synccycles/init.go @@ -59,7 +59,7 @@ func DeviceIDGen(oldDeviceID, prefix string) (string, string, string, error) { cleanupOnFail() return "", "", "", errors.New("unable to register device ID with server: " + err.Error()) } - sshEntryRootSSHAgeDirSSHIsWindows := strings.Split(output, global.FSSpace) + sshEntryRootSSHAgeDirSSHIsWindows := strings.Split(string(output), global.FSSpace) _ = sshClient.Close() // ignore error; non-critical/unlikely/not much could be done about it // remove old device ID file (locally; may not exist) diff --git a/syncserver/server.go b/syncserver/server.go index 828e29e..86dbe6c 100644 --- a/syncserver/server.go +++ b/syncserver/server.go @@ -1,6 +1,7 @@ package syncserver import ( + "encoding/json" "fmt" "os" "strings" @@ -14,63 +15,71 @@ import ( // Lists in output are separated by FSSpace. // Output is meant to be captured over SSH for interpretation by the client. func GetRemoteDataFromServer(clientDeviceID string) { - entryList, dirList, _ := synccommon.WalkEntryDir() + entryList, dirList, err := synccommon.WalkEntryDir() + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } modList := synccommon.GetModTimes(entryList) - deletionsList, _ := os.ReadDir(global.ConfigDir + global.PathSeparator + "deletions") - vanityPathsToTimestamps, _ := synccommon.GetEntryAges() - var ageVanityPaths []string - var ageTimestamps []int64 - for vanityPath, timestamp := range vanityPathsToTimestamps { - ageVanityPaths = append(ageVanityPaths, vanityPath) - ageTimestamps = append(ageTimestamps, timestamp) + deletionsList, err := os.ReadDir(global.ConfigDir + global.PathSeparator + "deletions") + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } + vanityPathsToTimestamps, err := synccommon.GetEntryAges() + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } + var fetchResp synccommon.FetchResp + + // server time + fetchResp.ServerTime = time.Now().Unix() + + // folders (initialize keys in map) + fetchResp.FoldersToEntries = make(map[string][]synccommon.Entry) + for _, folder := range dirList { + if _, exists := fetchResp.FoldersToEntries[folder]; !exists { + fetchResp.FoldersToEntries[folder] = []synccommon.Entry{} + } } - // print the current UNIX timestamp to stdout - fmt.Print(time.Now().Unix()) - - // print the lists to stdout - // entry list - fmt.Print(global.FSSpace) - for _, entry := range entryList { - fmt.Print(global.FSMisc + entry) + // entries + var folder string + for i := range entryList { + var ageTimestamp *int64 + if timestamp, exists := vanityPathsToTimestamps[entryList[i]]; exists { + ageTimestamp = ×tamp + } + folder = entryList[i][:strings.LastIndex(entryList[i], "/")] + fetchResp.FoldersToEntries[folder] = append(fetchResp.FoldersToEntries[folder], synccommon.Entry{VanityPath: entryList[i], ModTime: modList[i], AgeTimestamp: ageTimestamp}) } - // modification time list - fmt.Print(global.FSSpace) - for _, mod := range modList { - fmt.Print(global.FSMisc) - fmt.Print(mod) - } - - // age file list - fmt.Print(global.FSSpace) - for _, file := range ageVanityPaths { - fmt.Print(global.FSMisc + file) - } - - // age file timestamp list - fmt.Print(global.FSSpace) - for _, timestamp := range ageTimestamps { - fmt.Print(global.FSMisc) - fmt.Print(timestamp) - } - - // directory/folder list - fmt.Print(global.FSSpace) - for _, dir := range dirList { - fmt.Print(global.FSMisc + dir) - } - - // deletions list - fmt.Print(global.FSSpace) + // deletions for _, deletion := range deletionsList { // perform deletion if it is relevant to the current client device affectedIDVanityPath := strings.Split(deletion.Name(), global.FSSpace) if affectedIDVanityPath[0] == clientDeviceID { - fmt.Print(global.FSMisc + strings.ReplaceAll(affectedIDVanityPath[1]+global.FSSpace+affectedIDVanityPath[2], global.FSPath, "/")) + var isAgeFile bool + if affectedIDVanityPath[1] == "age" { + isAgeFile = true + } + fetchResp.Deletions = append(fetchResp.Deletions, synccommon.Deletion{VanityPath: strings.ReplaceAll(affectedIDVanityPath[2], global.FSPath, "/"), IsAgeFile: isAgeFile}) // assume successful client deletion and remove deletions file (if assumption is somehow false, worst case scenario is that the client will re-upload the deleted entry) - _ = os.RemoveAll(global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator + deletion.Name()) // error ignored; function not run from a user-facing argument and thus the error would not be visible + err = os.RemoveAll(global.ConfigDir + global.PathSeparator + "deletions" + global.PathSeparator + deletion.Name()) // error ignored; function not run from a user-facing argument and thus the error would not be visible + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } } } + + // marshal and print response to send to client + fetchRespBytes, err := json.Marshal(fetchResp) + if err != nil { + fmt.Printf("{\"errMsg\":\"%s\"}", err.Error()) + return + } + fmt.Print(string(fetchRespBytes)) }