Re-implement age data upload via SFTP (download will continue to use new method)

config/cfg.go

@@ -16,6 +16,7 @@ type CfgT struct {
 		SSHIP            *string `json:"sshIP"`
 		SSHPort          *string `json:"sshPort"`
 		SSHEntryRootPath *string `json:"sshEntryRootPath"`
+		SSHAgeDirPath    *string `json:"sshAgeDirPath"`
 		SSHKeyPath       *string `json:"sshKeyPath"`
 		SSHKeyProtected  *bool   `json:"sshKeyProtected"`
 		SSHIsWindows     *bool   `json:"sshIsWindows"`

core/init.go

@@ -81,12 +81,13 @@ func LibmuttonInit(inputCB func(prompt string) string, rcwPassword []byte, appen
 			return err
 		}
 		// generate and register device ID
-		sshEntryRoot, sshIsWindows, err := syncclient.GenDeviceID(oldDeviceID, deviceIDPrefix)
+		sshEntryRoot, sshAgeDir, sshIsWindows, err := syncclient.GenDeviceID(oldDeviceID, deviceIDPrefix)
 		if err != nil {
 			return errors.New("unable to generate device ID: " + err.Error())
 		}
 		// update config file
 		newCfg.Libmutton.SSHEntryRootPath = &sshEntryRoot
+		newCfg.Libmutton.SSHAgeDirPath = &sshAgeDir
 		newCfg.Libmutton.SSHIsWindows = &sshIsWindows
 		if err = config.Write(newCfg, true); err != nil {
 			return err

libmuttonserver.go

@@ -125,8 +125,8 @@ func main() {
 			}
 		}
 
-		// print EntryRoot and bool indicating OS type to stdout for client to store in config
+		// print EntryRoot, AgeDir and bool indicating OS type to stdout for client to store in config
-		registerResp := synccommon.RegisterResp{EntryRoot: global.EntryRoot, IsWindows: global.IsWindows}
+		registerResp := synccommon.RegisterResp{EntryRoot: global.EntryRoot, AgeDir: global.AgeDir, IsWindows: global.IsWindows}
 		registerRespBytes, err := json.Marshal(registerResp)
 		if err != nil {
 			fmt.Printf("{\"errMsg\":\"%s\"}", err.Error())

syncclient/client.go

@@ -25,20 +25,20 @@ import (
 // sshIsWindows (whether the remote server is running Windows),
 // sshEntryRoot (the root directory for entries on the remote server),
 // Only supports key-based authentication (passwords are supported for CLI-based implementations).
-func GetSSHClient() (*ssh.Client, bool, *bool, *string, error) {
+func GetSSHClient() (*ssh.Client, bool, *bool, *string, *string, error) {
 	// get SSH config info
 	cfg, err := config.Load()
 	if err != nil {
-		return nil, false, nil, nil, errors.New("unable to parse SSH config: " + err.Error())
+		return nil, false, nil, nil, nil, errors.New("unable to parse SSH config: " + err.Error())
 	}
 	if *cfg.Libmutton.OfflineMode {
-		return nil, true, nil, nil, nil
+		return nil, true, nil, nil, nil, nil
 	}
 
 	// read private key
 	key, err := os.ReadFile(*cfg.Libmutton.SSHKeyPath)
 	if err != nil {
-		return nil, false, nil, nil, errors.New("unable to read private key: " + *cfg.Libmutton.SSHKeyPath)
+		return nil, false, nil, nil, nil, errors.New("unable to read private key: " + *cfg.Libmutton.SSHKeyPath)
 	}
 
 	// parse private key
@@ -49,14 +49,14 @@ func GetSSHClient() (*ssh.Client, bool, *bool, *string, error) {
 		parsedKey, err = ssh.ParsePrivateKeyWithPassphrase(key, global.GetPassword("Enter password for your SSH keyfile:"))
 	}
 	if err != nil {
-		return nil, false, nil, nil, errors.New("unable to parse private key: " + *cfg.Libmutton.SSHKeyPath)
+		return nil, false, nil, nil, nil, errors.New("unable to parse private key: " + *cfg.Libmutton.SSHKeyPath)
 	}
 
 	// read known hosts file
 	var hostKeyCallback ssh.HostKeyCallback
 	hostKeyCallback, err = knownhosts.New(back.Home + global.PathSeparator + ".ssh" + global.PathSeparator + "known_hosts")
 	if err != nil {
-		return nil, false, nil, nil, errors.New("unable to read known hosts file: " + err.Error())
+		return nil, false, nil, nil, nil, errors.New("unable to read known hosts file: " + err.Error())
 	}
 
 	// configure SSH client
@@ -72,10 +72,10 @@ func GetSSHClient() (*ssh.Client, bool, *bool, *string, error) {
 	// connect to SSH server
 	sshClient, err := ssh.Dial("tcp", *cfg.Libmutton.SSHIP+":"+*cfg.Libmutton.SSHPort, sshCfg)
 	if err != nil {
-		return nil, false, nil, nil, errors.New("unable to connect to remote server: " + err.Error())
+		return nil, false, nil, nil, nil, errors.New("unable to connect to remote server: " + err.Error())
 	}
 
-	return sshClient, false, cfg.Libmutton.SSHIsWindows, cfg.Libmutton.SSHEntryRootPath, nil
+	return sshClient, false, cfg.Libmutton.SSHIsWindows, cfg.Libmutton.SSHEntryRootPath, cfg.Libmutton.SSHAgeDirPath, nil
 }
 
 // GetSSHOutput runs a command over SSH and returns the output as a string.
@@ -138,8 +138,16 @@ func getRealPathSFTP(vanityPath, serverEntryRoot string, serverIsWindows bool) s
 	return serverEntryRoot + strings.ReplaceAll(vanityPath, "/", "\\")
 }
 
+// getRealPathSFTP formats the vanityPath to match the remote server's entry/age file directory and path separator.
+func getRealAgePathSFTP(vanityPath, serverAgeDir string, serverIsWindows bool) string {
+	if !serverIsWindows {
+		return serverAgeDir + "/" + strings.ReplaceAll(vanityPath, "/", global.FSPath)
+	}
+	return serverAgeDir + "\\" + strings.ReplaceAll(vanityPath, "/", global.FSPath)
+}
+
 // sftpSync takes two slices of entries (one for downloads and one for uploads) and syncs them between the client and server using SFTP.
-func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, downloadList, uploadList []string) error {
+func sftpSync(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindows bool, downloadList, uploadList []string) error {
 	// create an SFTP client from sshClient
 	sftpClient, err := sftp.NewClient(sshClient)
 	if err != nil {
@@ -240,7 +248,12 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow
 		}
 
 		// store path to remote entry
-		remoteFileRealPath := getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows)
+		var remoteFileRealPath string
+		if isAgeFile {
+			remoteFileRealPath = getRealAgePathSFTP(vanityPath, sshAgeDir, sshIsWindows)
+		} else {
+			remoteFileRealPath = getRealPathSFTP(vanityPath, sshEntryRoot, sshIsWindows)
+		}
 
 		// create remote file
 		var remoteFile *sftp.File
@@ -279,7 +292,7 @@ func sftpSync(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, dow
 
 // syncLists determines which entries need to be downloaded and uploaded
 // for synchronization and calls sftpSync with this information.
-func syncLists(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, timeSyncedErr error, localEntryMap, remoteEntryMap synccommon.EntriesMap) ([3][]string, error) {
+func syncLists(sshClient *ssh.Client, sshEntryRoot, sshAgeDir string, sshIsWindows bool, timeSyncedErr error, localEntryMap, remoteEntryMap synccommon.EntriesMap) ([3][]string, error) {
 	// initialize slices to store entries that need to be downloaded or uploaded
 	var downloadList, uploadList []string
 
@@ -300,12 +313,18 @@ func syncLists(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, ti
 			} else if remoteInfo.ModTime < localInfo.ModTime {
 				fmt.Println(back.AnsiBlue+vanityPath+back.AnsiReset, "is newer on client, adding to upload list")
 				uploadList = append(uploadList, vanityPath)
+				if localInfo.AgeTimestamp != nil && localInfo.AgeTimestamp != remoteInfo.AgeTimestamp {
+					uploadList = append(uploadList, global.FSMisc+vanityPath)
+				}
 			}
 			// remove entry from remote map (process of elimination)
 			delete(remoteEntryMap, vanityPath)
 		} else {
 			fmt.Println(back.AnsiBlue+vanityPath+back.AnsiReset, "does not exist on server, adding to upload list")
 			uploadList = append(uploadList, vanityPath)
+			if localInfo.AgeTimestamp != nil {
+				uploadList = append(uploadList, global.FSMisc+vanityPath)
+			}
 		}
 	}
 
@@ -326,7 +345,7 @@ func syncLists(sshClient *ssh.Client, sshEntryRoot string, sshIsWindows bool, ti
 	// call sftpSync with the download and upload lists
 	if timeSyncedErr == nil && (max(len(downloadList), len(uploadList)) > 0) { // only call sftpSync if there are entries to download or upload
 		fmt.Println() // add a gap between list-add messages and the actual sync messages from sftpSync
-		if err := sftpSync(sshClient, sshEntryRoot, sshIsWindows, downloadList, uploadList); err != nil {
+		if err := sftpSync(sshClient, sshEntryRoot, sshAgeDir, sshIsWindows, downloadList, uploadList); err != nil {
 			return [3][]string{nil, nil, nil}, errors.New("unable to sync entries: " + err.Error())
 		}
 		fmt.Println("Client is synchronized with server")
@@ -360,7 +379,7 @@ func deletionSync(deletions []synccommon.Deletion) error {
 // and uploads lists for the client to report to the user.
 func RunJob() ([3][]string, error) {
 	// get SSH client to re-use throughout the sync process
-	sshClient, offlineMode, sshIsWindows, sshEntryRoot, err := GetSSHClient()
+	sshClient, offlineMode, sshIsWindows, sshEntryRoot, sshAgeDir, err := GetSSHClient()
 	if offlineMode {
 		return [3][]string{nil, nil, nil}, nil
 	}
@@ -397,7 +416,7 @@ func RunJob() ([3][]string, error) {
 
 	// sync new and updated entries
 	// if time is not synced, the time sync error and upload/download lists will be returned here
-	lists, err := syncLists(sshClient, *sshEntryRoot, *sshIsWindows, timeSyncedErr, localEntryMap, remoteEntryMap)
+	lists, err := syncLists(sshClient, *sshEntryRoot, *sshAgeDir, *sshIsWindows, timeSyncedErr, localEntryMap, remoteEntryMap)
 	if err != nil {
 		return lists, errors.New("unable to sync entries: " + err.Error())
 	}

syncclient/oneOff.go

@@ -29,7 +29,7 @@ func ShearRemote(vanityPath string, onlyShearAgeFile bool) error {
 
 	var modifier string
 	var output []byte
-	sshClient, offlineMode, _, _, err := GetSSHClient()
+	sshClient, offlineMode, _, _, _, err := GetSSHClient()
 	if offlineMode {
 		goto end
 	}
@@ -88,7 +88,7 @@ func RenameRemote(oldVanityPath, newVanityPath string) error {
 
 	// create an SSH client
 	var output []byte
-	sshClient, offlineMode, _, _, err := GetSSHClient()
+	sshClient, offlineMode, _, _, _, err := GetSSHClient()
 	if offlineMode {
 		goto end
 	}
@@ -131,7 +131,7 @@ func AddFolderRemote(vanityPath string) error {
 
 	// create an SSH client
 	var output []byte
-	sshClient, offlineMode, _, _, err := GetSSHClient()
+	sshClient, offlineMode, _, _, _, err := GetSSHClient()
 	if offlineMode {
 		goto end
 	}
@@ -163,8 +163,8 @@ end:
 // Device IDs are only needed for online synchronization.
 // Device IDs are guaranteed unique as the current UNIX time is appended to them.
 // Leave prefix empty to use the current hostname as the prefix.
-// Returns: the remote EntryRoot and OS type indicator.
+// Returns: the remote EntryRoot, the remote AgeDir, and OS type indicator.
-func GenDeviceID(oldDeviceID, prefix string) (string, bool, error) {
+func GenDeviceID(oldDeviceID, prefix string) (string, string, bool, error) {
 	// generate new device ID
 	if prefix == "" {
 		prefix, _ = os.Hostname()
@@ -176,7 +176,7 @@ func GenDeviceID(oldDeviceID, prefix string) (string, bool, error) {
 	oldDeviceIDPath := global.CfgDir + global.PathSeparator + "devices" + global.PathSeparator + oldDeviceID
 	f, err := os.OpenFile(newDeviceIDPath, os.O_CREATE|os.O_WRONLY, 0600)
 	if err != nil {
-		return "", false, errors.New("unable to create local device ID file: " + err.Error())
+		return "", "", false, errors.New("unable to create local device ID file: " + err.Error())
 	}
 	_ = f.Close() // error ignored; if the file could be created, it can probably be closed
 
@@ -195,32 +195,32 @@ func GenDeviceID(oldDeviceID, prefix string) (string, bool, error) {
 	// register new device ID with server and fetch remote EntryRoot and OS type
 	// also removes the old device ID file (remotely)
 	// if registration fails, remove the new device ID file locally and return before removing the old one
-	sshClient, _, _, _, err := GetSSHClient()
+	sshClient, _, _, _, _, err := GetSSHClient()
 	if err != nil {
 		cleanupOnFail()
-		return "", false, errors.New("unable to connect to SSH client: " + err.Error())
+		return "", "", false, errors.New("unable to connect to SSH client: " + err.Error())
 	}
 	output, err := GetSSHOutput(sshClient, "libmuttonserver register", newDeviceID+"\n"+oldDeviceID)
 	if err != nil {
 		cleanupOnFail()
-		return "", false, errors.New("unable to register device ID with server: " + err.Error())
+		return "", "", false, errors.New("unable to register device ID with server: " + err.Error())
 	}
 	var registerResp synccommon.RegisterResp
 	if err = json.Unmarshal(output, &registerResp); err != nil {
 		cleanupOnFail()
-		return "", false, errors.New("unable to unmarshal server register response: " + err.Error())
+		return "", "", false, errors.New("unable to unmarshal server register response: " + err.Error())
 	}
 	if registerResp.ErrMsg != nil {
 		cleanupOnFail()
-		return "", false, errors.New("unable to complete register; server-side error occurred: " + strings.ReplaceAll(*registerResp.ErrMsg, global.FSSpace, "\n"))
+		return "", "", false, errors.New("unable to complete register; server-side error occurred: " + strings.ReplaceAll(*registerResp.ErrMsg, global.FSSpace, "\n"))
 	}
 	_ = sshClient.Close() // ignore error; non-critical/unlikely/not much could be done about it
 
 	// remove old device ID file (locally; may not exist)
 	if err = os.RemoveAll(oldDeviceIDPath); err != nil {
 		cleanupOnFail()
-		return "", false, errors.New("unable to remove old device ID file (locally): " + err.Error())
+		return "", "", false, errors.New("unable to remove old device ID file (locally): " + err.Error())
 	}
 
-	return registerResp.EntryRoot, registerResp.IsWindows, nil
+	return registerResp.EntryRoot, registerResp.AgeDir, registerResp.IsWindows, nil
 }

synccommon/common.go

@@ -38,6 +38,7 @@ type Entry struct {
 type RegisterResp struct {
 	ErrMsg    *string `json:"errMsg"` // nil if no error occurred
 	EntryRoot string  `json:"entryRoot"`
+	AgeDir    string  `json:"ageDir"`
 	IsWindows bool    `json:"isWindows"`
 }