From b60efd86d6b1dcadfa2d96465d1d009b9fbb2c3a Mon Sep 17 00:00:00 2001 From: Cuan Date: Wed, 27 Oct 2021 01:12:57 -0400 Subject: [PATCH] sftp/sshync syncing progress --- README.md | 3 + bin/rpass | 8 +-- bin/sshync.py | 160 ++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 136 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index ba7dc5d..510f2ad 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +# NOTE +In the coming days, "rpass" will be renamed to "sshyp" and re-launched, starting with a new, custom syncing system (called sshync) based on sftp (rsync support is being abandoned, hence the project rename). "rpass" will recieve one more update to fix up the note copying feature, but after this patch, all efforts will be on "sshyp". "sshyp" will feature more reliable syncing (almost ready), a newly themed CLI interface, and a re-write of lots of the "rpass" code. + # rpass A very simple password manager with rsync integration - alternative to GNU pass diff --git a/bin/rpass b/bin/rpass index 166dcce..024830e 100755 --- a/bin/rpass +++ b/bin/rpass @@ -113,7 +113,7 @@ if argument == 'version' or argument == '-v': print('/ /') print('/ rpass Copyright (C) 2021 Randall Winkhart /') print('/ /') - print('/ Version 2021.10.13.mr5.1 /') + print('/ Version 2021.10.18.mr5.1 /') print('/ The Notetaker Update /') print('/ /') print('////////////////////////////////////////////////////////\n') @@ -193,10 +193,8 @@ if argument == 'config': else: directoryssh = ('/home/' + usernamessh + '/.rpass-store/') open("/var/lib/rpass/rpass-device", 'w').write(devicetype) - sshync.make_sentinel_profile('/var/lib/rpass/rpass.sshync', '/home/' + environ.get('USER') + - '/.rpass-store/', directoryssh, '/home/' + environ.get('USER') + - '/.ssh/rpass', ip, port, usernamessh, '/var/lib/rpass/mod_time', - '/home/nixon/mod_time') + sshync.make_profile('/var/lib/rpass/rpass.sshync', '/home/' + environ.get('USER') + '/.rpass-store/', + directoryssh, '/home/' + environ.get('USER') + '/.ssh/rpass', ip, port, usernamessh) print('\nConfiguration complete!\n') except KeyboardInterrupt: print('\n\nConfiguration cancelled.\n') diff --git a/bin/sshync.py b/bin/sshync.py index f1f0621..2b3bf26 100755 --- a/bin/sshync.py +++ b/bin/sshync.py @@ -1,8 +1,16 @@ #!/usr/bin/python3 -# sshync 2021.10.12.unreleased1 +# Main Targets: +# TODO allow proper uploading of files (currently broken) +# TODO Truncate mod times for sftp compatibility +# TODO Make it so that when a new folder is created in sshyp, it is automatically created on the remote server +# TODO Replace references to rpass/sshyp directory with a changeable variable -# sshync is a wrapper around sftp that functions as a Python library alternative to scp +# sshync 2021.10.27.unreleased2 +# sshync was created to solve a problem with rpass/sshyp, and as such, sshync will be bundled with sshyp until it is +# polished enough for a standalone release + +# sshync is a wrapper around sftp that allows easy remote directory synchronization via Python # openssh (on POSIX compliant Unix) is required # sshync will always overwrite the older files with the newer files, regardless of all other conditions @@ -12,23 +20,63 @@ # external modules +from os import walk +from os import system as term from os.path import getmtime -from subprocess import getoutput as term_output +from os.path import join as path_join # internal modules +# recursively scans a local directory for all files and saves file paths and mod times to lists +def get_file_paths_mod(directory, name_database, time_database): + file_paths = [] + temp_mod_times = [] + mod_times = [] + for root, directories, files in walk(directory): + for filename in files: + filepath = path_join(root, filename) + file_paths.append(filepath) + temp_mod_times.append(str(getmtime(filepath))) + for time in temp_mod_times: + mod_times.append(round(float(time), 2)) + open(name_database, 'w').write(str(file_paths)) + open(time_database, 'w').write(str(mod_times)) + return file_paths, mod_times + + +# recursively scans a remote directory for all files and saves file paths and mod times to lists +def get_file_paths_mod_remote(profile_dir): + # import profile data + profile_data = get_profile(profile_dir) + user = profile_data[0].replace('\n', '') + ip = profile_data[1].replace('\n', '') + port = profile_data[2].replace('\n', '') + remote_dir = profile_data[4].replace('\n', '') + identity = profile_data[5].replace('\n', '') + term("ssh -i " + "'" + identity + "'" + " -p " + port + " " + user + "@" + ip + " \"python -c \'import sshync; " + "sshync.get_file_paths_mod(" + + "\"'\"" + remote_dir + "/\"'\", " + "\"'\"/var/lib/rpass/sshyncdatabase_names\"'\", " + + "\"'\"/var/lib/rpass/sshyncdatabase_times\"'\")\'\"") + term( + 'sftp -P ' + port + ' ' + user + '@' + ip + ':/var/lib/rpass/sshyncdatabase_names ' + '/var/lib/rpass/sshyncdatabase_names') + term( + 'sftp -P ' + port + ' ' + user + '@' + ip + ':/var/lib/rpass/sshyncdatabase_times ' + '/var/lib/rpass/sshyncdatabase_times') + remote_mod_names = (open('/var/lib/rpass/sshyncdatabase_names').read()).replace('[', '').replace(']', '').replace( + ' ', '').replace("'", '').split(',') + remote_mod_times = (open('/var/lib/rpass/sshyncdatabase_times').read()).replace('[', '').replace(']', '').replace( + ' ', '').replace("'", '').split(',') + return remote_mod_names, remote_mod_times + + +# regular profiles check the mod time of every individual file in the directory, making them slow but versatile def make_profile(profile_dir, local_dir, remote_dir, identity, ip, port, user): open(profile_dir, 'w').write(user + '\n' + ip + '\n' + port + '\n' + local_dir + '\n' + remote_dir + '\n' + identity - + '\n\n\n') - - -def make_sentinel_profile(profile_dir, local_dir, remote_dir, identity, ip, port, user, local_sentinel, rem_sentinel): - open(profile_dir, 'w').write(user + '\n' + ip + '\n' + port + '\n' + local_dir + '\n' + remote_dir + '\n' + identity - + '\n' + local_sentinel + '\n' + rem_sentinel + '\n') + + '\n') +# returns the settings for a specified profile def get_profile(profile_dir): try: profile_data = open(profile_dir).readlines() @@ -37,6 +85,30 @@ def get_profile(profile_dir): return profile_data +# sorts name lists so that local and remote are in the same order +def name_sort(local_names, remote_names): + remote_names_new = [] + for ele in local_names: + if ele in remote_names: + remote_names_new.append(ele) + for ele in remote_names: + if ele not in remote_names_new: + remote_names_new.append(ele) + return remote_names_new + + +# sorts time lists to compensate for name list re-ordering +def time_sort(old_names, new_names, old_times): + remote_mod_times_new = [] + new_pos = -1 + for title in new_names: + new_pos += 1 + if title in old_names: + old_pos = old_names.index(title) + remote_mod_times_new.append(old_times[old_pos]) + return remote_mod_times_new + + def run_profile(profile_dir): # import profile data profile_data = get_profile(profile_dir) @@ -46,25 +118,53 @@ def run_profile(profile_dir): local_dir = profile_data[3].replace('\n', '') remote_dir = profile_data[4].replace('\n', '') identity = profile_data[5].replace('\n', '') - local_sentinel = profile_data[6].replace('\n', '') - rem_sentinel = profile_data[7].replace('\n', '') # get file modification times - if local_sentinel == '': - local_modification_time = getmtime(local_dir) - remote_modification_time = float( - term_output("ssh -i " + "'" + identity + "'" + " -p " + port + " " + user + "@" + - ip + " \"python -c 'from os.path import getmtime; mod_time = " - "(getmtime(" + '\\"' + remote_dir + '\\"' + ")); print(mod_time);" - "'" + "\"")) - else: - local_modification_time = getmtime(local_sentinel) - remote_modification_time = float( - term_output("ssh -i " + "'" + identity + "'" + " -p " + port + " " + user + "@" + - ip + " \"python -c 'from os.path import getmtime; mod_time = " - "(getmtime(" + '\\"' + rem_sentinel + '\\"' + ")); print(mod_time);" - "'" + "\"")) - # compare file modification times - if local_modification_time > remote_modification_time: - print('\nLocal is newer, uploading...\n') # upload - else: - print('\nRemote is newer, downloading...\n') # download + local_mod_names, local_mod_times = get_file_paths_mod(local_dir + '/', '/var/lib/rpass/sshyncdatabase_names', + '/var/lib/rpass/sshyncdatabase_times') + remote_mod_times = [] + remote_mod_names, remote_mod_times_temp = get_file_paths_mod_remote('/var/lib/rpass/rpass.sshync') + for time in remote_mod_times_temp: # remove quotes from remote_mod_times + remote_mod_times.append(float(time)) + lmod_short = [] + rmod_short = [] + matching_names = [] + matching_times_local = [] + matching_times_remote = [] + for lmod in local_mod_names: + lmod_short.append(lmod.replace(local_dir, '')) + for rmod in remote_mod_names: + rmod_short.append(rmod.replace(remote_dir, '')) + rmod_short_old = rmod_short + rmod_short = name_sort(lmod_short, rmod_short) + remote_mod_times = time_sort(rmod_short_old, rmod_short, remote_mod_times) + i = -1 + for lshort in lmod_short: + i += 1 + if lshort in rmod_short: + matching_names.append(lshort) + matching_times_local.append(local_mod_times[i]) + else: + print('Uploading ' + lshort) + term('sftp -p -P ' + port + ' ' + user + '@' + ip + ':' + remote_dir + lshort.replace('/', '', 1) + " <<< $'put " + local_dir + lshort.replace('/', '', 1) + "'") + i = -1 + for rshort in rmod_short: + i += 1 + if rshort in lmod_short: + matching_times_remote.append(float(remote_mod_times[i])) + else: + print('Downloading ' + rshort) + term('sftp -p -P ' + port + ' ' + user + '@' + ip + ':' + remote_dir + rshort.replace('/', '', 1) + ' ' + local_dir + rshort.replace('/', '', 1)) + i = -1 + for ltime in matching_times_local: + i += 1 + if float(ltime) == matching_times_remote[i]: + print('Local ' + matching_names[i] + ' is up to date, not syncing.') + elif float(ltime) > matching_times_remote[i]: + print('Local ' + matching_names[i] + ' is newer, uploading.') + # TODO SYNC UPLOAD lmod + elif float(ltime) < matching_times_remote[i]: + print('Local ' + matching_names[i] + ' is older, downloading.') + # TODO SYNC DOWNLOAD rmod + + +run_profile('/var/lib/rpass/rpass.sshync')