From 05c034652f4f7321d863f48c023ccd1d65082121 Mon Sep 17 00:00:00 2001 From: Randall Winkhart Date: Fri, 25 Nov 2022 21:11:07 -0500 Subject: [PATCH] Replaced database downloads in sync() with newer output capture logic from sshync - much faster syncing Former-commit-id: f6a9dce34263b3385440e6c166f9880f4781b5d5 [formerly 8276b9164e57c02e1faa90a0b0050aea345259e0] Former-commit-id: 850bf5889c2cf401ad9e91da5512a7455a353e6d --- lib/sshyp.py | 48 ++++++++++++++-------------------------------- lib/sshypRemote.py | 18 ++++++++--------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/lib/sshyp.py b/lib/sshyp.py index 446c7d9..bdf7920 100755 --- a/lib/sshyp.py +++ b/lib/sshyp.py @@ -5,6 +5,7 @@ from pathlib import Path from random import randint, SystemRandom from shutil import get_terminal_size, move, rmtree import sshync +from sshypRemote import delete as offline_delete from subprocess import CalledProcessError, DEVNULL, Popen, PIPE, run from sys import argv, exit as s_exit @@ -490,43 +491,23 @@ def read_shortcut(): # shortcut to quickly read an entry def sync(): # calls sshync to sync changes to the user's server print('\nsyncing entries with the server device...\n') # check for deletions - system(f"ssh -i '{expanduser('~/.ssh/sshyp')}' -p {port} {username_ssh}@{ip} \"cd /lib/sshyp; python -c " - f"'import sshypRemote; sshypRemote.deletion_check(\"'\"{client_device_id}\"'\")'\"") - system(f"scp -pqs -P {port} -i '{expanduser('~/.ssh/sshyp')}' {username_ssh}@{ip}:'/home/{username_ssh}" - f"/.config/sshyp/deletion_database' {expanduser('~/.config/sshyp/')}") - try: - _deletion_database = open(expanduser('~/.config/sshyp/deletion_database')).readlines() - except (FileNotFoundError, IndexError): - print('\n\u001b[38;5;9merror: the deletion database does not exist or is corrupted\u001b[0m\n') - _deletion_database = None - s_exit(6) + _deletion_database = run(['ssh', '-i', expanduser('~/.ssh/sshyp'), '-p', port, f"{username_ssh}@{ip}", + f'cd /lib/sshyp; python -c \'import sshypRemote; sshypRemote.deletion_check("' + f'{client_device_id}")\''], stdout=PIPE, text=True).stdout.split('\n') for _file in _deletion_database: - try: + if _file != '': if silent_sync != 1: - print(f"\u001b[38;5;208m{_file[:-1]}\u001b[0m has been sheared, removing...") - if _file[:-1].endswith('/'): - rmtree(f"{directory}{_file[:-1]}") - else: - remove(f"{directory}{_file[:-1]}.gpg") - except FileNotFoundError: - if silent_sync != 1: - print('location does not exist locally') + print(f"\u001b[38;5;208m{_file}\u001b[0m has been sheared, removing...") + offline_delete(_file, 'locally') # check for new folders - system(f"ssh -i '{expanduser('~/.ssh/sshyp')}' -p {port} {username_ssh}@{ip} \"cd /lib/sshyp; python -c " - f"'import sshypRemote; sshypRemote.folder_check()'\"") - system(f"scp -pqs -P {port} -i '{expanduser('~/.ssh/sshyp')}' {username_ssh}@{ip}:'/home/{username_ssh}" - f"/.config/sshyp/folder_database' {expanduser('~/.config/sshyp/')}") - try: - _folder_database = open(expanduser('~/.config/sshyp/folder_database')).readlines() - except (FileNotFoundError, IndexError): - print('\n\u001b[38;5;9merror: the folder database does not exist or is corrupted\u001b[0m\n') - _folder_database = None - s_exit(6) + _folder_database = run(['ssh', '-i', expanduser('~/.ssh/sshyp'), '-p', port, f"{username_ssh}@{ip}", + "cd /lib/sshyp; python -c 'import sshypRemote; sshypRemote.folder_check()'"], + stdout=PIPE, text=True).stdout.split('\n') for _folder in _folder_database: - if not Path(f"{expanduser('~')}{_folder[:-1]}").is_dir(): - print(f"\u001b[38;5;2m{_folder.replace('/.local/share/sshyp/', '')[:-1]}/\u001b[0m does not exist locally, " + if _folder != '' and not Path(f"{expanduser('~')}{_folder}").is_dir(): + print(f"\u001b[38;5;2m{_folder.replace('/.local/share/sshyp/', '')}/\u001b[0m does not exist locally, " f"creating...") - Path(f"{expanduser('~')}{_folder[:-1]}").mkdir(0o700, parents=True, exist_ok=True) + Path(f"{expanduser('~')}{_folder}").mkdir(0o700, parents=True, exist_ok=True) # set permissions before uploading system('find ' + directory + ' -type d -exec chmod -R 700 {} +') system('find ' + directory + ' -type f -exec chmod -R 600 {} +') @@ -832,8 +813,7 @@ def remove_data(): # deletes an entry from the server and flags it for local de system(f"ssh -i '{expanduser('~/.ssh/sshyp')}' -p {port} {username_ssh}@{ip} \"cd /lib/sshyp; python -c " f"'import sshypRemote; sshypRemote.delete(\"'\"{_entry_name}\"'\", \"'\"remotely\"'\")'\"") else: - from sshypRemote import delete as offline_delete - offline_delete(_entry_name, '') + offline_delete(_entry_name, 'locally') if __name__ == "__main__": diff --git a/lib/sshypRemote.py b/lib/sshypRemote.py index aaf18d7..6f1d0fb 100755 --- a/lib/sshypRemote.py +++ b/lib/sshypRemote.py @@ -4,7 +4,7 @@ from os.path import expanduser from shutil import rmtree -def delete(_file_path, _target_database): # deletes an entry or folder, should be run remotely via ssh +def delete(_file_path, _target_database): # deletes an entry or folder try: if _file_path.endswith('/'): rmtree(f"{expanduser('~/.local/share/sshyp/')}{_file_path}") @@ -12,26 +12,24 @@ def delete(_file_path, _target_database): # deletes an entry or folder, should remove(f"{expanduser('~/.local/share/sshyp/')}{_file_path}.gpg") except FileNotFoundError: print(f"location does not exist {_target_database}") - for _device_name in listdir(expanduser('~/.config/sshyp/devices')): - open(expanduser('~/.config/sshyp/deleted/') + _file_path.replace('/', '\x1e') + '\x1f' + _device_name, 'w') + if _target_database == 'remotely': + for _device_name in listdir(expanduser('~/.config/sshyp/devices')): + open(expanduser('~/.config/sshyp/deleted/') + _file_path.replace('/', '\x1e') + '\x1f' + _device_name, 'w') -def deletion_check(_client_device_name): # creates a list of entries and folders to be deleted from a local machine - open(expanduser('~/.config/sshyp/deletion_database'), 'w').write('') +def deletion_check(_client_device_name): # creates a list of entries and folders to be deleted from a client device for _file in listdir(expanduser('~/.config/sshyp/deleted')): _file_path, _sep, _device = _file.partition('\x1f') _file_path = _file_path.replace('\x1e', '/') if _device == _client_device_name: + print(_file_path) try: remove(f"{expanduser('~/.config/sshyp/deleted/')}{_file}") except FileNotFoundError: pass - open(expanduser('~/.config/sshyp/deletion_database'), 'a').write(_file_path + '\n') -def folder_check(): # creates a list of folders to be compared with those of a local machine - open(expanduser('~/.config/sshyp/folder_database'), 'w').write('') +def folder_check(): # creates a list of folders to be compared with those of a client device for _root, _directories, _files in walk(expanduser('~/.local/share/sshyp')): for _dir in _directories: - open(expanduser('~/.config/sshyp/folder_database'), 'a')\ - .write(f"{_root.replace(expanduser('~'), '')}/{_dir}\n") + print(f"{_root.replace(expanduser('~'), '')}/{_dir}")