From 1a35aaf33b597e51a8de690f98d0e8f8e0ff5520 Mon Sep 17 00:00:00 2001 From: Randall Winkhart Date: Wed, 7 Dec 2022 15:46:33 -0500 Subject: [PATCH] Combined all list fetching functions into one to drastically reduce the number of SSH connections (speeds up sshync exponentially) Former-commit-id: 11290b3915a2f3d92696e0953997c3fb293ec7e3 [formerly 6080726dc61107c27533a85033d0e8e82bbdee07] Former-commit-id: 3f799d1c8e2d3cd694cad90ba9f4444bb2673934 --- lib/sshync.py | 101 +++++++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/lib/sshync.py b/lib/sshync.py index 0488521..ba2cf26 100755 --- a/lib/sshync.py +++ b/lib/sshync.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 from os import listdir, remove, walk from os.path import expanduser, getmtime, join -from pathlib import Path -from subprocess import PIPE, run +from subprocess import CalledProcessError, PIPE, run from sys import exit as s_exit # REMOTE -def deletion_check(_client_device_name): # creates a list of files and folders to be deleted from a client device +def remote_list_gen(_client_device_name, _remote_dir): # prints all necessary remote data to stdout + # deletions for _file in listdir(expanduser('~/.config/sshyp/deleted')): _file_path, _sep, _device = _file.partition('\x1f') _file_path = _file_path.replace('\x1e', '/') @@ -18,12 +18,13 @@ def deletion_check(_client_device_name): # creates a list of files and folders remove(f"{expanduser('~/.config/sshyp/deleted/')}{_file}") except FileNotFoundError: pass - - -def folder_check(): # creates a list of folders to be compared with those of a client device + print('\x1d') + # folders for _root, _directories, _files in walk(expanduser('~/.local/share/sshyp')): for _dir in _directories: print(f"{_root.replace(expanduser('~'), '')}/{_dir}") + print('\x1d') + get_local_data(_remote_dir, 'server') # titles and mod times # HYBRID @@ -58,13 +59,38 @@ def get_local_data(_directory, _device): # retrieves and returns titles and mod # LOCAL -def get_remote_data(_user_data): # retrieves and returns titles and mod times from the remote server - _titles_mods = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}", - f'cd /lib/sshyp; python3 -c \'from sshync import get_local_data; get_local_data' - f'("{_user_data[4]}", "server")\''], stdout=PIPE, text=True).stdout.split('\n') - _title_list = _titles_mods[:len(_titles_mods)//2] - _mod_list = _titles_mods[len(_titles_mods)//2:] - return _title_list, _mod_list +def remote_list_fetch(_user_data): # captures and returns all necessary data from the remote server + try: + _remote_data = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}", + f'cd /lib/sshyp; python3 -c \'from sshync import remote_list_gen; remote_list_gen' + f'("{_user_data[6]}", "{_user_data[4]}")\''], stdout=PIPE, text=True, check=True + ).stdout.split('\x1d') + except CalledProcessError: + print('\n\u001b[38;5;9merror: failed to connect to the remote server\u001b[0m\n') + _remote_data = '' + s_exit(3) + _deletion_database = _remote_data[0].strip().split('\n') + _folder_database = _remote_data[1].strip().split('\n') + _titles_mods = _remote_data[2].strip().split('\n') + return _deletion_database, _folder_database, _titles_mods[:len(_titles_mods)//2], \ + _titles_mods[len(_titles_mods)//2:] + + +def deletion_sync(_deletion_database, _silent): # checks for and acts upon files and folders marked for deletion + for _file in _deletion_database: + if _file != '': + if _silent != 1: + print(f"\u001b[38;5;208m{_file}\u001b[0m has been sheared, removing...") + delete(_file, 'locally') + + +def folder_sync(_folder_database): # creates matches of remote folders on the local client + from pathlib import Path + for _folder in _folder_database: + if _folder != '' and not Path(f"{expanduser('~')}{_folder}").is_dir(): + print(f"\u001b[38;5;2m{_folder.replace('/.local/share/sshyp/', '')}/\u001b[0m does not exist locally, " + f"creating...") + Path(f"{expanduser('~')}{_folder}").mkdir(mode=0o700, parents=True, exist_ok=True) def sort_titles_mods(_list_1, _list_2): # creates and returns two lists (of titles and mod times) generated by sorting @@ -83,28 +109,6 @@ def sort_titles_mods(_list_1, _list_2): # creates and returns two lists (of tit return _title_list_2_sorted, _mod_list_2_sorted -def deletion_sync(_user_data, _silent): # checks for and acts upon files and folders marked for deletion - _deletion_database = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}", - f'cd /lib/sshyp; python3 -c \'from sshync import deletion_check; deletion_check' - f'("{_user_data[6]}")\''], stdout=PIPE, text=True).stdout.split('\n') - for _file in _deletion_database: - if _file != '': - if _silent != 1: - print(f"\u001b[38;5;208m{_file}\u001b[0m has been sheared, removing...") - delete(_file, 'locally') - - -def folder_sync(_user_data): # creates matches of remote folders on the local client - _folder_database = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}", - "cd /lib/sshyp; python3 -c 'from sshync import folder_check; folder_check()'"], - stdout=PIPE, text=True).stdout.split('\n') - for _folder in _folder_database: - if _folder != '' and not Path(f"{expanduser('~')}{_folder}").is_dir(): - print(f"\u001b[38;5;2m{_folder.replace('/.local/share/sshyp/', '')}/\u001b[0m does not exist locally, " - f"creating...") - Path(f"{expanduser('~')}{_folder}").mkdir(mode=0o700, parents=True, exist_ok=True) - - def make_profile(_profile_dir, _local_dir, _remote_dir, _identity, _ip, _port, _user): # creates a sshync job profile open(_profile_dir, 'w').write(f"{_user}\n{_ip}\n{_port}\n{_local_dir}\n{_remote_dir}\n{_identity}\n") @@ -113,7 +117,7 @@ def get_profile(_profile_dir): # returns a list of data read from a sshync job try: _profile_data = open(_profile_dir).readlines() except (FileNotFoundError, IndexError): - print('\n\u001b[38;5;9merror: the profile does not exist or is corrupted.\u001b[0m\n') + print('\n\u001b[38;5;9merror: the profile does not exist or is corrupted\u001b[0m\n') _profile_data = None s_exit(3) _user = _profile_data[0].rstrip() @@ -128,22 +132,25 @@ def get_profile(_profile_dir): # returns a list of data read from a sshync job def run_profile(_profile_dir, _silent): # runs a sshync job profile _user_data = get_profile(_profile_dir) # import profile data - deletion_sync(_user_data, _silent) # check for and sync deletions - folder_sync(_user_data) # check for and sync folders + # fetch and organize remote lists + _deletion_database, _folder_database, _remote_titles, _remote_mods = remote_list_fetch(_user_data) + _remote_titles_mods = (_remote_titles, _remote_mods) + _index_local = sort_titles_mods(_remote_titles_mods, get_local_data(_user_data[3], 'client')) + _index_remote = sort_titles_mods(_index_local, _remote_titles_mods) + # sync deletions and folders + deletion_sync(_deletion_database, _silent) + folder_sync(_folder_database) # sync new and updated files - _remote_titles_mods = get_remote_data(_user_data) - _index_l = sort_titles_mods(_remote_titles_mods, get_local_data(_user_data[3], 'client')) - _index_r = sort_titles_mods(_index_l, _remote_titles_mods) _i = -1 - for _title in _index_l[0]: + for _title in _index_local[0]: _i += 1 - if _title in _index_r[0]: + if _title in _index_remote[0]: # compare mod times and sync - if int(_index_l[1][_i]) > int(_index_r[1][_i]): + if int(_index_local[1][_i]) > int(_index_remote[1][_i]): print(f"\u001b[38;5;4m{_title[:-4]}\u001b[0m is newer locally, uploading...") run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5], _user_data[3] + _title, f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{'/'.join(_title.split('/')[:-1]) + '/'}"]) - elif int(_index_l[1][_i]) < int(_index_r[1][_i]): + elif int(_index_local[1][_i]) < int(_index_remote[1][_i]): print(f"\u001b[38;5;2m{_title[:-4]}\u001b[0m is newer remotely, downloading...") run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5], f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{_title}", @@ -152,8 +159,8 @@ def run_profile(_profile_dir, _silent): # runs a sshync job profile print(f"\u001b[38;5;4m{_title[:-4]}\u001b[0m is not on remote server, uploading...") run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5], _user_data[3] + _title, f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{'/'.join(_title.split('/')[:-1]) + '/'}"]) - for _title in _index_r[0]: - if _title not in _index_l[0]: + for _title in _index_remote[0]: + if _title not in _index_local[0]: print(f"\u001b[38;5;2m{_title[:-4]}\u001b[0m is not in local directory, downloading...") run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5], f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{_title}",