mirror of
https://github.com/rwinkhart/sshyp.git
synced 2026-08-28 04:46:34 -04:00
Combined all list fetching functions into one to drastically reduce the number of SSH connections (speeds up sshync exponentially)
Former-commit-id: 11290b3915a2f3d92696e0953997c3fb293ec7e3 [formerly 6080726dc6]
Former-commit-id: 3f799d1c8e2d3cd694cad90ba9f4444bb2673934
This commit is contained in:
+54
-47
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
from os import listdir, remove, walk
|
||||
from os.path import expanduser, getmtime, join
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, run
|
||||
from subprocess import CalledProcessError, PIPE, run
|
||||
from sys import exit as s_exit
|
||||
|
||||
|
||||
# REMOTE
|
||||
|
||||
def deletion_check(_client_device_name): # creates a list of files and folders to be deleted from a client device
|
||||
def remote_list_gen(_client_device_name, _remote_dir): # prints all necessary remote data to stdout
|
||||
# deletions
|
||||
for _file in listdir(expanduser('~/.config/sshyp/deleted')):
|
||||
_file_path, _sep, _device = _file.partition('\x1f')
|
||||
_file_path = _file_path.replace('\x1e', '/')
|
||||
@@ -18,12 +18,13 @@ def deletion_check(_client_device_name): # creates a list of files and folders
|
||||
remove(f"{expanduser('~/.config/sshyp/deleted/')}{_file}")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def folder_check(): # creates a list of folders to be compared with those of a client device
|
||||
print('\x1d')
|
||||
# folders
|
||||
for _root, _directories, _files in walk(expanduser('~/.local/share/sshyp')):
|
||||
for _dir in _directories:
|
||||
print(f"{_root.replace(expanduser('~'), '')}/{_dir}")
|
||||
print('\x1d')
|
||||
get_local_data(_remote_dir, 'server') # titles and mod times
|
||||
|
||||
|
||||
# HYBRID
|
||||
@@ -58,13 +59,38 @@ def get_local_data(_directory, _device): # retrieves and returns titles and mod
|
||||
|
||||
# LOCAL
|
||||
|
||||
def get_remote_data(_user_data): # retrieves and returns titles and mod times from the remote server
|
||||
_titles_mods = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}",
|
||||
f'cd /lib/sshyp; python3 -c \'from sshync import get_local_data; get_local_data'
|
||||
f'("{_user_data[4]}", "server")\''], stdout=PIPE, text=True).stdout.split('\n')
|
||||
_title_list = _titles_mods[:len(_titles_mods)//2]
|
||||
_mod_list = _titles_mods[len(_titles_mods)//2:]
|
||||
return _title_list, _mod_list
|
||||
def remote_list_fetch(_user_data): # captures and returns all necessary data from the remote server
|
||||
try:
|
||||
_remote_data = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}",
|
||||
f'cd /lib/sshyp; python3 -c \'from sshync import remote_list_gen; remote_list_gen'
|
||||
f'("{_user_data[6]}", "{_user_data[4]}")\''], stdout=PIPE, text=True, check=True
|
||||
).stdout.split('\x1d')
|
||||
except CalledProcessError:
|
||||
print('\n\u001b[38;5;9merror: failed to connect to the remote server\u001b[0m\n')
|
||||
_remote_data = ''
|
||||
s_exit(3)
|
||||
_deletion_database = _remote_data[0].strip().split('\n')
|
||||
_folder_database = _remote_data[1].strip().split('\n')
|
||||
_titles_mods = _remote_data[2].strip().split('\n')
|
||||
return _deletion_database, _folder_database, _titles_mods[:len(_titles_mods)//2], \
|
||||
_titles_mods[len(_titles_mods)//2:]
|
||||
|
||||
|
||||
def deletion_sync(_deletion_database, _silent): # checks for and acts upon files and folders marked for deletion
|
||||
for _file in _deletion_database:
|
||||
if _file != '':
|
||||
if _silent != 1:
|
||||
print(f"\u001b[38;5;208m{_file}\u001b[0m has been sheared, removing...")
|
||||
delete(_file, 'locally')
|
||||
|
||||
|
||||
def folder_sync(_folder_database): # creates matches of remote folders on the local client
|
||||
from pathlib import Path
|
||||
for _folder in _folder_database:
|
||||
if _folder != '' and not Path(f"{expanduser('~')}{_folder}").is_dir():
|
||||
print(f"\u001b[38;5;2m{_folder.replace('/.local/share/sshyp/', '')}/\u001b[0m does not exist locally, "
|
||||
f"creating...")
|
||||
Path(f"{expanduser('~')}{_folder}").mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def sort_titles_mods(_list_1, _list_2): # creates and returns two lists (of titles and mod times) generated by sorting
|
||||
@@ -83,28 +109,6 @@ def sort_titles_mods(_list_1, _list_2): # creates and returns two lists (of tit
|
||||
return _title_list_2_sorted, _mod_list_2_sorted
|
||||
|
||||
|
||||
def deletion_sync(_user_data, _silent): # checks for and acts upon files and folders marked for deletion
|
||||
_deletion_database = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}",
|
||||
f'cd /lib/sshyp; python3 -c \'from sshync import deletion_check; deletion_check'
|
||||
f'("{_user_data[6]}")\''], stdout=PIPE, text=True).stdout.split('\n')
|
||||
for _file in _deletion_database:
|
||||
if _file != '':
|
||||
if _silent != 1:
|
||||
print(f"\u001b[38;5;208m{_file}\u001b[0m has been sheared, removing...")
|
||||
delete(_file, 'locally')
|
||||
|
||||
|
||||
def folder_sync(_user_data): # creates matches of remote folders on the local client
|
||||
_folder_database = run(['ssh', '-i', _user_data[5], '-p', _user_data[2], f"{_user_data[0]}@{_user_data[1]}",
|
||||
"cd /lib/sshyp; python3 -c 'from sshync import folder_check; folder_check()'"],
|
||||
stdout=PIPE, text=True).stdout.split('\n')
|
||||
for _folder in _folder_database:
|
||||
if _folder != '' and not Path(f"{expanduser('~')}{_folder}").is_dir():
|
||||
print(f"\u001b[38;5;2m{_folder.replace('/.local/share/sshyp/', '')}/\u001b[0m does not exist locally, "
|
||||
f"creating...")
|
||||
Path(f"{expanduser('~')}{_folder}").mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def make_profile(_profile_dir, _local_dir, _remote_dir, _identity, _ip, _port, _user): # creates a sshync job profile
|
||||
open(_profile_dir, 'w').write(f"{_user}\n{_ip}\n{_port}\n{_local_dir}\n{_remote_dir}\n{_identity}\n")
|
||||
|
||||
@@ -113,7 +117,7 @@ def get_profile(_profile_dir): # returns a list of data read from a sshync job
|
||||
try:
|
||||
_profile_data = open(_profile_dir).readlines()
|
||||
except (FileNotFoundError, IndexError):
|
||||
print('\n\u001b[38;5;9merror: the profile does not exist or is corrupted.\u001b[0m\n')
|
||||
print('\n\u001b[38;5;9merror: the profile does not exist or is corrupted\u001b[0m\n')
|
||||
_profile_data = None
|
||||
s_exit(3)
|
||||
_user = _profile_data[0].rstrip()
|
||||
@@ -128,22 +132,25 @@ def get_profile(_profile_dir): # returns a list of data read from a sshync job
|
||||
|
||||
def run_profile(_profile_dir, _silent): # runs a sshync job profile
|
||||
_user_data = get_profile(_profile_dir) # import profile data
|
||||
deletion_sync(_user_data, _silent) # check for and sync deletions
|
||||
folder_sync(_user_data) # check for and sync folders
|
||||
# fetch and organize remote lists
|
||||
_deletion_database, _folder_database, _remote_titles, _remote_mods = remote_list_fetch(_user_data)
|
||||
_remote_titles_mods = (_remote_titles, _remote_mods)
|
||||
_index_local = sort_titles_mods(_remote_titles_mods, get_local_data(_user_data[3], 'client'))
|
||||
_index_remote = sort_titles_mods(_index_local, _remote_titles_mods)
|
||||
# sync deletions and folders
|
||||
deletion_sync(_deletion_database, _silent)
|
||||
folder_sync(_folder_database)
|
||||
# sync new and updated files
|
||||
_remote_titles_mods = get_remote_data(_user_data)
|
||||
_index_l = sort_titles_mods(_remote_titles_mods, get_local_data(_user_data[3], 'client'))
|
||||
_index_r = sort_titles_mods(_index_l, _remote_titles_mods)
|
||||
_i = -1
|
||||
for _title in _index_l[0]:
|
||||
for _title in _index_local[0]:
|
||||
_i += 1
|
||||
if _title in _index_r[0]:
|
||||
if _title in _index_remote[0]:
|
||||
# compare mod times and sync
|
||||
if int(_index_l[1][_i]) > int(_index_r[1][_i]):
|
||||
if int(_index_local[1][_i]) > int(_index_remote[1][_i]):
|
||||
print(f"\u001b[38;5;4m{_title[:-4]}\u001b[0m is newer locally, uploading...")
|
||||
run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5], _user_data[3] + _title,
|
||||
f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{'/'.join(_title.split('/')[:-1]) + '/'}"])
|
||||
elif int(_index_l[1][_i]) < int(_index_r[1][_i]):
|
||||
elif int(_index_local[1][_i]) < int(_index_remote[1][_i]):
|
||||
print(f"\u001b[38;5;2m{_title[:-4]}\u001b[0m is newer remotely, downloading...")
|
||||
run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5],
|
||||
f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{_title}",
|
||||
@@ -152,8 +159,8 @@ def run_profile(_profile_dir, _silent): # runs a sshync job profile
|
||||
print(f"\u001b[38;5;4m{_title[:-4]}\u001b[0m is not on remote server, uploading...")
|
||||
run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5], _user_data[3] + _title,
|
||||
f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{'/'.join(_title.split('/')[:-1]) + '/'}"])
|
||||
for _title in _index_r[0]:
|
||||
if _title not in _index_l[0]:
|
||||
for _title in _index_remote[0]:
|
||||
if _title not in _index_local[0]:
|
||||
print(f"\u001b[38;5;2m{_title[:-4]}\u001b[0m is not in local directory, downloading...")
|
||||
run(['scp', '-pqs', '-P', _user_data[2], '-i', _user_data[5],
|
||||
f"{_user_data[0]}@{_user_data[1]}:{_user_data[4]}{_title}",
|
||||
|
||||
Reference in New Issue
Block a user