Full re-write of sshync

bin/sshync.py

@@ -1,216 +1,106 @@
 #!/usr/bin/python3
 
-# Main Targets:
+# sshync 2021.12.01.unreleased6
-# TODO Make it so that when a new folder is created in sshyp, it is automatically created on the remote server
-# TODO Replace references to sshyp directory with a changeable variable
-# TODO Clean everything up and move sshync to a separate package
-# TODO Add dry run support
-# TODO use partition command instead of split for separating paths
-# TODO use classes!
-
-# sshync 2021.11.05.unreleased5
-# sshync was created to solve a problem with rpass/sshyp, and as such, sshync will be bundled with sshyp until it is
-# polished enough for a standalone release
-
-# sshync is a wrapper around sftp that allows easy remote directory synchronization via Python
-# openssh (on POSIX compliant Unix) is required
-
-# sshync will always overwrite the older files with the newer files, regardless of all other conditions
-#   ^the age is determined by the date and time of the last modification to the files
-
-# syncing via sshync is always done recursively
 
 # external modules
 
-from os import walk
+from os import system, remove, walk
-from os import system as term
+from os.path import getmtime, join
-from os.path import getmtime
+from sys import exit as s_exit
-from os.path import join as path_join
 
 
-# internal modules
+# utility functions
+
+def get_titles_mods(_directory, _destination, _user_data):
+    _title_list, _mod_list = [], []
+    # local fetching
+    if _destination == 'l':
+        if type(_user_data) == str:
+            _user_data = str(_user_data).strip('(').strip(')').replace(' ', '').split(',')
+        remove(_user_data[6] + 'sshync_database')
+        for _root, _directories, _files in walk(_directory):
+            for _filename in _files:
+                _title_list.append(join(_root.replace(_directory, '', 1), _filename))
+                _mod_list.append(int(getmtime(join(_root, _filename))))
+        i = -1
+        for _title in _title_list:
+            open(_user_data[6] + 'sshync_database', 'a').write(str(_title) + '\n')
+        open(_user_data[6] + 'sshync_database', 'a').write('^&*\n')
+        for _mod in _mod_list:
+            open(_user_data[6] + 'sshync_database', 'a').write(str(_mod) + '\n')
+    # remote fetching
+    if _destination == 'r':
+        system(f"ssh -i '{_user_data[5]}' -p {_user_data[2]} {_user_data[0]}@{_user_data[1]} \"python -c 'import sshync"
+               f"; sshync.get_titles_mods(\"'\"{_user_data[4]}\"'\", \"'\"l\"'\", \"'\"{_user_data}\"'\")'\"")
+        system(f"sftp -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:'{_user_data[6]}sshync_database' "
+               f"'{_user_data[6]}sshync_database'")
+        _titles, _sep, _mods = ' '.join(open(_user_data[6] + 'sshync_database').readlines()).replace('\n', '').partition('^&*')
+        _title_list, _mod_list = _titles.split(' ')[:-1], _mods.split(' ')[1:]
+    return _title_list, _mod_list
 
 
-# recursively scans a local directory for all files and saves file paths and mod times to lists
+def sort_titles_mods(_list_1, _list_2):
-def get_file_paths_mod(directory, name_database, time_database):
+    _title_list_2_sorted, _mod_list_2_sorted = [], []
-    file_paths = []
+    # title sorting
-    temp_mod_times = []
+    for _title in _list_1[0]:
-    mod_times = []
+        if _title in _list_2[0]:
-    for root, directories, files in walk(directory):
+            _title_list_2_sorted.append(_title)
-        for filename in files:
+    for _title in _list_2[0]:
-            filepath = path_join(root, filename)
+        if _title not in _title_list_2_sorted:
-            file_paths.append(filepath)
+            _title_list_2_sorted.append(_title)
-            temp_mod_times.append(str(getmtime(filepath)))
+    # mod time sorting
-    for time in temp_mod_times:
+    for _title in _title_list_2_sorted:
-        mod_times.append(round(float(time), 2))
+        _mod_list_2_sorted.append(_list_2[1][_list_2[0].index(_title)])
-    open(name_database, 'w').write(str(file_paths))
+    return _title_list_2_sorted, _mod_list_2_sorted
-    open(time_database, 'w').write(str(mod_times))
-    return file_paths, mod_times
 
 
-# recursively scans a remote directory for all files and saves file paths and mod times to lists
+def make_profile(_profile_dir, _local_dir, _remote_dir, _identity, _ip, _port, _user):
-def get_file_paths_mod_remote(profile_dir):
+    open(_profile_dir, 'w').write(_user + '\n' + _ip + '\n' + _port + '\n' + _local_dir + '\n' + _remote_dir + '\n' +
-    # import profile data
+                                  _identity + '\n')
-    profile_data = get_profile(profile_dir)
-    user = profile_data[0].replace('\n', '')
-    ip = profile_data[1].replace('\n', '')
-    port = profile_data[2].replace('\n', '')
-    remote_dir = profile_data[4].replace('\n', '')
-    identity = profile_data[5].replace('\n', '')
-    term("ssh -i " + "'" + identity + "'" + " -p " + port + " " + user + "@" + ip + " \"python -c \'import sshync; "
-                                                                                    "sshync.get_file_paths_mod(" +
-         "\"'\"" + remote_dir + "/\"'\", " + "\"'\"/var/lib/sshyp/sshyncdatabase_names\"'\", " +
-         "\"'\"/var/lib/sshyp/sshyncdatabase_times\"'\")\'\"")
-    term(
-        'sftp -P ' + port + ' ' + user + '@' + ip + ':/var/lib/sshyp/sshyncdatabase_names ' +
-        '/var/lib/sshyp/sshyncdatabase_names')
-    term(
-        'sftp -P ' + port + ' ' + user + '@' + ip + ':/var/lib/sshyp/sshyncdatabase_times ' +
-        '/var/lib/sshyp/sshyncdatabase_times')
-    remote_mod_names = (open('/var/lib/sshyp/sshyncdatabase_names').read()).replace('[', '').replace(']', '').replace(
-        ' ', '').replace("'", '').split(',')
-    remote_mod_times = (open('/var/lib/sshyp/sshyncdatabase_times').read()).replace('[', '').replace(']', '').replace(
-        ' ', '').replace("'", '').split(',')
-    return remote_mod_names, remote_mod_times
 
 
-# regular profiles check the mod time of every individual file in the directory, making them slow but versatile
+def get_profile(_profile_dir):
-def make_profile(profile_dir, local_dir, remote_dir, identity, ip, port, user):
-    open(profile_dir, 'w').write(user + '\n' + ip + '\n' + port + '\n' + local_dir + '\n' + remote_dir + '\n' + identity
-                                 + '\n')
-
-
-# returns the settings for a specified profile
-def get_profile(profile_dir):
     try:
-        profile_data = open(profile_dir).readlines()
+        _profile_data = open(_profile_dir).readlines()
     except (FileNotFoundError, IndexError):
-        profile_data = 0
+        print('\nEither the profile does not exist, or it is corrupted.\n')
-    return profile_data
+        _profile_data = None
+        s_exit()
+    # extract data from profile
+    _user = _profile_data[0].replace('\n', '')
+    _ip = _profile_data[1].replace('\n', '')
+    _port = _profile_data[2].replace('\n', '')
+    _local_dir = _profile_data[3].replace('\n', '')
+    _remote_dir = _profile_data[4].replace('\n', '')
+    _identity = _profile_data[5].replace('\n', '')
+    _data_dir = ' '.join(_profile_dir.split('/')[:-1]).replace(' ', '/') + '/'
+    return _user, _ip, _port, _local_dir, _remote_dir, _identity, _data_dir
 
 
-# sorts name lists so that local and remote are in the same order
+def run_profile(_profile_dir):
-def name_sort(local_names, remote_names):
+    _user_data = get_profile(_profile_dir)
-    remote_names_new = []
+    _remote_titles_mods_saver = get_titles_mods(_user_data[4], 'r', _user_data)  # saved to prevent re-walking directory
-    for ele in local_names:
+    _index_l = sort_titles_mods(_remote_titles_mods_saver, get_titles_mods(_user_data[3], 'l', _user_data))
-        if ele in remote_names:
+    _index_r = sort_titles_mods(_index_l, _remote_titles_mods_saver)
-            remote_names_new.append(ele)
+    _i = -1
-    for ele in remote_names:
+    for _title in _index_l[0]:
-        if ele not in remote_names_new:
+        _i += 1
-            remote_names_new.append(ele)
+        if _title in _index_r[0]:
-    return remote_names_new
+            # compare mod times and sync
-
+            if int(_index_l[1][_i]) > int(_index_r[1][_i]):
-
+                print(f"[{_title}] Local is newer, uploading...")
-# sorts time lists to compensate for name list re-ordering
+                system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:"
-def time_sort(old_names, new_names, old_times):
+                       f"{_user_data[4]}{_title} <<< $'put {_user_data[3]}{_title}'")
-    remote_mod_times_new = []
+            elif int(_index_l[1][_i]) < int(_index_r[1][_i]):
-    new_pos = -1
+                print(f"[{_title}] Remote is newer, downloading...")
-    for title in new_names:
+                system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:"
-        new_pos += 1
+                       f"{_user_data[4]}{_title} {_user_data[3]}{_title}")
-        if title in old_names:
-            old_pos = old_names.index(title)
-            try:
-                remote_mod_times_new.append(old_times[old_pos])
-            except IndexError:
-                pass
-    return remote_mod_times_new
-
-
-getter_loop_count = -1
-
-
-def loop_getter(split_remote_dir):
-    global getter_loop_count
-    getter_loop_count += 1
-    new_var = split_remote_dir[getter_loop_count] + '/'
-    return new_var
-
-
-def run_profile(profile_dir):
-    # import profile data
-    profile_data = get_profile(profile_dir)
-    user = profile_data[0].replace('\n', '')
-    ip = profile_data[1].replace('\n', '')
-    port = profile_data[2].replace('\n', '')
-    local_dir = profile_data[3].replace('\n', '')
-    remote_dir = profile_data[4].replace('\n', '')
-    identity = profile_data[5].replace('\n', '')
-    # get file modification times
-    local_mod_names, local_mod_times = get_file_paths_mod(local_dir + '/', '/var/lib/sshyp/sshyncdatabase_names',
-                                                          '/var/lib/sshyp/sshyncdatabase_times')
-    remote_mod_times = []
-    remote_mod_names, remote_mod_times_temp = get_file_paths_mod_remote('/var/lib/sshyp/sshyp.sshync')
-    try:
-        for time in remote_mod_times_temp:  # remove quotes from remote_mod_times
-            remote_mod_times.append(float(time))
-    except ValueError:
-        pass
-    lmod_short = []
-    rmod_short = []
-    matching_names = []
-    matching_times_local = []
-    matching_times_remote = []
-    for lmod in local_mod_names:
-        lmod_short.append(lmod.replace(local_dir, ''))
-    for rmod in remote_mod_names:
-        rmod_short.append(rmod.replace(remote_dir, ''))
-    rmod_short_old = rmod_short
-    rmod_short = name_sort(lmod_short, rmod_short)
-    remote_mod_times = time_sort(rmod_short_old, rmod_short, remote_mod_times)
-    i = -1
-    for lshort in lmod_short:
-        i += 1
-        if lshort in rmod_short:
-            matching_names.append(lshort)
-            matching_times_local.append(local_mod_times[i])
         else:
-            print('Uploading ' + lshort)
+            print(f"[{_title}] Not on remote server, uploading...")
-            lshort_split = lshort.replace('/', '', 1).split('/')
+            system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:{_user_data[4]}"
-
+                   f"{_title} <<< $'put {_user_data[3]}{_title}'")
-            c = -1
+    for _title in _index_r[0]:
-            for _ in lshort_split:
+        if _title not in _index_l[0]:
-                c += 1
+            print(f"[{_title}] Not in local directory, downloading...")
-
+            system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:{_user_data[4]}"
-            global getter_loop_count
+                   f"{_title} {_user_data[3]}{_title}")
-            getter_loop_count = -1
-            q = ''
-            for _ in range(c):
-                q += loop_getter(lshort_split)
-
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir + q +
-                 " <<< $'put " + local_dir + lshort.replace('/', '', 1) + "'")
-    i = -1
-    for rshort in rmod_short:
-        i += 1
-        if rshort in lmod_short:
-            matching_times_remote.append(float(remote_mod_times[i]))
-        else:
-            print('Downloading ' + rshort)
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir +
-                 rshort.replace('/', '', 1) + ' ' + local_dir + rshort.replace('/', '', 1))
-    i = -1
-    for ltime in matching_times_local:
-        i += 1
-        ltime = int(ltime)
-        if ltime == int(matching_times_remote[i]):
-            print('Local ' + matching_names[i] + ' is up to date, not syncing.')
-        elif ltime > int(matching_times_remote[i]):
-            print('Local ' + matching_names[i] + ' is newer, uploading.')
-            matching_names_split = matching_names[i].replace('/', '', 1).split('/')
-
-            c = -1
-            for _ in matching_names_split:
-                c += 1
-
-            getter_loop_count = -1
-            q = ''
-            for _ in range(c):
-                q += loop_getter(matching_names_split)
-
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir + q +
-                 " <<< $'put " + local_dir + matching_names[i].replace('/', '', 1) + "'")
-
-        elif ltime < int(matching_times_remote[i]):
-            print('Local ' + matching_names[i] + ' is older, downloading.')
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir +
-                 matching_names[i].replace('/', '', 1) + ' ' + local_dir + matching_names[i].replace('/', '', 1))