Updated PKGBUILD for release

PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Randall Winkhart <idgr at tutanota dot com>
 
 pkgname=sshyp
-pkgver=2021.11.05.fr1
+pkgver=2021.12.01.fr2
 pkgrel=1
 pkgdesc='A self-hosted, synchronized password manager'
 url='https://github.com/rwinkhart/sshyp'
@@ -9,10 +9,10 @@ arch=('x86_64' 'aarch64')
 license=('GPL3')
 depends=( python gnupg openssh nano xclip wl-clipboard)
 
-source_x86_64=('https://github.com/rwinkhart/sshyp/releases/download/v2021.11.05.fr1/sshyp-2021.11.05.fr1.tar.xz')
-source_aarch64=('https://github.com/rwinkhart/sshyp/releases/download/v2021.11.05.fr1/sshyp-2021.11.05.fr1.tar.xz')
-sha512sums_x86_64=('dcc2fe2e751120ffed86cad5f170bdf6ba4ef96df4c4cc784496289c7fda1ccef5ded030ce20f4ea6571a0def5bc185037b39f0958958a1aadb1ead63291b879')
-sha512sums_aarch64=('dcc2fe2e751120ffed86cad5f170bdf6ba4ef96df4c4cc784496289c7fda1ccef5ded030ce20f4ea6571a0def5bc185037b39f0958958a1aadb1ead63291b879')
+source_x86_64=('https://github.com/rwinkhart/sshyp/releases/download/v2021.12.01.fr2/sshyp-2021.12.01.fr2.tar.xz')
+source_aarch64=('https://github.com/rwinkhart/sshyp/releases/download/v2021.12.01.fr2/sshyp-2021.12.01.fr2.tar.xz')
+sha512sums_x86_64=('88b1db43a0234e00cc9b2b5aa6aff642a350109e101a1ebc95e06e7ebc294b09c8369eead06fd16bbffd1f6ce564fb1e8736685ee716f7806532710d7357a5cb')
+sha512sums_aarch64=('88b1db43a0234e00cc9b2b5aa6aff642a350109e101a1ebc95e06e7ebc294b09c8369eead06fd16bbffd1f6ce564fb1e8736685ee716f7806532710d7357a5cb')
 
 package() {
 

README.md

@@ -50,11 +50,11 @@ Packaging for other distributions coming soon.
 Upon initial installation, be sure to run:
 
 ```
-sshyp config
+sshyp tweak
 ```
 
 This command will allow you to configure the settings necessary for sshyp to function.
-As of right now, sshyp is rapidly changing, and as such, it is a good idea to run "sshyp config" after each update.
+As of right now, sshyp is rapidly changing, and as such, it is a good idea to run "sshyp tweak" after each update.
 
 All available options can be found with:
 
@@ -65,7 +65,6 @@ sshyp --help
 # Roadmap
 Short-term Goals:
 
-- re-write bad, leftover "rpass" code
 - create new file list w/color and w/o file extensions
 - create a man page
 - overhaul argument system
@@ -74,9 +73,18 @@ Short-term Goals:
 
 Medium-term Goals:
 
-- create minimal GUI apps (Linux x86_64, Linux aarch64, Android aarch64)
+- create minimal GUI apps (Linux x86_64, Linux aarch64)
 - auto-completion on tab for CLI version
 
 Long-term Goals:
 
 - seize the thrones, shear the humans
+
+# Known Issues
+Broken Features in Latest Release:
+
+- the 'gen' function uses the older notetaking system
+
+Broken Features in Source:
+
+- the 'gen' function uses the older notetaking system

bin/sshync.py

@@ -1,216 +1,106 @@
 #!/usr/bin/python3
 
-# Main Targets:
-# TODO Make it so that when a new folder is created in sshyp, it is automatically created on the remote server
-# TODO Replace references to sshyp directory with a changeable variable
-# TODO Clean everything up and move sshync to a separate package
-# TODO Add dry run support
-# TODO use partition command instead of split for separating paths
-# TODO use classes!
-
-# sshync 2021.11.05.unreleased5
-# sshync was created to solve a problem with rpass/sshyp, and as such, sshync will be bundled with sshyp until it is
-# polished enough for a standalone release
-
-# sshync is a wrapper around sftp that allows easy remote directory synchronization via Python
-# openssh (on POSIX compliant Unix) is required
-
-# sshync will always overwrite the older files with the newer files, regardless of all other conditions
-#   ^the age is determined by the date and time of the last modification to the files
-
-# syncing via sshync is always done recursively
+# sshync 2021.12.01.unreleased6
 
 # external modules
 
-from os import walk
-from os import system as term
-from os.path import getmtime
-from os.path import join as path_join
+from os import system, remove, walk
+from os.path import getmtime, join
+from sys import exit as s_exit
 
 
-# internal modules
+# utility functions
+
+def get_titles_mods(_directory, _destination, _user_data):
+    _title_list, _mod_list = [], []
+    # local fetching
+    if _destination == 'l':
+        if type(_user_data) == str:
+            _user_data = str(_user_data).strip('(').strip(')').replace(' ', '').split(',')
+        remove(_user_data[6] + 'sshync_database')
+        for _root, _directories, _files in walk(_directory):
+            for _filename in _files:
+                _title_list.append(join(_root.replace(_directory, '', 1), _filename))
+                _mod_list.append(int(getmtime(join(_root, _filename))))
+        i = -1
+        for _title in _title_list:
+            open(_user_data[6] + 'sshync_database', 'a').write(str(_title) + '\n')
+        open(_user_data[6] + 'sshync_database', 'a').write('^&*\n')
+        for _mod in _mod_list:
+            open(_user_data[6] + 'sshync_database', 'a').write(str(_mod) + '\n')
+    # remote fetching
+    if _destination == 'r':
+        system(f"ssh -i '{_user_data[5]}' -p {_user_data[2]} {_user_data[0]}@{_user_data[1]} \"python -c 'import sshync"
+               f"; sshync.get_titles_mods(\"'\"{_user_data[4]}\"'\", \"'\"l\"'\", \"'\"{_user_data}\"'\")'\"")
+        system(f"sftp -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:'{_user_data[6]}sshync_database' "
+               f"'{_user_data[6]}sshync_database'")
+        _titles, _sep, _mods = ' '.join(open(_user_data[6] + 'sshync_database').readlines()).replace('\n', '').partition('^&*')
+        _title_list, _mod_list = _titles.split(' ')[:-1], _mods.split(' ')[1:]
+    return _title_list, _mod_list
 
 
-# recursively scans a local directory for all files and saves file paths and mod times to lists
-def get_file_paths_mod(directory, name_database, time_database):
-    file_paths = []
-    temp_mod_times = []
-    mod_times = []
-    for root, directories, files in walk(directory):
-        for filename in files:
-            filepath = path_join(root, filename)
-            file_paths.append(filepath)
-            temp_mod_times.append(str(getmtime(filepath)))
-    for time in temp_mod_times:
-        mod_times.append(round(float(time), 2))
-    open(name_database, 'w').write(str(file_paths))
-    open(time_database, 'w').write(str(mod_times))
-    return file_paths, mod_times
+def sort_titles_mods(_list_1, _list_2):
+    _title_list_2_sorted, _mod_list_2_sorted = [], []
+    # title sorting
+    for _title in _list_1[0]:
+        if _title in _list_2[0]:
+            _title_list_2_sorted.append(_title)
+    for _title in _list_2[0]:
+        if _title not in _title_list_2_sorted:
+            _title_list_2_sorted.append(_title)
+    # mod time sorting
+    for _title in _title_list_2_sorted:
+        _mod_list_2_sorted.append(_list_2[1][_list_2[0].index(_title)])
+    return _title_list_2_sorted, _mod_list_2_sorted
 
 
-# recursively scans a remote directory for all files and saves file paths and mod times to lists
-def get_file_paths_mod_remote(profile_dir):
-    # import profile data
-    profile_data = get_profile(profile_dir)
-    user = profile_data[0].replace('\n', '')
-    ip = profile_data[1].replace('\n', '')
-    port = profile_data[2].replace('\n', '')
-    remote_dir = profile_data[4].replace('\n', '')
-    identity = profile_data[5].replace('\n', '')
-    term("ssh -i " + "'" + identity + "'" + " -p " + port + " " + user + "@" + ip + " \"python -c \'import sshync; "
-                                                                                    "sshync.get_file_paths_mod(" +
-         "\"'\"" + remote_dir + "/\"'\", " + "\"'\"/var/lib/sshyp/sshyncdatabase_names\"'\", " +
-         "\"'\"/var/lib/sshyp/sshyncdatabase_times\"'\")\'\"")
-    term(
-        'sftp -P ' + port + ' ' + user + '@' + ip + ':/var/lib/sshyp/sshyncdatabase_names ' +
-        '/var/lib/sshyp/sshyncdatabase_names')
-    term(
-        'sftp -P ' + port + ' ' + user + '@' + ip + ':/var/lib/sshyp/sshyncdatabase_times ' +
-        '/var/lib/sshyp/sshyncdatabase_times')
-    remote_mod_names = (open('/var/lib/sshyp/sshyncdatabase_names').read()).replace('[', '').replace(']', '').replace(
-        ' ', '').replace("'", '').split(',')
-    remote_mod_times = (open('/var/lib/sshyp/sshyncdatabase_times').read()).replace('[', '').replace(']', '').replace(
-        ' ', '').replace("'", '').split(',')
-    return remote_mod_names, remote_mod_times
+def make_profile(_profile_dir, _local_dir, _remote_dir, _identity, _ip, _port, _user):
+    open(_profile_dir, 'w').write(_user + '\n' + _ip + '\n' + _port + '\n' + _local_dir + '\n' + _remote_dir + '\n' +
+                                  _identity + '\n')
 
 
-# regular profiles check the mod time of every individual file in the directory, making them slow but versatile
-def make_profile(profile_dir, local_dir, remote_dir, identity, ip, port, user):
-    open(profile_dir, 'w').write(user + '\n' + ip + '\n' + port + '\n' + local_dir + '\n' + remote_dir + '\n' + identity
-                                 + '\n')
-
-
-# returns the settings for a specified profile
-def get_profile(profile_dir):
+def get_profile(_profile_dir):
     try:
-        profile_data = open(profile_dir).readlines()
+        _profile_data = open(_profile_dir).readlines()
     except (FileNotFoundError, IndexError):
-        profile_data = 0
-    return profile_data
+        print('\nEither the profile does not exist, or it is corrupted.\n')
+        _profile_data = None
+        s_exit()
+    # extract data from profile
+    _user = _profile_data[0].replace('\n', '')
+    _ip = _profile_data[1].replace('\n', '')
+    _port = _profile_data[2].replace('\n', '')
+    _local_dir = _profile_data[3].replace('\n', '')
+    _remote_dir = _profile_data[4].replace('\n', '')
+    _identity = _profile_data[5].replace('\n', '')
+    _data_dir = ' '.join(_profile_dir.split('/')[:-1]).replace(' ', '/') + '/'
+    return _user, _ip, _port, _local_dir, _remote_dir, _identity, _data_dir
 
 
-# sorts name lists so that local and remote are in the same order
-def name_sort(local_names, remote_names):
-    remote_names_new = []
-    for ele in local_names:
-        if ele in remote_names:
-            remote_names_new.append(ele)
-    for ele in remote_names:
-        if ele not in remote_names_new:
-            remote_names_new.append(ele)
-    return remote_names_new
-
-
-# sorts time lists to compensate for name list re-ordering
-def time_sort(old_names, new_names, old_times):
-    remote_mod_times_new = []
-    new_pos = -1
-    for title in new_names:
-        new_pos += 1
-        if title in old_names:
-            old_pos = old_names.index(title)
-            try:
-                remote_mod_times_new.append(old_times[old_pos])
-            except IndexError:
-                pass
-    return remote_mod_times_new
-
-
-getter_loop_count = -1
-
-
-def loop_getter(split_remote_dir):
-    global getter_loop_count
-    getter_loop_count += 1
-    new_var = split_remote_dir[getter_loop_count] + '/'
-    return new_var
-
-
-def run_profile(profile_dir):
-    # import profile data
-    profile_data = get_profile(profile_dir)
-    user = profile_data[0].replace('\n', '')
-    ip = profile_data[1].replace('\n', '')
-    port = profile_data[2].replace('\n', '')
-    local_dir = profile_data[3].replace('\n', '')
-    remote_dir = profile_data[4].replace('\n', '')
-    identity = profile_data[5].replace('\n', '')
-    # get file modification times
-    local_mod_names, local_mod_times = get_file_paths_mod(local_dir + '/', '/var/lib/sshyp/sshyncdatabase_names',
-                                                          '/var/lib/sshyp/sshyncdatabase_times')
-    remote_mod_times = []
-    remote_mod_names, remote_mod_times_temp = get_file_paths_mod_remote('/var/lib/sshyp/sshyp.sshync')
-    try:
-        for time in remote_mod_times_temp:  # remove quotes from remote_mod_times
-            remote_mod_times.append(float(time))
-    except ValueError:
-        pass
-    lmod_short = []
-    rmod_short = []
-    matching_names = []
-    matching_times_local = []
-    matching_times_remote = []
-    for lmod in local_mod_names:
-        lmod_short.append(lmod.replace(local_dir, ''))
-    for rmod in remote_mod_names:
-        rmod_short.append(rmod.replace(remote_dir, ''))
-    rmod_short_old = rmod_short
-    rmod_short = name_sort(lmod_short, rmod_short)
-    remote_mod_times = time_sort(rmod_short_old, rmod_short, remote_mod_times)
-    i = -1
-    for lshort in lmod_short:
-        i += 1
-        if lshort in rmod_short:
-            matching_names.append(lshort)
-            matching_times_local.append(local_mod_times[i])
+def run_profile(_profile_dir):
+    _user_data = get_profile(_profile_dir)
+    _remote_titles_mods_saver = get_titles_mods(_user_data[4], 'r', _user_data)  # saved to prevent re-walking directory
+    _index_l = sort_titles_mods(_remote_titles_mods_saver, get_titles_mods(_user_data[3], 'l', _user_data))
+    _index_r = sort_titles_mods(_index_l, _remote_titles_mods_saver)
+    _i = -1
+    for _title in _index_l[0]:
+        _i += 1
+        if _title in _index_r[0]:
+            # compare mod times and sync
+            if int(_index_l[1][_i]) > int(_index_r[1][_i]):
+                print(f"[{_title}] Local is newer, uploading...")
+                system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:"
+                       f"{_user_data[4]}{_title} <<< $'put {_user_data[3]}{_title}'")
+            elif int(_index_l[1][_i]) < int(_index_r[1][_i]):
+                print(f"[{_title}] Remote is newer, downloading...")
+                system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:"
+                       f"{_user_data[4]}{_title} {_user_data[3]}{_title}")
         else:
-            print('Uploading ' + lshort)
-            lshort_split = lshort.replace('/', '', 1).split('/')
-
-            c = -1
-            for _ in lshort_split:
-                c += 1
-
-            global getter_loop_count
-            getter_loop_count = -1
-            q = ''
-            for _ in range(c):
-                q += loop_getter(lshort_split)
-
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir + q +
-                 " <<< $'put " + local_dir + lshort.replace('/', '', 1) + "'")
-    i = -1
-    for rshort in rmod_short:
-        i += 1
-        if rshort in lmod_short:
-            matching_times_remote.append(float(remote_mod_times[i]))
-        else:
-            print('Downloading ' + rshort)
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir +
-                 rshort.replace('/', '', 1) + ' ' + local_dir + rshort.replace('/', '', 1))
-    i = -1
-    for ltime in matching_times_local:
-        i += 1
-        ltime = int(ltime)
-        if ltime == int(matching_times_remote[i]):
-            print('Local ' + matching_names[i] + ' is up to date, not syncing.')
-        elif ltime > int(matching_times_remote[i]):
-            print('Local ' + matching_names[i] + ' is newer, uploading.')
-            matching_names_split = matching_names[i].replace('/', '', 1).split('/')
-
-            c = -1
-            for _ in matching_names_split:
-                c += 1
-
-            getter_loop_count = -1
-            q = ''
-            for _ in range(c):
-                q += loop_getter(matching_names_split)
-
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir + q +
-                 " <<< $'put " + local_dir + matching_names[i].replace('/', '', 1) + "'")
-
-        elif ltime < int(matching_times_remote[i]):
-            print('Local ' + matching_names[i] + ' is older, downloading.')
-            term('sftp -p -i ' + "'" + identity + "' " + '-P ' + port + ' ' + user + '@' + ip + ':' + remote_dir +
-                 matching_names[i].replace('/', '', 1) + ' ' + local_dir + matching_names[i].replace('/', '', 1))
+            print(f"[{_title}] Not on remote server, uploading...")
+            system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:{_user_data[4]}"
+                   f"{_title} <<< $'put {_user_data[3]}{_title}'")
+    for _title in _index_r[0]:
+        if _title not in _index_l[0]:
+            print(f"[{_title}] Not in local directory, downloading...")
+            system(f"sftp -p -i '{_user_data[5]}' -P {_user_data[2]} {_user_data[0]}@{_user_data[1]}:{_user_data[4]}"
+                   f"{_title} {_user_data[3]}{_title}")

extra/changelog-total

@@ -1,5 +1,64 @@
 <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
 
+sshyp 2021.12.01.fr2
+
+The Lighter Update
+
+This release features a major re-write of old "rpass" code.
+sshyp is now more reliable and better documented.
+
+New features:
+
+- a full re-write of the old "rpass" code used in the last release
+    ^ includes more reliable argument parsing and various optimizations
+- a full re-write of "sshync", many issues fixed
+    ^fixed many instances where sshync would refuse to upload/download
+    ^fixed errors that were being thrown despite sshync working as intended
+- new folders are now also created on the server
+
+Changes:
+
+- nested folders actually work now
+- multi-word entry and folder titles are fixed
+
+An fr2.1 patch is planned for this version. It will include the following:
+
+- fixes for any new bugs that may be discovered in the new sshyp and sshync code
+- the gen command in sshyp will be updated to use the newer notetaking system
+- printing functions in sshyp will be combined into one function (arguments will determine section to print)
+- file not found exceptions will be re-implemented
+- the argument parser will be moved to the global process (no need for it to be in a function)
+- folders will be created natively in python
+
+Planned for next major update (The Fluffier Update):
+
+- new visual features
+    ^ this includes the new file list and thematic text art. sshyp will be
+    getting a retro computing theme.
+- packages for more environments
+    ^ "sshyp" will be officially packaged for Arch Linux, Alpine Linux, and Termux (Android).
+    Debian/Ubuntu and Fedora packaging will come later.
+- imporove password generator to guarantee more secure results
+
+Planned for next, next major update (The Shears Update Pt. 1):
+
+- allow for copying entire notes (not just first line)
+- allow for multi-client deletion... somehow
+- detatch sshync and make it a separate package
+- manpage
+
+Planned for next, next, next major update (The Shears Update Pt. 2):
+
+- ability to pass entries as arguments for more functions
+- auto-completion on tab (currently unsure of best way to make this work)
+
+Backlog:
+
+- package for more Linux distributions, such as Debian and Fedora
+- create separate gui frontend targetting mobile and desktop Linux
+
+<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+
 sshyp 2021.11.05.fr1
 
 First public release of sshyp.

share/doc/sshyp/changelog

@@ -1,49 +1,61 @@
-sshyp 2021.11.05.fr1
+sshyp 2021.12.01.fr2
 
-First public release of sshyp.
+The Lighter Update
 
-This release fixes major bugs from rpass and rebrands the project.
+This release features a major re-write of old "rpass" code.
+sshyp is now more reliable and better documented.
 
 New features:
 
-- an entirely new syncing back-end called sshync (custom Python wrapper around sftp)
-    ^ in thoery, all syncing-related data loss should now be fixed, however, this new
-    system has just been created and is still under testing. Maybe wait for a few patches
-    before trusting this system.
+- a full re-write of the old "rpass" code used in the last release
+    ^ includes more reliable argument parsing and various optimizations
+- a full re-write of "sshync", many issues fixed
+    ^fixed many instances where sshync would refuse to upload/download
+    ^fixed errors that were being thrown despite sshync working as intended
+- new folders are now also created on the server
 
 Changes:
 
-- branding has been updated to reflect new project name and goals
-- custom directories, custom ssh keys, and offline only mode are removed (may be temporary)
-    ^ the configuration wizard is being re-written in the next major update
+- nested folders actually work now
+- multi-word entry and folder titles are fixed
 
-Planned for next update (The Fluffy Update):
+An fr2.1 patch is planned for this version. It will include the following:
+
+- fixes for any new bugs that may be discovered in the new sshyp and sshync code
+- the gen command in sshyp will be updated to use the newer notetaking system
+- printing functions in sshyp will be combined into one function (arguments will determine section to print)
+- file not found exceptions will be re-implemented
+- the argument parser will be moved to the global process (no need for it to be in a function)
+- folders will be created natively in python
+
+Planned for next major update (The Fluffier Update):
 
-- modularized and optimized code
-    ^ some older, bad code from "rpass" will be re-written.
 - new visual features
     ^ this includes the new file list and thematic text art. sshyp will be
     getting a retro computing theme.
 - packages for more environments
     ^ "sshyp" will be officially packaged for Arch Linux, Alpine Linux, and Termux (Android).
     Debian/Ubuntu and Fedora packaging will come later.
+- imporove password generator to guarantee more secure results
 
-Planned for next update (The Sheep w/Shears Update):
+Planned for next, next major update (The Shears Update Pt. 1):
 
 - allow for copying entire notes (not just first line)
 - allow for multi-client deletion... somehow
-- ability to pass entries as arguments for more functions
 - detatch sshync and make it a separate package
 - manpage
 
-Planned, no timeline:
+Planned for next, next, next major update (The Shears Update Pt. 2):
 
-- create separate gui frontend targetting mobile devices (Android + Linux phones)
-    ^ the gui will also function on desktop linux
-- package for more Linux distributions, such as Debian and Fedora)
+- ability to pass entries as arguments for more functions
 - auto-completion on tab (currently unsure of best way to make this work)
 
+Backlog:
+
+- package for more Linux distributions, such as Debian and Fedora
+- create separate gui frontend targetting mobile and desktop Linux
+
 <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
 
-The full history of changes to "sshyp" and "rpass" can be found on the following page:
+The full history of changes to "sshyp" can be found on the following page:
 https://raw.githubusercontent.com/rwinkhart/sshyp/main/extra/changelog-total