initial commit

.gitignore

@@ -0,0 +1 @@
+__pycache__/

README.md

@@ -1,3 +1,124 @@
 # encrypted-tables
 
-Operate an ordered sequence of dicts, referred here as a table, in the command line and store them (optionally encrypted) on the filesystem.
+Operate an ordered sequence of dicts (referred here as a table) in the command line, and store them in the filesystem.
+
+## Description
+
+Currently, we support base64 encoded (no encryption) or the fernet symmetric entryption algorithm.
+
+Warning: Storing in base64 format without encryption provides a false sense of security and should be avoided, it is kept for backward compatibility only.
+
+The symmetric key is obtained by a key derivation function taking your password and a randomly-generated salt as input.
+
+I don't know why anyone would need this, why not use a database?
+
+## How to run
+
+Environment: Python 3.8 (other versions might work as well), cryptography==39.0.1
+
+### Basic usage
+
+#### 1.Create a new table
+
+`python peek.py example fernet --create`. You will be prompted to enter a password for this table first.
+
+The "table" in this program is different from the table in the database sense, it is an ordered sequence of rows, where each row is a dictionary (collection of key-value pairs).
+
+#### 2. Add data rows and printing the table
+
+Enter kv pairs to initialize / add a row. Use command 'a' to add row and 'p' to print rows.
+```
+>a
+k:name
+v:alice
+k:sport
+v:skiing
+k:
+>a
+k:name
+v:bob
+k:sport
+v:cycling
+k:
+>p
+0
+name:alice  sport:skiing
+1
+name:bob    sport:cycling
+```
+
+#### 3. Modify a row / delete a row
+
+Use command 'm' to modify a row (add new kv pairs or overwrite existing kv pairs); use 'd' to delete a row by its current index in the table.
+```
+>m
+index:1
+k:lang
+v:en
+k:
+>d
+index:0
+deleting 1:name:alice   sport:skiing
+>p
+0
+name:bob    sport:cycling   lang:en
+```
+
+#### 4. Saving, quitting, and reopening
+
+Use the command 'wq' to save and quit, 'q!' to discard changes and quit.
+```
+>wq
+# python peek.py example fernet
+password:<yourpassword>
+>p
+name:bob    sport:cycling   lang:en
+```
+
+### Using shortcuts
+
+Key names must have at least 2 characters; you can setup and use single-character shortcuts to refer to the key name.
+
+#### Defining shortcuts
+
+Use 'sc' to define shortcuts; Running this command will overwrite all existing shortcuts, you must exit and re-enter the program for the shortcut to take effect.
+```
+>sc
+{}
+k:n
+v:name
+k:s
+v:sport
+k:
+>wq
+```
+
+#### Using shortcuts
+
+You can use shortcuts to add rows more efficiently.
+```
+>a
+k:n
+v:carol
+k:s
+v:swimming
+```
+
+#### Format print using shortcuts
+
+You can print the table in the format specified using the command 'f'. It prints an table with the keys ordered by the string you give (use shortcut to represent keys); only rows that have all the keys available will be printed. use '+' to print any additional keys in each row.
+
+```
+>f
+ns
+|#|name|sport|
+|---|---|---|
+|0|bob|cycling|
+|1|carol|swimming|
+>f
+keys:sn+
+|#|sport|name|+|
+|---|---|---|---|
+|0|cycling|bob|lang:en;|
+|1|swimming|carol||
+```

encrypt_json.py

@@ -0,0 +1,98 @@
+import os
+import json
+import base64
+from cryptography.fernet import Fernet, InvalidToken
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+# https://cryptography.io/en/latest/fernet/#using-passwords-with-fernet
+
+rescue_msg = {
+    "readme": "The salt (bytes) and the encrypted object (bytes) are both stored in base64 format.\
+               The key, 32 bytes long, is derived using PBKDF2 using the salt and your password.\
+               Decrypt the object using the AES-CBC algorithm with the key.",
+    "code_to_decrypt": """
+                import base64
+                from cryptography.fernet import Fernet
+                from cryptography.hazmat.primitives import hashes
+                from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+                def s2b(s:str)->bytes:
+                    return base64.decodebytes(s.encode('utf-8'))
+                salt = s2b(save_obj["salt"])
+                obj_encrypted = s2b(save_obj["obj"])
+                password = input("password:").encode('utf-8')
+                kdf = PBKDF2HMAC(
+                        algorithm=hashes.SHA256(),
+                        length=32,
+                        salt=salt,
+                        iterations=480000,
+                )
+                key = base64.urlsafe_b64encode(kdf.derive(password))
+                obj_decrypted = Fernet(key).decrypt(obj_encrypted).decode('utf-8')
+                print(obj_decrypted)
+            """
+}
+
+def get_fernet(password:bytes, salt:bytes):
+    kdf = PBKDF2HMAC(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=salt,
+        iterations=480000,
+    )
+    key = base64.urlsafe_b64encode(kdf.derive(password))
+    return Fernet(key)
+
+def b2s(b:bytes)->str:
+    return base64.encodebytes(b).decode('utf-8')
+
+def s2b(s:str)->bytes:
+    # s must be constructed from b2s
+    return base64.decodebytes(s.encode('utf-8'))
+
+def save_obj(obj, password:bytes, salt:bytes) -> str:
+    f = get_fernet(password, salt)
+    save_obj = dict()
+    save_obj["salt"] = b2s(salt)
+    obj_encrypted = f.encrypt(json.dumps(obj).encode('utf-8'))
+    save_obj["obj"] = b2s(obj_encrypted)
+    save_obj["rescue"] = rescue_msg
+    return json.dumps(save_obj,indent=4)
+
+def load_obj(s, password):
+    save_obj = json.loads(s)
+    salt = s2b(save_obj["salt"])
+    obj_encrypted = s2b(save_obj["obj"])
+    f = get_fernet(password, salt)
+    try:
+        obj_decrypted = f.decrypt(obj_encrypted)
+    except InvalidToken:
+        print("Invalid Password")
+        exit(-1)
+    obj = json.loads(obj_decrypted.decode('utf-8'))
+    return obj
+
+def perform_encrypt(infile, outfile):
+    password = input("password:").encode('utf-8')
+    salt = os.urandom(16)
+    obj = json.load(open(infile,'r'))
+    d = save_obj(obj, password, salt)
+    f = open(outfile,'w'); f.write(d); f.close()
+
+def perform_decrypt(infile, outfile):
+    f = open(infile,'r'); d=f.read(); f.close()
+    password = input("password:").encode('utf-8')
+    obj = load_obj(d, password)
+    json.dump(obj, open(outfile,'w'), indent=4)
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('mode', choices=['e', 'd'], help="e for encrypt, d for decrypt")
+    parser.add_argument('infile')
+    parser.add_argument('outfile')
+    args = parser.parse_args()
+    if args.mode == 'e':
+        perform_encrypt(args.infile, args.outfile)
+    elif args.mode == 'd':
+        perform_decrypt(args.infile, args.outfile)

peek.py

@@ -0,0 +1,197 @@
+"""
+Peek:
+Operate on a list of dicts in the cmdline and store them (optionally encrypted) on the filesystem
+I AM NOT A SECURITY EXPERT. NO GUARANTEE. USE AT YOUR OWN RISK
+"""
+
+import os
+import json
+import base64
+from encrypt_json import load_obj, save_obj
+
+def base64_encode(objs) -> str:
+    # Storing in the illegible base64 format instead of clear text creates
+    # A FALSE SENSE OF DATA SECURITY, SHOULD BE DISABlED
+
+    # In Python, bytes objects are immutable sequences of single bytes
+    # string <-e/d-> bytes <--base64 e/d--> bytes <-e/d-> string
+    s = json.dumps(objs)
+    return base64.encodebytes(s.encode('utf-8')).decode('utf-8')
+
+def base64_decode(s:str):
+    s = base64.decodebytes(s.encode('utf-8')).decode('utf-8')
+    return json.loads(s)
+
+def printraw(mem):
+    for i,obj in enumerate(mem):
+        print(i)
+        for k,v in obj.items():
+            print(k+":"+v,end='\t')
+        print('')
+
+def printtable(table):
+    # Prints a table in markdown style
+    table.insert(1,["---"]*len(table[0]))
+    for row in table:
+        print("|" + "|".join(row) + "|")
+
+def printfmt(mem, keys, shortcuts):
+    # Parse which keys are required
+    key_order = list()
+    list_others = False
+    keys = keys.split()
+    for c in keys[0]:
+        if c == '+':
+            list_others = True
+        elif c in shortcuts.keys():
+            key_order.append(shortcuts[c])
+        else:
+            print("unknown character shortcut '%s', cannot print formatted"%c)
+            return
+    key_order.extend(keys[1:])
+    # List dicts that has the required keys
+    table = list()
+    table.append(["#"] + key_order + (["+"] if list_others else []))
+    for i,obj in enumerate(mem):
+        if all([key in obj for key in key_order]):
+            row = [str(i)]
+            for key in key_order:
+                row.append(obj[key])
+            if list_others:
+                s = ""
+                for k,v in obj.items():
+                    if k not in key_order:
+                        s += k+":"+v+";"
+                row.append(s)
+            table.append(row)
+    printtable(table)
+
+def inputkv(obj, shortcuts):
+    while True:
+        k = input("k:")
+        if k=='':
+            break # end of input
+        if len(k)==1:
+            try:
+                k = shortcuts[k]
+            except KeyError:
+                print("unknown character shortcut %s, one-character key is reserved for shortcuts, \
+                        choose another key"%k)
+                continue # invalid input
+        if k in obj:
+            print("rewriting key %s, original value is "%k + obj[k])
+        v = input("v:")
+        obj[k] = v
+
+def inputshortcuts():
+    sc = {}
+    while True:
+        k = input("k:")
+        if k=='':
+            break
+        if len(k)!=1:
+            print("shortcut must be one-character")
+            continue
+        v = input("v:")
+        sc[k] = v
+    return sc
+
+def loaddb(db_name, encode_method, password):
+    if encode_method=="base64":
+        try:
+            f = open(db_name+".store",'r')
+            obj = base64_decode(f.read())
+            f.close()
+        except Exception as e:
+            print(type(e).__name__ + ":" + str(e))
+            print("Corrupted save file")
+            exit(-1)
+    elif encode_method=="fernet":
+        f = open(db_name+".json",'r')
+        d=f.read(); f.close()
+        obj = load_obj(d, password)
+    else:
+        print("unsupported encode method, exiting")
+        exit(-1)
+    return obj
+
+def savedb(db_name, encode_method, obj, password):
+    if encode_method=="base64":
+        f = open(db_name+".store",'w')
+        f.write(base64_encode(obj))
+    elif encode_method=="fernet":
+        f = open(db_name+".json",'w')
+        salt = os.urandom(16)
+        d = save_obj(obj, password, salt)
+        f.write(d)
+    else:
+        print("unsupported encode method, exiting")
+    f.close()
+
+def interact(db_name, decode_method, encode_method, password, create):
+    db_obj = {"mem":[], "shortcuts":{}} if create else loaddb(db_name, decode_method, password)
+    if type(db_obj)==list:
+        db_obj = {"mem":db_obj, "shortcuts":{}}
+    mem, shortcuts = db_obj["mem"], db_obj["shortcuts"]
+    # Main loop
+    help_msg = "p: Print Raw, f: Print Formatted, a: Add, m: Modify, d: Delete,\
+                wq: Save & Quit, q!: Don't Save, sc: define shortcuts"
+    print(help_msg)
+    modified = False
+    cmd = ""
+    while True:
+        cmd = input('>')
+        if cmd=='p':
+            printraw(mem)
+        elif cmd=='f':
+            keys = input("keys:")
+            printfmt(mem, keys, shortcuts)
+        elif cmd=='a':
+            modified = True
+            obj = dict()
+            inputkv(obj, shortcuts)
+            mem.append(obj)
+        elif cmd=='m':
+            modified = True
+            pos = int(input("index:"))
+            inputkv(mem[pos], shortcuts)
+        elif cmd=='d':
+            modified = True
+            pos = int(input("index:"))
+            print("deleting %d: "%pos + '\t'.join([k+":"+v for k,v in mem[pos].items()]))
+            del mem[pos]
+        elif cmd=='q':
+            if modified:
+                print("Memory has been modified, to save, use 'wq', to exit without save, use 'q!'")
+            else:
+                break
+        elif cmd=='wq':
+            savedb(db_name, encode_method, db_obj, password)
+            break
+        elif cmd=='q!':
+            break
+        elif cmd=="sc":
+            print(shortcuts)
+            db_obj["shortcuts"] = inputshortcuts()
+        else:
+            print("Invalid Command, "+help_msg)
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description='Operate on a list of dicts in the cmdline and\
+                                    store them encrypted on the filesystem')
+    parser.add_argument('db_name')
+    parser.add_argument('decode_method')
+    parser.add_argument('encode_method', nargs="?")
+    parser.add_argument('--create', action='store_true')
+    # db_pass should not be passed in as cmdline args as they are logged and appear in .bash_history
+    args = parser.parse_args()
+    if args.encode_method==None:
+        args.encode_method = args.decode_method
+    if args.create and (os.path.isfile(args.db_name+'.store') or os.path.isfile(args.db_name+'.json')):
+        print("Cannot overwrite existing db")
+        exit(-1)
+    password=None
+    if args.decode_method=="fernet" or args.encode_method=="fernet":
+        password = input("password:").encode('utf-8')
+    interact(args.db_name, args.decode_method, args.encode_method, password, args.create)

requirements.txt

@@ -0,0 +1 @@
+cryptography==39.0.1