Update framework

.gitignore

@@ -9,4 +9,6 @@ cool_query.py
 __pycache__
 future_plans.*
 secret_trading_tools.py
-*.bak
+*.bak*
+orderer.zip
+*.db-*

Makefile

@@ -1,13 +1,13 @@
-
-.PHONY: compile
-compile:
-	echo "IMPORTANT: Make sure the client is not currently running."
-	cp debug.py debug.py.bak
-	echo debug = False > debug.py
-	python -OO -m PyInstaller run_client.py -y
-	rm debug.py
-	mv debug.py.bak debug.py
-	cd dist; \
-	"C:\Program Files\7-Zip\7z.exe" a \
-	orderer.zip \
-	run_client\*
+
+.PHONY: compile
+compile:
+	echo "IMPORTANT: Make sure the client is not currently running."
+	cp debug.py debug.py.bak
+	echo debug = False > debug.py
+	python -O -m PyInstaller run_client.py -y
+	rm debug.py
+	mv debug.py.bak debug.py
+	cd dist; \
+	"C:\Program Files\7-Zip\7z.exe" a \
+	../frontend/orderer.zip \
+	run_client\*

README.md

@@ -1,36 +1,38 @@
-# README
-
-## How to play
-### Windows (64 Bit only???)
-You can either 
-- (Recommended) download a PowerShell-script that automatically installs and/or updates Orderer from
-http://koljastrohm-games.com/downloads/orderer_installer.zip
-- or download the files for the current client from 
-http://koljastrohm-games.com/downloads/orderer.zip
-- clone this repository, install python (3.6 recommended) and follow the instructions below for python users.
-
-### Any machine capable of running python 3
-You can clone this repository and use python (3.6 recommended) to execute the client.
-Make sure you set the `debug` flag in `debug.py` to `False`, otherwise you will not connect to the server.
-
-If you know python and want to develop any cool features for you personal client, feel free to fork this repository and create a pull request.
-
-## Compilation
-### Server
-The server is intended to be run within a python 3.6 environment on the host specified in `connection.py`.
-
-On the server you can install the required packages using pip:
-```
-pip3 install bottle requests tabulate
-```
-and start the server using
-```
-python3 -OO run_server.py
-```
-
-### Client
-The client is intended to be compiled with
-```
-make compile
-```
-and requires `pyinstaller` which can be installed using pip.
+# README
+
+## How to play
+### Windows (64 Bit only???)
+You can either 
+- (Recommended) download a PowerShell-script that automatically installs and/or updates Orderer from
+http://koljastrohm-games.com/downloads/orderer_installer.zip
+- or download the files for the current client from 
+http://koljastrohm-games.com/downloads/orderer.zip
+- clone this repository, install python (3.6 recommended) and follow the instructions below for python users.
+
+### Any machine capable of running python 3
+You can clone this repository and use python (3.6 recommended) to execute the client.
+Make sure you set the `debug` flag in `debug.py` to `False`, otherwise you will not connect to the server.
+
+If you know python and want to develop any cool features for you personal client, feel free to fork this repository and create a pull request.
+
+## Compilation
+### Server
+The server is intended to be run within a python 3.6 environment on the host specified in `connection.py`.
+
+On the server you can install the required packages using pip:
+```
+pip3 install bottle requests tabulate
+```
+and start the server using
+```
+python3 -OO run_server.py
+```
+
+### Client
+The client is intended to be compiled with
+```
+make compile
+```
+and requires `pyinstaller` which can be installed using pip.
+If there is a UnicodeDecodeError you may try this fix: 
+`https://stackoverflow.com/questions/47692960/error-when-using-pyinstaller-unicodedecodeerror-utf-8-codec-cant-decode-byt`

client_controller.py

@@ -1,422 +1,424 @@
-import sys
-from getpass import getpass
-from inspect import signature
-
-import connection
-from connection import client_request
-from debug import debug
-from game import DEFAULT_ORDER_EXPIRY
-from run_client import allowed_commands, fake_loading_bar
-from util import my_tabulate, yn_dialog
-
-exiting = False
-
-
-def login(username=None, password=None):
-    if connection.session_id is not None:
-        fake_loading_bar('Signing out', duration=0.7)
-        connection.session_id = None
-
-    if username is None:
-        username = input('Username: ')
-
-    if password is None:
-        if sys.stdin.isatty():
-            password = getpass('Password: ')
-        else:
-            password = input('Password: ')
-
-    fake_loading_bar('Signing in', duration=2.3)
-    response = client_request('login', {"username": username, "password": password})
-    success = 'session_id' in response
-    if success:
-        connection.session_id = response['session_id']
-        print('Login successful.')
-    else:
-        if 'error_message' in response:
-            print('Login failed with message:', response['error_message'])
-        else:
-            print('Login failed.')
-
-
-def register(username=None, game_key='', password=None, retype_pw=None):
-    if connection.session_id is not None:
-        connection.session_id = None
-        fake_loading_bar('Signing out', duration=0.7)
-
-    if username is None:
-        username = input('Username: ')
-
-    if password is None:
-        if sys.stdin.isatty():
-            password = getpass('New password: ')
-            retype_pw = getpass('Retype password: ')
-        else:
-            password = input('New password: ')
-            retype_pw = input('Retype password: ')
-        if password != retype_pw:
-            print('Passwords do not match.')
-            return
-    elif retype_pw is None:
-        if sys.stdin.isatty():
-            retype_pw = getpass('Retype password: ')
-        else:
-            retype_pw = input('Retype password: ')
-        if password != retype_pw:
-            print('Passwords do not match.')
-            return
-
-    if not debug:
-        if game_key == '':
-            print('Entering a game key will provide you with some starting money and other useful stuff.')
-            game_key = input('Game key (leave empty if you don\'t have one): ')
-
-    fake_loading_bar('Validating Registration', duration=5.2)
-    if game_key != '':
-        fake_loading_bar('Validating Game Key', duration=0.4)
-    response = client_request('register', {"username": username, "password": password, "game_key": game_key})
-
-    if 'error_message' in response:
-        print('Registration failed with message:', response['error_message'])
-
-
-def cancel_order(order_no=None):
-    if order_no is None:
-        order_no = input('Order No.: ')
-
-    fake_loading_bar('Validating Request', duration=0.6)
-    response = client_request('cancel_order', {"session_id": connection.session_id, "order_id": order_no})
-
-    if 'error_message' in response:
-        print('Order cancelling failed with message:', response['error_message'])
-
-
-def change_pw(password=None, retype_pw=None):
-    if password != retype_pw:
-        password = None
-    if password is None:
-        if sys.stdin.isatty():
-            password = getpass('New password: ')
-            retype_pw = getpass('Retype password: ')
-        else:
-            password = input('New password: ')
-            retype_pw = input('Retype password: ')
-        if password != retype_pw:
-            print('Passwords do not match.')
-            return
-    elif retype_pw is None:
-        if sys.stdin.isatty():
-            retype_pw = getpass('Retype password: ')
-        else:
-            retype_pw = input('Retype password: ')
-        if password != retype_pw:
-            print('Passwords do not match.')
-            return
-
-    fake_loading_bar('Validating password', duration=1.2)
-    fake_loading_bar('Changing password', duration=5.2)
-    response = client_request('change_password', {"session_id": connection.session_id, "password": password})
-
-    if 'error_message' in response:
-        print('Changing password failed with message:', response['error_message'])
-
-    fake_loading_bar('Signing out', duration=0.7)
-    connection.session_id = None
-
-
-# noinspection PyShadowingBuiltins
-def help():
-    print('Allowed commands:')
-    command_table = []
-    for cmd in allowed_commands:
-        this_module = sys.modules[__name__]
-        method = getattr(this_module, cmd)
-        params = signature(method).parameters
-        command_table.append([cmd] + [p for p in params])
-
-    print(my_tabulate(command_table, tablefmt='pipe', headers=['command',
-                                                               'param 1',
-                                                               'param 2',
-                                                               'param 3',
-                                                               'param 4',
-                                                               'param 5',
-                                                               ]))
-    print('NOTE:')
-    print('  All parameters for all commands are optional!')
-    print('  Commands can be combined in one line with ; between them.')
-    print('  Parameters are separated with whitespace.')
-    print('  Use . as a decimal separator.')
-    print('  Use 0/1 for boolean parameters (other strings might work).')
-
-
-def depot():
-    fake_loading_bar('Loading data', duration=1.3)
-    response = client_request('depot', {"session_id": connection.session_id})
-    success = 'data' in response and 'own_wealth' in response
-    if success:
-        data = response['data']
-        for row in data:
-            row.append(row[1] * row[2])
-        print(my_tabulate(data,
-                          headers=['Object', 'Amount', 'Course', 'Bid', 'Ask', 'Est. Value'],
-                          tablefmt="pipe"))
-        print('This corresponds to a wealth of roughly', response['own_wealth'])
-    else:
-        if 'error_message' in response:
-            print('Depot access failed with message:', response['error_message'])
-        else:
-            print('Depot access failed.')
-
-
-def leaderboard():
-    fake_loading_bar('Loading data', duration=1.3)
-    response = client_request('leaderboard', {})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'], headers=['Trader', 'Wealth'], tablefmt="pipe"))
-        # print('Remember that the goal is to be as rich as possible, not to be richer than other traders!')
-    else:
-        if 'error_message' in response:
-            print('Leaderboard access failed with message:', response['error_message'])
-        else:
-            print('Leaderboard access failed.')
-
-
-def activate_key(key=''):
-    if key == '':
-        print('Entering a game key may get you some money or other useful stuff.')
-        key = input('Key: ')
-
-    if key == '':
-        print('Invalid key.')
-
-    fake_loading_bar('Validating Key', duration=0.4)
-    response = client_request('activate_key', {"session_id": connection.session_id, 'key': key})
-    if 'error_message' in response:
-        print('Key activation failed with message:', response['error_message'])
-
-
-def _order(is_buy_order, obj_name, amount, limit, stop_loss, expiry):
-    if stop_loss not in [None, '1', '0']:
-        print('Invalid value for flag stop loss (only 0 or 1 allowed).')
-        return
-    if obj_name is None:  # TODO list some available objects
-        obj_name = input('Name of object to sell: ')
-    if amount is None:
-        amount = input('Amount: ')
-
-    if limit is None:
-        set_limit = yn_dialog('Do you want to place a limit?')
-        if set_limit:
-            limit = input('Limit: ')
-            stop_loss = yn_dialog('Is this a stop-loss limit?')
-
-    if limit is not None:
-        try:
-            limit = float(limit)
-        except ValueError:
-            print('Invalid limit.')
-            return
-
-        if stop_loss is None:
-            stop_loss = yn_dialog('Is this a stop-loss limit?')
-
-        question = 'Are you sure you want to use such a low limit (limit=' + str(limit) + ')?:'
-        if not is_buy_order and limit <= 0 and not yn_dialog(question):
-            print('Order was not placed.')
-            return
-
-    if expiry is None:
-        expiry = input('Time until order expires (minutes, default ' + str(DEFAULT_ORDER_EXPIRY) + '):')
-        if expiry == '':
-            expiry = DEFAULT_ORDER_EXPIRY
-    try:
-        expiry = float(expiry)
-    except ValueError:
-        print('Invalid expiration time.')
-        return
-
-    fake_loading_bar('Sending Data', duration=1.3)
-    response = client_request('order', {"buy": is_buy_order,
-                                        "session_id": connection.session_id,
-                                        "amount": amount,
-                                        "ownable": obj_name,
-                                        "limit": limit,
-                                        "stop_loss": stop_loss,
-                                        "time_until_expiration": expiry})
-    if 'error_message' in response:
-        print('Order placement failed with message:', response['error_message'])
-    else:
-        print('You might want to use the `trades` or `depot` commands',
-              'to see if the order has been executed already.')
-
-
-def sell(obj_name=None, amount=None, limit=None, stop_loss=None, expiry=None):
-    _order(is_buy_order=False,
-           obj_name=obj_name,
-           amount=amount,
-           limit=limit,
-           stop_loss=stop_loss,
-           expiry=expiry)
-
-
-def buy(obj_name=None, amount=None, limit=None, stop_loss=None, expiry=None):
-    _order(is_buy_order=True,
-           obj_name=obj_name,
-           amount=amount,
-           limit=limit,
-           stop_loss=stop_loss,
-           expiry=expiry)
-
-
-def orders():
-    fake_loading_bar('Loading Data', duration=0.9)
-    response = client_request('orders', {"session_id": connection.session_id})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['Buy?', 'Name', 'Size', 'Limit', 'stop-loss', 'Expires', 'No.'],
-                          tablefmt="pipe"))
-    else:
-        if 'error_message' in response:
-            print('Order access failed with message:', response['error_message'])
-        else:
-            print('Order access failed.')
-
-
-def orders_on(obj_name=None):
-    if obj_name is None:  # TODO list some available objects
-        obj_name = input('Name of object to check: ')
-    fake_loading_bar('Loading Data', duration=2.3)
-    response = client_request('orders_on', {"session_id": connection.session_id, "ownable": obj_name})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['My', 'Buy?', 'Name', 'Size', 'Limit', 'Expires', 'No.'],
-                          tablefmt="pipe"))
-    else:
-        if 'error_message' in response:
-            print('Order access failed with message:', response['error_message'])
-        else:
-            print('Order access failed.')
-
-
-def gift(username=None, obj_name=None, amount=None):
-    if username is None:
-        username = input('Username of recipient: ')
-    if obj_name is None:
-        obj_name = input('Name of object to give: ')
-    if amount is None:
-        amount = input('How many?: ')
-    fake_loading_bar('Sending Gift', duration=4.2)
-    response = client_request('gift',
-                              {"session_id": connection.session_id,
-                               "username": username,
-                               "object_name": obj_name,
-                               "amount": amount})
-    if 'error_message' in response:
-        print('Order access failed with message:', response['error_message'])
-    elif 'message' in response:
-        print(response['message'])
-
-
-def news():
-    fake_loading_bar('Loading Data', duration=0.76)
-    response = client_request('news', {})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['Date', 'Title'],
-                          tablefmt="pipe"))
-    else:
-        if 'error_message' in response:
-            print('News access failed with message:', response['error_message'])
-        else:
-            print('News access failed.')
-
-
-def tradables():
-    fake_loading_bar('Loading Data', duration=12.4)
-    response = client_request('tradables', {})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['Name', 'Course', 'Market Cap.'],
-                          tablefmt="pipe"))
-        world_wealth = 0
-        for row in response['data']:
-            if row[2] is not None:
-                world_wealth += row[2]
-        print('Estimated worldwide wealth:', world_wealth)
-
-    else:
-        if 'error_message' in response:
-            print('Data access failed with message:', response['error_message'])
-        else:
-            print('Data access failed.')
-
-
-def trades_on(obj_name=None, limit=5):
-    limit = float(limit)
-    if obj_name is None:  # TODO list some available objects
-        obj_name = input('Name of object to check: ')
-    fake_loading_bar('Loading Data', duration=0.34*limit)
-    response = client_request('trades_on', {"session_id": connection.session_id,
-                                            "ownable": obj_name,
-                                            "limit": limit})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['Time', 'Volume', 'Price'],
-                          tablefmt="pipe"))
-    else:
-        if 'error_message' in response:
-            print('Trades access failed with message:', response['error_message'])
-        else:
-            print('Trades access failed.')
-
-
-def trades(limit=10):
-    limit = float(limit)
-    fake_loading_bar('Loading Data', duration=limit * 0.21)
-    response = client_request('trades', {"session_id": connection.session_id,
-                                         "limit": limit})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['Buy?', 'Name', 'Volume', 'Price', 'Time'],
-                          tablefmt="pipe"))
-    else:
-        if 'error_message' in response:
-            print('Trades access failed with message:', response['error_message'])
-        else:
-            print('Trades access failed.')
-
-
-def old_orders(include_canceled=None, include_executed=None, limit=10):
-    limit = float(limit)
-    if include_canceled is None:
-        include_canceled = yn_dialog('Include canceled/expired orders in list?')
-    if include_executed is None:
-        include_executed = yn_dialog('Include fully executed orders in list?')
-    fake_loading_bar('Loading Data', duration=limit * 0.27)
-    response = client_request('old_orders', {"session_id": connection.session_id,
-                                             "include_canceled": include_canceled,
-                                             "include_executed": include_executed,
-                                             "limit": limit})
-    success = 'data' in response
-    if success:
-        print(my_tabulate(response['data'],
-                          headers=['Buy?', 'Name', 'Size', 'Limit', 'Expiry', 'No.', 'Status'],
-                          tablefmt="pipe"))
-    else:
-        if 'error_message' in response:
-            print('Order access failed with message:', response['error_message'])
-        else:
-            print('Order access failed.')
-
-
-# noinspection PyShadowingBuiltins
-def exit():
-    global exiting
-    exiting = True
+import sys
+from getpass import getpass
+from inspect import signature
+
+import connection
+from connection import client_request
+from debug import debug
+from game import DEFAULT_ORDER_EXPIRY
+from run_client import allowed_commands, fake_loading_bar
+from util import my_tabulate, yn_dialog
+
+exiting = False
+
+
+def login(username=None, password=None):
+    if connection.session_id is not None:
+        fake_loading_bar('Signing out', duration=0.7)
+        connection.session_id = None
+
+    if username is None:
+        username = input('Username: ')
+
+    if password is None:
+        if sys.stdin.isatty():
+            password = getpass('Password: ')
+        else:
+            password = input('Password: ')
+
+    fake_loading_bar('Signing in', duration=2.3)
+    response = client_request('login', {"username": username, "password": password})
+    success = 'session_id' in response
+    if success:
+        connection.session_id = response['session_id']
+        print('Login successful.')
+    else:
+        if 'error_message' in response:
+            print('Login failed with message:', response['error_message'])
+        else:
+            print('Login failed.')
+
+
+def register(username=None, game_key='', password=None, retype_pw=None):
+    if connection.session_id is not None:
+        connection.session_id = None
+        fake_loading_bar('Signing out', duration=0.7)
+
+    if username is None:
+        username = input('Username: ')
+
+    if password is None:
+        if sys.stdin.isatty():
+            password = getpass('New password: ')
+            retype_pw = getpass('Retype password: ')
+        else:
+            password = input('New password: ')
+            retype_pw = input('Retype password: ')
+        if password != retype_pw:
+            print('Passwords do not match.')
+            return
+    elif retype_pw is None:
+        if sys.stdin.isatty():
+            retype_pw = getpass('Retype password: ')
+        else:
+            retype_pw = input('Retype password: ')
+        if password != retype_pw:
+            print('Passwords do not match.')
+            return
+
+    if not debug:
+        if game_key == '':
+            print('Entering a game key will provide you with some starting money and other useful stuff.')
+            game_key = input('Game key (leave empty if you don\'t have one): ')
+
+    fake_loading_bar('Validating Registration', duration=5.2)
+    if game_key != '':
+        fake_loading_bar('Validating Game Key', duration=0.4)
+    response = client_request('register', {"username": username, "password": password, "game_key": game_key})
+
+    if 'error_message' in response:
+        print('Registration failed with message:', response['error_message'])
+
+
+def cancel_order(order_no=None):
+    if order_no is None:
+        order_no = input('Order No.: ')
+
+    fake_loading_bar('Validating Request', duration=0.6)
+    response = client_request('cancel_order', {"session_id": connection.session_id, "order_id": order_no})
+
+    if 'error_message' in response:
+        print('Order cancelling failed with message:', response['error_message'])
+
+
+def change_pw(password=None, retype_pw=None):
+    if password != retype_pw:
+        password = None
+    if password is None:
+        if sys.stdin.isatty():
+            password = getpass('New password: ')
+            retype_pw = getpass('Retype password: ')
+        else:
+            password = input('New password: ')
+            retype_pw = input('Retype password: ')
+        if password != retype_pw:
+            print('Passwords do not match.')
+            return
+    elif retype_pw is None:
+        if sys.stdin.isatty():
+            retype_pw = getpass('Retype password: ')
+        else:
+            retype_pw = input('Retype password: ')
+        if password != retype_pw:
+            print('Passwords do not match.')
+            return
+
+    fake_loading_bar('Validating password', duration=1.2)
+    fake_loading_bar('Changing password', duration=5.2)
+    response = client_request('change_password', {"session_id": connection.session_id, "password": password})
+
+    if 'error_message' in response:
+        print('Changing password failed with message:', response['error_message'])
+
+    fake_loading_bar('Signing out', duration=0.7)
+    connection.session_id = None
+
+
+# noinspection PyShadowingBuiltins
+def help():
+    print('Allowed commands:')
+    command_table = []
+    for cmd in allowed_commands:
+        this_module = sys.modules[__name__]
+        method = getattr(this_module, cmd)
+        params = signature(method).parameters
+        command_table.append([cmd] + [p for p in params])
+
+    print(my_tabulate(command_table, tablefmt='pipe', headers=['command',
+                                                               'param 1',
+                                                               'param 2',
+                                                               'param 3',
+                                                               'param 4',
+                                                               'param 5',
+                                                               ]))
+    print('NOTE:')
+    print('  All parameters for all commands are optional!')
+    print('  Commands can be combined in one line with ; between them.')
+    print('  Parameters are separated with whitespace.')
+    print('  Use . as a decimal separator.')
+    print('  Use 0/1 for boolean parameters (other strings might work).')
+
+
+def depot():
+    fake_loading_bar('Loading data', duration=1.3)
+    response = client_request('depot', {"session_id": connection.session_id})
+    success = 'data' in response and 'own_wealth' in response
+    if success:
+        data = response['data']
+        for row in data:
+            row.append(row[1] * row[2])
+        print(my_tabulate(data,
+                          headers=['Object', 'Amount', 'Course', 'Bid', 'Ask', 'Est. Value'],
+                          tablefmt="pipe"))
+        print('This corresponds to a wealth of roughly', response['own_wealth'])
+    else:
+        if 'error_message' in response:
+            print('Depot access failed with message:', response['error_message'])
+        else:
+            print('Depot access failed.')
+
+
+def leaderboard():
+    fake_loading_bar('Loading data', duration=1.3)
+    response = client_request('leaderboard', {})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'], headers=['Trader', 'Wealth'], tablefmt="pipe"))
+        # print('Remember that the goal is to be as rich as possible, not to be richer than other traders!')
+    else:
+        if 'error_message' in response:
+            print('Leaderboard access failed with message:', response['error_message'])
+        else:
+            print('Leaderboard access failed.')
+
+
+def activate_key(key=''):
+    if key == '':
+        print('Entering a game key may get you some money or other useful stuff.')
+        key = input('Key: ')
+
+    if key == '':
+        print('Invalid key.')
+
+    fake_loading_bar('Validating Key', duration=0.4)
+    response = client_request('activate_key', {"session_id": connection.session_id, 'key': key})
+    if 'error_message' in response:
+        print('Key activation failed with message:', response['error_message'])
+
+
+def _order(is_buy_order, obj_name, amount, limit, stop_loss, expiry):
+    if stop_loss not in [None, '1', '0']:
+        print('Invalid value for flag stop loss (only 0 or 1 allowed).')
+        return
+    if obj_name is None:  # TODO list some available objects
+        obj_name = input('Name of object to sell: ')
+    if amount is None:
+        amount = input('Amount: ')
+
+    if limit is None:
+        set_limit = yn_dialog('Do you want to place a limit?')
+        if set_limit:
+            limit = input('Limit: ')
+            stop_loss = yn_dialog('Is this a stop-loss limit?')
+
+    if limit is not None:
+        try:
+            limit = float(limit)
+        except ValueError:
+            print('Invalid limit.')
+            return
+
+        if stop_loss is None:
+            stop_loss = yn_dialog('Is this a stop-loss limit?')
+
+        question = 'Are you sure you want to use such a low limit (limit=' + str(limit) + ')?:'
+        if not is_buy_order and limit <= 0 and not yn_dialog(question):
+            print('Order was not placed.')
+            return
+
+    if expiry is None:
+        expiry = input('Time until order expires (minutes, default ' + str(DEFAULT_ORDER_EXPIRY) + '):')
+        if expiry == '':
+            expiry = DEFAULT_ORDER_EXPIRY
+    try:
+        expiry = float(expiry)
+    except ValueError:
+        print('Invalid expiration time.')
+        return
+
+    fake_loading_bar('Sending Data', duration=1.3)
+    response = client_request('order', {
+        "buy": is_buy_order,
+        "session_id": connection.session_id,
+        "amount": amount,
+        "ownable": obj_name,
+        "limit": limit,
+        "stop_loss": stop_loss,
+        "time_until_expiration": expiry
+    })
+    if 'error_message' in response:
+        print('Order placement failed with message:', response['error_message'])
+    else:
+        print('You might want to use the `trades` or `depot` commands',
+              'to see if the order has been executed already.')
+
+
+def sell(obj_name=None, amount=None, limit=None, stop_loss=None, expiry=None):
+    _order(is_buy_order=False,
+           obj_name=obj_name,
+           amount=amount,
+           limit=limit,
+           stop_loss=stop_loss,
+           expiry=expiry)
+
+
+def buy(obj_name=None, amount=None, limit=None, stop_loss=None, expiry=None):
+    _order(is_buy_order=True,
+           obj_name=obj_name,
+           amount=amount,
+           limit=limit,
+           stop_loss=stop_loss,
+           expiry=expiry)
+
+
+def orders():
+    fake_loading_bar('Loading Data', duration=0.9)
+    response = client_request('orders', {"session_id": connection.session_id})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['Buy?', 'Name', 'Size', 'Limit', 'stop-loss', 'Expires', 'No.'],
+                          tablefmt="pipe"))
+    else:
+        if 'error_message' in response:
+            print('Order access failed with message:', response['error_message'])
+        else:
+            print('Order access failed.')
+
+
+def orders_on(obj_name=None):
+    if obj_name is None:  # TODO list some available objects
+        obj_name = input('Name of object to check: ')
+    fake_loading_bar('Loading Data', duration=2.3)
+    response = client_request('orders_on', {"session_id": connection.session_id, "ownable": obj_name})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['My', 'Buy?', 'Name', 'Size', 'Limit', 'Expires', 'No.'],
+                          tablefmt="pipe"))
+    else:
+        if 'error_message' in response:
+            print('Order access failed with message:', response['error_message'])
+        else:
+            print('Order access failed.')
+
+
+def gift(username=None, obj_name=None, amount=None):
+    if username is None:
+        username = input('Username of recipient: ')
+    if obj_name is None:
+        obj_name = input('Name of object to give: ')
+    if amount is None:
+        amount = input('How many?: ')
+    fake_loading_bar('Sending Gift', duration=4.2)
+    response = client_request('gift',
+                              {"session_id": connection.session_id,
+                               "username": username,
+                               "object_name": obj_name,
+                               "amount": amount})
+    if 'error_message' in response:
+        print('Order access failed with message:', response['error_message'])
+    elif 'message' in response:
+        print(response['message'])
+
+
+def news():
+    fake_loading_bar('Loading Data', duration=0.76)
+    response = client_request('news', {})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['Date', 'Title'],
+                          tablefmt="pipe"))
+    else:
+        if 'error_message' in response:
+            print('News access failed with message:', response['error_message'])
+        else:
+            print('News access failed.')
+
+
+def tradables():
+    fake_loading_bar('Loading Data', duration=12.4)
+    response = client_request('tradables', {})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['Name', 'Course', 'Market Cap.'],
+                          tablefmt="pipe"))
+        world_wealth = 0
+        for row in response['data']:
+            if row[2] is not None:
+                world_wealth += row[2]
+        print('Estimated worldwide wealth:', world_wealth)
+
+    else:
+        if 'error_message' in response:
+            print('Data access failed with message:', response['error_message'])
+        else:
+            print('Data access failed.')
+
+
+def trades_on(obj_name=None, limit=5):
+    limit = float(limit)
+    if obj_name is None:  # TODO list some available objects
+        obj_name = input('Name of object to check: ')
+    fake_loading_bar('Loading Data', duration=0.34 * limit)
+    response = client_request('trades_on', {"session_id": connection.session_id,
+                                            "ownable": obj_name,
+                                            "limit": limit})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['Time', 'Volume', 'Price'],
+                          tablefmt="pipe"))
+    else:
+        if 'error_message' in response:
+            print('Trades access failed with message:', response['error_message'])
+        else:
+            print('Trades access failed.')
+
+
+def trades(limit=10):
+    limit = float(limit)
+    fake_loading_bar('Loading Data', duration=limit * 0.21)
+    response = client_request('trades', {"session_id": connection.session_id,
+                                         "limit": limit})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['Buy?', 'Name', 'Volume', 'Price', 'Time'],
+                          tablefmt="pipe"))
+    else:
+        if 'error_message' in response:
+            print('Trades access failed with message:', response['error_message'])
+        else:
+            print('Trades access failed.')
+
+
+def old_orders(include_canceled=None, include_executed=None, limit=10):
+    limit = float(limit)
+    if include_canceled is None:
+        include_canceled = yn_dialog('Include canceled/expired orders in list?')
+    if include_executed is None:
+        include_executed = yn_dialog('Include fully executed orders in list?')
+    fake_loading_bar('Loading Data', duration=limit * 0.27)
+    response = client_request('old_orders', {"session_id": connection.session_id,
+                                             "include_canceled": include_canceled,
+                                             "include_executed": include_executed,
+                                             "limit": limit})
+    success = 'data' in response
+    if success:
+        print(my_tabulate(response['data'],
+                          headers=['Buy?', 'Name', 'Size', 'Limit', 'Expiry', 'No.', 'Status'],
+                          tablefmt="pipe"))
+    else:
+        if 'error_message' in response:
+            print('Order access failed with message:', response['error_message'])
+        else:
+            print('Order access failed.')
+
+
+# noinspection PyShadowingBuiltins
+def exit():
+    global exiting
+    exiting = True

connection.py

@@ -1,25 +1,182 @@
-import json
-
-import requests
-
-from debug import debug
-
-port = 8451
-if debug:
-    host = 'http://localhost:' + str(port)
-else:
-    host = 'http://koljastrohm-games.com:' + str(port)
-json_headers = {'Content-type': 'application/json'}
-
-
-def client_request(route, data):
-    if debug:
-        print('Sending to Server: ' + str(json.dumps(data)))
-    r = requests.post(host + '/' + route, data=json.dumps(data),
-                      headers=json_headers)
-    if debug:
-        print('Request returned: ' + str(r.content))
-    return r.json()
-
-
-session_id = None
+import datetime
+import json
+from typing import Dict, List, Any
+
+import requests
+from geventwebsocket import WebSocketError
+from geventwebsocket.websocket import WebSocket
+
+import model
+from debug import debug
+from my_types import MessageType, Message, UserIdentification, UserId, MessageQueue
+from routes import push_message_types
+
+PORT = 58317
+if debug:
+    host = 'http://localhost:' + str(PORT)
+else:
+    host = 'http://koljastrohm-games.com:' + str(PORT)
+
+websockets_for_user: Dict[UserIdentification, List[WebSocket]] = {}
+users_for_websocket: Dict[WebSocket, List[UserIdentification]] = {}
+push_message_queue: MessageQueue = []
+session_id = None
+
+
+class HttpError(Exception):
+    def __init__(self, code: int, message: Any = None, prepend_code=True):
+        Exception.__init__(self)
+        self.body = dict()
+        self.code = code
+        if message is None:
+            message = str(code)
+
+        if prepend_code and isinstance(message, str) and not message.startswith(str(code)):
+            message = str(code) + ': ' + message
+            if code // 100 == 2:
+                self.body['message'] = message
+            else:
+                self.body['error'] = message
+        elif isinstance(message, str):
+            if code // 100 == 2:
+                self.body['message'] = message
+            else:
+                self.body['error'] = message
+        elif isinstance(message, dict):
+            self.body = message.copy()
+        else:  # for example a list or number
+            self.body['data'] = message
+
+
+class NotFound(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 404, message, prepend_code)
+
+
+class Success(HttpError):
+    def __init__(self, message: Any = None, prepend_code=False):
+        HttpError.__init__(self, 200, message, prepend_code)
+
+
+class UnavailableForLegalReasons(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 451, message, prepend_code)
+
+
+class Forbidden(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 403, message, prepend_code)
+
+
+class Unauthorized(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 401, message, prepend_code)
+
+
+class BadRequest(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 400, message, prepend_code)
+
+
+class InternalServerError(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 500, message, prepend_code)
+
+
+class Locked(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 423, message, prepend_code)
+
+
+class PreconditionFailed(HttpError):
+    def __init__(self, message: Any = None, prepend_code=True):
+        HttpError.__init__(self, 412, message, prepend_code)
+
+
+def check_missing_attributes(request_json: Dict, attributes: List[str]):
+    for attr in attributes:
+        if attr not in request_json:
+            if str(attr) == 'session_id':
+                raise Unauthorized('You are not signed in.')
+            raise BadRequest('Missing value for attribute ' + str(attr))
+        if str(attr) == 'session_id':
+            if not model.valid_session_id(request_json['session_id']):
+                raise Unauthorized('Invalid value for attribute ' + str(attr))
+
+
+def client_request(route, data):
+    return json_request(host + '/json/' + route, data)
+
+
+def json_request(url, data):
+    if debug:
+        print('Sending to ' + url + ': ' + str(json.dumps(data)))
+    r = requests.post(url,
+                      data=json.dumps(data),
+                      headers={'Content-type': 'application/json; charset=latin-1'})
+    if debug:
+        print('Request returned: ' + str(r.content))
+    return r.json()
+
+
+def push_message(recipient_ids: List[UserIdentification], contents: Message, message_type: MessageType):
+    if message_type not in push_message_types:
+        raise AssertionError('Invalid message type.')
+    sockets = {socket for user_id in recipient_ids for socket in websockets_for_user.get(user_id, [])}
+    if len(sockets) > 0:
+        message = {'message_type': message_type, 'contents': contents}
+        for ws in sockets:
+            if ws.closed:
+                ws_cleanup(ws)
+                continue
+            message = json.dumps({'message_type': message_type, 'contents': contents})
+            ws.send(message)
+        print(message_type,
+              'to',
+              len(sockets),
+              'sockets',
+              datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+              len(message))
+
+
+def enqueue_push_message(recipient_ids: List[UserId], contents: Dict, message_type: str):
+    assert message_type in push_message_types
+    recipient_ids = [user_id for user_id in recipient_ids]
+    if len(recipient_ids) == 0:
+        return
+    recipient_ids = [(model.current_db_name, user_id) for user_id in recipient_ids]
+    push_message_queue.append((recipient_ids, contents, message_type))
+
+
+def ws_cleanup(ws):
+    if ws in users_for_websocket:
+        users_affected = users_for_websocket[ws]
+        for user_id in users_affected:
+            # remove the websocket from all lists
+            websockets_for_user[user_id][:] = filter(lambda s: s != ws,
+                                                     websockets_for_user[user_id])
+        del users_for_websocket[ws]
+        if not ws.closed:
+            ws.close()
+    print('websocket connection ended',
+          *ws.handler.client_address,
+          datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), )
+
+
+def preprocess_push_message_queue(queue: MessageQueue) -> MessageQueue:
+    return queue
+
+
+def push_messages_in_queue():
+    global push_message_queue
+
+    push_message_queue = preprocess_push_message_queue(push_message_queue)
+
+    for message in push_message_queue:
+        try:
+            push_message(*message)
+        except WebSocketError:
+            continue
+        except ConnectionResetError:
+            continue
+    del push_message_queue[:]

db_setup.py

@@ -1,498 +0,0 @@
-from sqlite3 import OperationalError
-
-from game import CURRENCY_NAME, MINIMUM_ORDER_AMOUNT
-
-
-def setup(cursor):
-    print('Database setup...')
-
-    drop_triggers(cursor)
-
-    tables(cursor)
-
-    create_triggers(cursor)
-
-    create_indices(cursor)
-
-    seed(cursor)
-
-
-def drop_triggers(cursor):
-    print(' - Dropping all triggers...')
-    cursor.execute("DROP TRIGGER IF EXISTS owned_amount_not_negative_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS owned_amount_not_negative_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS amount_positive_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS amount_positive_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS order_limit_not_negative_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS order_limit_not_negative_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS order_amount_positive_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS order_amount_positive_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS not_more_executed_than_ordered_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS not_more_executed_than_ordered_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS not_more_ordered_than_available_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS not_more_ordered_than_available_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS expiry_dt_in_future_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS expiry_dt_in_future_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS stop_loss_requires_limit_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS stop_loss_requires_limit_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS limit_requires_stop_loss_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS limit_requires_stop_loss_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS minimum_order_amount_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS integer_amount_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS dt_monotonic_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS dt_monotonic_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS orders_rowid_sorted_by_creation_time_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS news_dt_monotonic_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS not_nullify_buyer_id_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS buyer_id_not_null_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS not_nullify_seller_id_after_update")
-    cursor.execute("DROP TRIGGER IF EXISTS seller_id_not_null_after_insert")
-    cursor.execute("DROP TRIGGER IF EXISTS order_history_no_update")
-
-
-def seed(cursor):
-    print(' - Seeding initial data...')
-    # ₭ollar
-    cursor.execute('''
-                    INSERT OR IGNORE INTO ownables
-                    (name)
-                    VALUES (?)
-                    ''', (CURRENCY_NAME,))
-    # The bank/external investors
-    cursor.execute('''
-                    INSERT OR IGNORE INTO users
-                    (username,password)
-                    VALUES ('bank','')
-                    ''')
-
-    # bank owns all the money that is not owned by players, 1000 * num_used_key - player_money
-    cursor.execute('''
-                    INSERT OR IGNORE INTO ownership
-                    (user_id, ownable_id, amount)
-                    VALUES ((SELECT rowid FROM users WHERE username = 'bank'), 
-                            (SELECT rowid FROM ownables WHERE name = ?),
-                            1000 * (SELECT COUNT(used_by_user_id) FROM keys) - (SELECT SUM(amount) 
-                             FROM ownership 
-                             WHERE ownable_id = (SELECT rowid FROM ownables WHERE name = ?)))
-                    ''', (CURRENCY_NAME, CURRENCY_NAME,))
-
-    # bank owns some stuff (₭ollar is be dealt with separately)
-    cursor.execute('''
-    INSERT OR IGNORE INTO ownership
-    (user_id, ownable_id, amount)
-    SELECT (SELECT rowid FROM users WHERE username = 'bank'), 
-            ownables.rowid, 
-            (SELECT COALESCE(SUM(amount),0) FROM ownership WHERE ownable_id = ownables.rowid)
-    FROM ownables WHERE
-    name <> ?
-    ''', (CURRENCY_NAME,))
-    cursor.executemany('''
-    INSERT INTO global_control_values (value_name, value)
-    WITH new_value AS (SELECT ? AS name, ? AS value)
-    SELECT new_value.name, new_value.value
-    FROM new_value
-    WHERE NOT EXISTS(SELECT * -- tODO test if this works
-                     FROM global_control_values v2
-                     WHERE v2.value_name = new_value.name
-                       AND v2.value = new_value.value
-                       AND v2.dt = (SELECT MAX(v3.dt)
-                                    FROM global_control_values v3
-                                    WHERE v3.value_name = new_value.name
-                                      AND v3.value = new_value.value))
-    ''', [('banking_license_price', 5e6),
-          ('personal_loan_interest_rate', 0.1),  # may seem a lot but actually this is a credit that you get without any assessment involved
-          ('deposit_facility', -0.05),  # ECB 2020
-          ('marginal_lending_facility', 0.25),  # ECB 2020
-          ('cash_reserve_ratio', 0.01),  # Eurozone 2020
-          ('cash_reserve_free_amount', 1e5),  # Eurozone 2020
-          ])
-
-
-def create_triggers(cursor):
-    print(' - Creating triggers...')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS owned_amount_not_negative_after_insert
-                AFTER INSERT ON ownership
-                WHEN NEW.amount < 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not own an amount less than 0.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS owned_amount_not_negative_after_update
-                AFTER UPDATE ON ownership
-                WHEN NEW.amount < 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not own an amount less than 0.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS amount_positive_after_insert
-                AFTER INSERT ON transactions
-                WHEN NEW.amount <= 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not perform empty transactions.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS amount_positive_after_update
-                AFTER UPDATE ON transactions
-                WHEN NEW.amount <= 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not perform empty transactions.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS order_limit_not_negative_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW."limit" IS NOT NULL AND NEW."limit" < 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not set a limit less than 0.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS order_limit_not_negative_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW."limit" IS NOT NULL AND NEW."limit" < 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not set a limit less than 0.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS order_amount_positive_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW.ordered_amount <= 0 OR NEW.executed_amount < 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not order 0 or less.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS order_amount_positive_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW.ordered_amount <= 0 OR NEW.executed_amount < 0
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not order 0 or less.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS not_more_executed_than_ordered_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW.ordered_amount < NEW.executed_amount
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not execute more than ordered.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS not_more_executed_than_ordered_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW.ordered_amount < NEW.executed_amount
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not execute more than ordered.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS not_more_ordered_than_available_after_insert
-                AFTER INSERT ON orders
-                WHEN NOT NEW.buy AND 0 >
-                    -- owned_amount
-                    COALESCE (
-                        (SELECT amount
-                         FROM ownership
-                         WHERE ownership.rowid = NEW.ownership_id), 0)
-                    - -- sell_ordered_amount
-                    (SELECT COALESCE(SUM(orders.ordered_amount - orders.executed_amount),0)
-                     FROM orders, ownership
-                     WHERE ownership.rowid = orders.ownership_id
-                     AND ownership.rowid = NEW.ownership_id
-                     AND NOT orders.buy) 
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not order more than you own.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS not_more_ordered_than_available_after_update
-                AFTER UPDATE ON orders
-                WHEN NOT NEW.buy AND 0 >
-                    -- owned_amount
-                    COALESCE (
-                        (SELECT amount
-                         FROM ownership
-                         WHERE ownership.rowid = NEW.ownership_id), 0)
-                    - -- sell_ordered_amount
-                    (SELECT COALESCE(SUM(orders.ordered_amount - orders.executed_amount),0)
-                     FROM orders, ownership
-                     WHERE ownership.rowid = orders.ownership_id
-                     AND ownership.rowid = NEW.ownership_id
-                     AND NOT orders.buy) 
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not order more than you own.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS expiry_dt_in_future_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW.expiry_dt <= datetime('now')
-                BEGIN SELECT RAISE(ROLLBACK, 'Order is already expired.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS expiry_dt_in_future_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW.expiry_dt <= datetime('now')
-                BEGIN SELECT RAISE(ROLLBACK, 'Order is already expired.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS stop_loss_requires_limit_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW."limit" IS NULL AND NEW.stop_loss IS NOT NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'Can only set `stop_loss` `for limit orders.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS stop_loss_requires_limit_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW."limit" IS NULL AND NEW.stop_loss IS NOT NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'Can only set `stop_loss` `for limit orders.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS limit_requires_stop_loss_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW."limit" IS NOT NULL AND NEW.stop_loss IS NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'Need to set stop_loss to either True or False for limit orders.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS limit_requires_stop_loss_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW."limit" IS NOT NULL AND NEW.stop_loss IS NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'Need to set stop_loss to either True or False for limit orders.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS minimum_order_amount_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW.ordered_amount < ?
-                BEGIN SELECT RAISE(ROLLBACK, 'There is a minimum amount for new orders.'); END
-                '''.replace('?', str(MINIMUM_ORDER_AMOUNT)))
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS integer_amount_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW.ordered_amount <> ROUND(NEW.ordered_amount)
-                BEGIN SELECT RAISE(ROLLBACK, 'Can only set integer amounts for new orders.'); END
-                '''.replace('?', str(MINIMUM_ORDER_AMOUNT)))
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS dt_monotonic_after_insert
-                AFTER INSERT ON transactions
-                WHEN NEW.dt < (SELECT MAX(dt) FROM transactions t2 WHERE t2.rowid < rowid)
-                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS dt_monotonic_after_update
-                AFTER INSERT ON transactions
-                WHEN NEW.dt < (SELECT MAX(dt) FROM transactions t2 WHERE t2.rowid < rowid)
-                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS news_dt_monotonic_after_insert
-                AFTER INSERT ON news
-                WHEN NEW.dt < (SELECT MAX(dt) FROM news t2 WHERE t2.rowid < rowid)
-                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS news_dt_monotonic_after_insert
-                AFTER INSERT ON news
-                WHEN NEW.dt < (SELECT MAX(dt) FROM news t2 WHERE t2.rowid < rowid)
-                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS orders_rowid_sorted_by_creation_time_after_insert
-                AFTER INSERT ON orders
-                WHEN NEW.rowid < (SELECT MAX(rowid) FROM orders o2)
-                BEGIN SELECT RAISE(ROLLBACK, 'Order-rowid programming bug (insert), not your fault.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS news_dt_monotonic_after_update
-                AFTER UPDATE ON orders
-                WHEN NEW.rowid <> OLD.rowid
-                BEGIN SELECT RAISE(ROLLBACK, 'Cannot change number of existing order.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS not_nullify_buyer_id_after_update
-                AFTER UPDATE ON transactions
-                WHEN NEW.buyer_id IS NULL AND OLD.buyer_id IS NOT NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'Cannot nullify buyer_id.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS buyer_id_not_null_after_insert
-                AFTER INSERT ON transactions
-                WHEN NEW.buyer_id IS NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'buyer_id must not be null for new transactions.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS not_nullify_seller_id_after_update
-                AFTER UPDATE ON transactions
-                WHEN NEW.seller_id IS NULL AND OLD.seller_id IS NOT NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'Cannot nullify seller_id.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS seller_id_not_null_after_insert
-                AFTER INSERT ON transactions
-                WHEN NEW.seller_id IS NULL
-                BEGIN SELECT RAISE(ROLLBACK, 'seller_id must not be null for new transactions.'); END
-                ''')
-    cursor.execute('''
-                CREATE TRIGGER IF NOT EXISTS order_history_no_update
-                BEFORE UPDATE ON order_history
-                BEGIN SELECT RAISE(ROLLBACK, 'Can not change order history.'); END
-                ''')
-
-
-def create_indices(cursor):
-    print(' - Creating indices...')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS ownership_ownable
-                ON ownership (ownable_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS transactions_ownable
-                ON transactions (ownable_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS orders_expiry
-                ON orders (expiry_dt)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS orders_ownership
-                ON orders (ownership_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS orders_limit
-                ON orders ("limit")
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS transactions_dt
-                ON transactions (dt)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS news_dt
-                ON news (dt)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS ownables_name
-                ON ownables (name)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS users_name
-                ON users (username)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS sessions_id
-                ON sessions (session_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS sessions_user
-                ON sessions (user_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS transactions_seller
-                ON transactions (seller_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS transactions_buyer
-                ON transactions (buyer_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS order_history_id
-                ON order_history (order_id)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS order_canceled
-                ON order_history (archived_dt)
-                ''')
-    cursor.execute('''
-                CREATE INDEX IF NOT EXISTS order_history_ownership
-                ON order_history (ownership_id)
-                ''')
-
-
-def tables(cursor):
-    print(' - Creating tables...')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS users(
-                    username VARCHAR(10) UNIQUE NOT NULL, 
-                    password VARCHAR(200) NOT NULL)
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS ownables(
-                    name VARCHAR(10) UNIQUE NOT NULL)
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS ownership(
-                    user_id INTEGER NOT NULL,
-                    ownable_id INTEGER NOT NULL,
-                    amount CURRENCY NOT NULL DEFAULT 0,
-                    FOREIGN KEY (user_id) REFERENCES users(rowid),
-                    FOREIGN KEY (ownable_id) REFERENCES ownables(rowid),
-                    UNIQUE (user_id, ownable_id)
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS sessions(
-                    user_id INTEGER NOT NULL,
-                    session_id STRING NOT NULL,
-                    FOREIGN KEY (user_id) REFERENCES users(rowid)
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS orders(
-                    ownership_id INTEGER NOT NULL,
-                    buy BOOLEAN NOT NULL,
-                    "limit" CURRENCY,
-                    stop_loss BOOLEAN,
-                    ordered_amount CURRENCY NOT NULL,
-                    executed_amount CURRENCY DEFAULT 0 NOT NULL,
-                    expiry_dt DATETIME NOT NULL,
-                    FOREIGN KEY (ownership_id) REFERENCES ownership(rowid)
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS order_history(
-                    ownership_id INTEGER NOT NULL,
-                    buy BOOLEAN NOT NULL,
-                    "limit" CURRENCY,
-                    ordered_amount CURRENCY NOT NULL,
-                    executed_amount CURRENCY NOT NULL,
-                    expiry_dt DATETIME NOT NULL,
-                    status VARCHAR(20) NOT NULL,
-                    order_id INTEGER NOT NULL, -- order_id is not a FOREIGN KEY since orders are deleted from order table afterwards
-                    archived_dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                    FOREIGN KEY (ownership_id) REFERENCES ownership(rowid)
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS transactions(
-                    dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                    price CURRENCY NOT NULL,
-                    ownable_id INTEGER NOT NULL,
-                    amount CURRENCY NOT NULL,
-                    FOREIGN KEY (ownable_id) REFERENCES ownables(rowid)
-                )
-                ''')
-    _add_column_if_not_exists(cursor, '''
-                -- there is a not null constraint for new values that is watched by triggers
-                ALTER TABLE transactions ADD COLUMN buyer_id INTEGER REFERENCES users(rowid)
-                ''')
-    _add_column_if_not_exists(cursor, '''
-                -- there is a not null constraint for new values that is watched by triggers
-                ALTER TABLE transactions ADD COLUMN seller_id INTEGER REFERENCES users(rowid)
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS keys(
-                    key STRING UNIQUE NOT NULL,
-                    used_by_user_id INTEGER,
-                    FOREIGN KEY (used_by_user_id) REFERENCES users(rowid)
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS news(
-                    dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                    title VARCHAR(50) NOT NULL
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS banks(
-                    user_id NOT NULL REFERENCES users(rowid)
-                )
-                ''')
-    cursor.execute('''
-                CREATE TABLE IF NOT EXISTS global_control_values(
-                    dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                    value_name VARCHAR NOT NULL,
-                    value FLOAT NOT NULL,
-                    UNIQUE (value_name, dt)
-                )
-                ''')
-
-
-def _add_column_if_not_exists(cursor, query):
-    if 'ALTER TABLE' not in query.upper():
-        raise ValueError('Only alter table queries allowed.')
-    if 'ADD COLUMN' not in query.upper():
-        raise ValueError('Only add column queries allowed.')
-    try:
-        cursor.execute(query)
-    except OperationalError as e:  # if the column already exists this will happen
-        if 'duplicate column name' not in e.args[0]:
-            raise

db_setup/__init__.py

@@ -0,0 +1,136 @@
+import math
+from math import ceil
+
+import scipy.stats
+
+from db_setup.create_triggers import create_triggers
+from db_setup.indices import create_indices
+from db_setup.seeds import seed
+from db_setup.tables import tables
+from game import DB_NAME
+
+
+class StDevFunc:
+    """
+    source: https://stackoverflow.com/a/24423341
+    """
+
+    def __init__(self):
+        self.M = 0.0
+        self.S = 0.0
+        self.k = 1
+
+    def step(self, value):
+        if value is None:
+            return
+        value = float(value)
+        tM = self.M
+        self.M += (value - tM) / self.k
+        self.S += (value - tM) * (value - self.M)
+        self.k += 1
+
+    def finalize(self):
+        if self.k == 1:
+            return None
+        if self.k == 2:
+            return 0.0
+        return math.sqrt(self.S / (self.k - 1))
+
+    def __call__(self, items):
+        for i in items:
+            self.step(i)
+        return self.finalize()
+
+
+class MeanFunc:
+    """
+    source: https://stackoverflow.com/a/24423341
+    """
+
+    def __init__(self):
+        self.sum = 0
+        self.count = 0
+
+    def step(self, value):
+        if value is None:
+            return
+        value = float(value)
+        self.sum += value
+        self.count += 1
+
+    def finalize(self):
+        return self.sum / self.count
+
+    def __call__(self, items):
+        for i in items:
+            self.step(i)
+        return self.finalize()
+
+
+class MeanConfidenceIntervalSizeFunc:
+    def __init__(self):
+        self.std = StDevFunc()
+        self.mean = MeanFunc()
+        self.count = 0
+
+    def step(self, value):
+        self.std.step(value)
+        self.mean.step(value)
+        self.count += 1
+
+    def finalize(self):
+        if self.count == 0:
+            return None  # same as nan for sqlite3
+        if self.count == 1:
+            return math.inf
+        std = self.std.finalize()
+        if std == 0:
+            return 0
+        return self.mean.finalize() - scipy.stats.t.interval(0.95, self.count - 1,
+                                                             loc=self.mean.finalize(),
+                                                             scale=std / math.sqrt(self.count))[0]
+
+    def __call__(self, items):
+        for i in items:
+            self.step(i)
+        return self.finalize()
+
+def str_to_float(s):
+    s = s.replace('.','').replace(',','.')
+    assert not '+' in s or not '-' in s
+    v = float(s.replace('%', ''))
+    if '%' in s:
+        v /= 100
+    return v
+
+
+def create_functions(connection):
+    connection.create_function('CEIL', 1, ceil)
+    connection.create_function('POWER', 2, lambda b, e: b ** e)
+    connection.create_aggregate('CONF', 1, MeanConfidenceIntervalSizeFunc)
+    connection.create_aggregate('TOFLOAT', 1, str_to_float)
+
+
+def set_pragmas(cursor):
+    cursor.execute('PRAGMA foreign_keys=1')
+    cursor.execute('PRAGMA journal_mode = WAL')
+    cursor.execute('PRAGMA synchronous = NORMAL')
+
+
+def setup(cursor):
+    print('Database setup...')
+
+    tables(cursor)
+
+    create_triggers(cursor)
+
+    create_indices(cursor)
+
+    seed(cursor)
+
+
+if __name__ == '__main__':
+    import model
+    model.connect(DB_NAME)
+    setup(model.current_cursor)
+    model.current_connection.commit()

db_setup/create_triggers.py

@@ -0,0 +1,264 @@
+import sqlite3
+from typing import List
+
+from game import MINIMUM_ORDER_AMOUNT
+
+
+def create_triggers(cursor: sqlite3.Cursor):
+    print(' - Creating triggers...')
+    # ensure that the internal rowids of any table are not updated after creation
+    create_triggers_that_restrict_rowid_update(cursor)
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS owned_amount_not_negative_after_insert
+                AFTER INSERT ON ownership
+                WHEN NEW.amount < 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not own an amount less than 0.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS owned_amount_not_negative_after_update
+                AFTER UPDATE ON ownership
+                WHEN NEW.amount < 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not own an amount less than 0.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS amount_positive_after_insert
+                AFTER INSERT ON transactions
+                WHEN NEW.amount <= 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not perform empty transactions.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS amount_positive_after_update
+                AFTER UPDATE ON transactions
+                WHEN NEW.amount <= 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not perform empty transactions.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS order_limit_not_negative_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW."limit" IS NOT NULL AND NEW."limit" < 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not set a limit less than 0.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS order_limit_not_negative_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW."limit" IS NOT NULL AND NEW."limit" < 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not set a limit less than 0.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS order_amount_positive_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW.ordered_amount <= 0 OR NEW.executed_amount < 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not order 0 or less.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS order_amount_positive_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW.ordered_amount <= 0 OR NEW.executed_amount < 0
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not order 0 or less.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS not_more_executed_than_ordered_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW.ordered_amount < NEW.executed_amount
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not execute more than ordered.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS not_more_executed_than_ordered_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW.ordered_amount < NEW.executed_amount
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not execute more than ordered.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS not_more_ordered_than_available_after_insert
+                AFTER INSERT ON orders
+                WHEN NOT NEW.buy AND 0 >
+                    -- owned_amount
+                    COALESCE (
+                        (SELECT amount
+                         FROM ownership
+                         WHERE ownership.rowid = NEW.ownership_id), 0)
+                    - -- sell_ordered_amount
+                    (SELECT COALESCE(SUM(orders.ordered_amount - orders.executed_amount),0)
+                     FROM orders, ownership
+                     WHERE ownership.rowid = orders.ownership_id
+                     AND ownership.rowid = NEW.ownership_id
+                     AND NOT orders.buy) 
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not order more than you own.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS not_more_ordered_than_available_after_update
+                AFTER UPDATE ON orders
+                WHEN NOT NEW.buy AND 0 >
+                    -- owned_amount
+                    COALESCE (
+                        (SELECT amount
+                         FROM ownership
+                         WHERE ownership.rowid = NEW.ownership_id), 0)
+                    - -- sell_ordered_amount
+                    (SELECT COALESCE(SUM(orders.ordered_amount - orders.executed_amount),0)
+                     FROM orders, ownership
+                     WHERE ownership.rowid = orders.ownership_id
+                     AND ownership.rowid = NEW.ownership_id
+                     AND NOT orders.buy) 
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not order more than you own.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS expiry_dt_in_future_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW.expiry_dt <= datetime('now')
+                BEGIN SELECT RAISE(ROLLBACK, 'Order is already expired.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS expiry_dt_in_future_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW.expiry_dt <= datetime('now')
+                BEGIN SELECT RAISE(ROLLBACK, 'Order is already expired.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS stop_loss_requires_limit_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW."limit" IS NULL AND NEW.stop_loss IS NOT NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'Can only set `stop_loss` `for limit orders.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS stop_loss_requires_limit_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW."limit" IS NULL AND NEW.stop_loss IS NOT NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'Can only set `stop_loss` `for limit orders.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS limit_requires_stop_loss_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW."limit" IS NOT NULL AND NEW.stop_loss IS NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'Need to set stop_loss to either True or False for limit orders.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS limit_requires_stop_loss_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW."limit" IS NOT NULL AND NEW.stop_loss IS NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'Need to set stop_loss to either True or False for limit orders.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS minimum_order_amount_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW.ordered_amount < ?
+                BEGIN SELECT RAISE(ROLLBACK, 'There is a minimum amount for new orders.'); END
+                '''.replace('?', str(MINIMUM_ORDER_AMOUNT)))
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS integer_amount_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW.ordered_amount <> ROUND(NEW.ordered_amount)
+                BEGIN SELECT RAISE(ROLLBACK, 'Can only set integer amounts for new orders.'); END
+                '''.replace('?', str(MINIMUM_ORDER_AMOUNT)))
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS dt_monotonic_after_insert
+                AFTER INSERT ON transactions
+                WHEN NEW.dt < (SELECT MAX(dt) FROM transactions t2 WHERE t2.rowid < rowid)
+                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS dt_monotonic_after_update
+                AFTER INSERT ON transactions
+                WHEN NEW.dt < (SELECT MAX(dt) FROM transactions t2 WHERE t2.rowid < rowid)
+                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS news_dt_monotonic_after_insert
+                AFTER INSERT ON news
+                WHEN NEW.dt < (SELECT MAX(dt) FROM news t2 WHERE t2.rowid < rowid)
+                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS news_dt_monotonic_after_insert
+                AFTER INSERT ON news
+                WHEN NEW.dt < (SELECT MAX(dt) FROM news t2 WHERE t2.rowid < rowid)
+                BEGIN SELECT RAISE(ROLLBACK, 'Transaction rowid programming bug, not your fault.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS orders_rowid_sorted_by_creation_time_after_insert
+                AFTER INSERT ON orders
+                WHEN NEW.rowid < (SELECT MAX(rowid) FROM orders o2)
+                BEGIN SELECT RAISE(ROLLBACK, 'Order-rowid programming bug (insert), not your fault.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS news_dt_monotonic_after_update
+                AFTER UPDATE ON orders
+                WHEN NEW.rowid <> OLD.rowid
+                BEGIN SELECT RAISE(ROLLBACK, 'Cannot change number of existing order.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS not_nullify_buyer_id_after_update
+                AFTER UPDATE ON transactions
+                WHEN NEW.buyer_id IS NULL AND OLD.buyer_id IS NOT NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'Cannot nullify buyer_id.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS buyer_id_not_null_after_insert
+                AFTER INSERT ON transactions
+                WHEN NEW.buyer_id IS NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'buyer_id must not be null for new transactions.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS not_nullify_seller_id_after_update
+                AFTER UPDATE ON transactions
+                WHEN NEW.seller_id IS NULL AND OLD.seller_id IS NOT NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'Cannot nullify seller_id.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS seller_id_not_null_after_insert
+                AFTER INSERT ON transactions
+                WHEN NEW.seller_id IS NULL
+                BEGIN SELECT RAISE(ROLLBACK, 'seller_id must not be null for new transactions.'); END
+                ''')
+    cursor.execute('''
+                CREATE TRIGGER IF NOT EXISTS order_history_no_update
+                BEFORE UPDATE ON order_history
+                BEGIN SELECT RAISE(ROLLBACK, 'Can not change order history.'); END
+                ''')
+
+
+def create_combination_cluster_triggers(cursor: sqlite3.Cursor,
+                                        table_name: str,
+                                        foreign_key_column_name: str,
+                                        referenced_tables: List[str],
+                                        kind_column_name: str = 'kind',):
+    valid_kind = '\n    OR '.join("(NEW.{0} = '{1}' AND EXISTS (SELECT * FROM {1} WHERE rowid = NEW.{2}))"
+                                  .format(kind_column_name, table, foreign_key_column_name)
+                                  for table in referenced_tables)
+    cursor.execute('''-- noinspection SqlResolveForFile
+        CREATE TRIGGER valid_{0}_{1}_after_insert
+        AFTER INSERT ON {0}
+        WHEN NOT ({2})
+        BEGIN SELECT RAISE(ROLLBACK, '{0}.{1} is invalid or violating a foreign key constraint.'); END
+    '''.format(table_name, kind_column_name, valid_kind))
+    cursor.execute('''-- noinspection SqlResolveForFile
+        CREATE TRIGGER valid_{0}_{1}_after_update
+        AFTER UPDATE ON {0}
+        WHEN NOT ({2})
+        BEGIN SELECT RAISE(ROLLBACK, '{0}.{1} is invalid or violating a foreign key constraint.'); END
+    '''.format(table_name, kind_column_name, valid_kind))
+    for referenced_table in referenced_tables:
+        cursor.execute('''-- noinspection SqlResolveForFile
+            CREATE TRIGGER {0}_{1}_{3}_foreign_key_before_delete
+            BEFORE DELETE ON {3}
+            WHEN EXISTS (
+                SELECT * FROM {0}
+                WHERE {0}.{4} = OLD.rowid
+                AND {0}.{1} = '{3}'
+            )
+            BEGIN SELECT RAISE(ROLLBACK, '{0}.{1} is violating a foreign key constraint.'); END
+        '''.format(table_name, kind_column_name, valid_kind, referenced_table, foreign_key_column_name))
+
+
+def create_triggers_that_restrict_rowid_update(cursor):
+    cursor.execute('''
+        SELECT name FROM sqlite_master WHERE type='table'
+    ''')
+    tables = [row[0] for row in cursor.fetchall()]
+    for table_name in tables:
+        cursor.execute('''-- noinspection SqlResolveForFile
+                    CREATE TRIGGER restrict_rowid_update_on_{0}
+                    AFTER UPDATE ON {0}
+                    WHEN OLD.rowid <> NEW.rowid
+                    BEGIN SELECT RAISE(ROLLBACK, 'The rowid can not be changed.'); END
+                    '''.format(table_name))

db_setup/indices.py

@@ -0,0 +1,70 @@
+from sqlite3 import Cursor
+
+
+def create_indices(cursor: Cursor):
+    print(' - Creating indices...')
+    print(' - Creating indices...')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS ownership_ownable
+                ON ownership (ownable_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS transactions_ownable
+                ON transactions (ownable_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS orders_expiry
+                ON orders (expiry_dt)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS orders_ownership
+                ON orders (ownership_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS orders_limit
+                ON orders ("limit")
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS transactions_dt
+                ON transactions (dt)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS news_dt
+                ON news (dt)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS ownables_name
+                ON ownables (name)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS users_name
+                ON users (username)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS sessions_id
+                ON sessions (session_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS sessions_user
+                ON sessions (user_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS transactions_seller
+                ON transactions (seller_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS transactions_buyer
+                ON transactions (buyer_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS order_history_id
+                ON order_history (order_id)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS order_canceled
+                ON order_history (archived_dt)
+                ''')
+    cursor.execute('''
+                CREATE INDEX IF NOT EXISTS order_history_ownership
+                ON order_history (ownership_id)
+                ''')

db_setup/seeds/.gitignore

@@ -0,0 +1 @@
+admin_data.py

db_setup/seeds/__init__.py

@@ -0,0 +1,49 @@
+from sqlite3 import Cursor
+
+from game import CURRENCY_NAME
+
+
+def seed(cursor: Cursor):
+    print(' - Seeding initial data...')
+    # ₭ollar
+    cursor.execute('''
+                    INSERT OR IGNORE INTO ownables
+                    (name)
+                    VALUES (?)
+                    ''', (CURRENCY_NAME,))
+    # The bank/external investors
+    cursor.execute('''
+                    INSERT OR IGNORE INTO users
+                    (username,password)
+                    VALUES ('bank','')
+                    ''')
+
+    # bank owns some stuff
+    cursor.execute('''
+    INSERT OR IGNORE INTO ownership
+    (user_id, ownable_id, amount)
+    SELECT (SELECT rowid FROM users WHERE username = 'bank'), 
+            ownables.rowid, 
+            (SELECT COALESCE(SUM(amount),0) FROM ownership WHERE ownable_id = ownables.rowid)
+    FROM ownables
+    ''')
+    cursor.executemany('''
+    INSERT INTO global_control_values (value_name, value)
+    WITH new_value AS (SELECT ? AS name, ? AS value)
+    SELECT new_value.name, new_value.value
+    FROM new_value
+    WHERE NOT EXISTS(SELECT * -- TODO test if this works
+                     FROM global_control_values v2
+                     WHERE v2.value_name = new_value.name
+                       AND v2.value = new_value.value
+                       AND v2.dt = (SELECT MAX(v3.dt)
+                                    FROM global_control_values v3
+                                    WHERE v3.value_name = new_value.name
+                                      AND v3.value = new_value.value))
+    ''', [('banking_license_price', 5e6),
+          ('personal_loan_interest_rate', 0.1),  # may seem a lot but actually this is a credit that you get without any assessment involved
+          ('deposit_facility', -0.05),  # ECB 2020
+          ('marginal_lending_facility', 0.25),  # ECB 2020
+          ('cash_reserve_ratio', 0.01),  # Eurozone 2020
+          ('cash_reserve_free_amount', 1e5),  # Eurozone 2020
+          ])

db_setup/tables.py

@@ -0,0 +1,129 @@
+from sqlite3 import OperationalError
+
+
+def tables(cursor):
+    print(' - Creating tables...')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS users(
+                    rowid INTEGER PRIMARY KEY,
+                    username VARCHAR(10) UNIQUE NOT NULL, 
+                    password VARCHAR(200) NOT NULL)
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS ownables(
+                    rowid INTEGER PRIMARY KEY,
+                    name VARCHAR(10) UNIQUE NOT NULL)
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS ownership(
+                    rowid INTEGER PRIMARY KEY,
+                    user_id INTEGER NOT NULL,
+                    ownable_id INTEGER NOT NULL,
+                    amount CURRENCY NOT NULL DEFAULT 0,
+                    FOREIGN KEY (user_id) REFERENCES users(rowid),
+                    FOREIGN KEY (ownable_id) REFERENCES ownables(rowid),
+                    UNIQUE (user_id, ownable_id)
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS sessions(
+                    rowid INTEGER PRIMARY KEY,
+                    user_id INTEGER NOT NULL,
+                    session_id STRING NOT NULL,
+                    FOREIGN KEY (user_id) REFERENCES users(rowid)
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS orders(
+                    rowid INTEGER PRIMARY KEY,
+                    ownership_id INTEGER NOT NULL,
+                    buy BOOLEAN NOT NULL,
+                    "limit" CURRENCY,
+                    stop_loss BOOLEAN,
+                    ordered_amount CURRENCY NOT NULL,
+                    executed_amount CURRENCY DEFAULT 0 NOT NULL,
+                    expiry_dt DATETIME NOT NULL,
+                    FOREIGN KEY (ownership_id) REFERENCES ownership(rowid)
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS order_history(
+                    rowid INTEGER PRIMARY KEY,
+                    ownership_id INTEGER NOT NULL,
+                    buy BOOLEAN NOT NULL,
+                    "limit" CURRENCY,
+                    ordered_amount CURRENCY NOT NULL,
+                    executed_amount CURRENCY NOT NULL,
+                    expiry_dt DATETIME NOT NULL,
+                    status VARCHAR(20) NOT NULL,
+                    order_id INTEGER NOT NULL, -- order_id is not a FOREIGN KEY since orders are deleted from order table afterwards
+                    archived_dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (ownership_id) REFERENCES ownership(rowid)
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS transactions(
+                    rowid INTEGER PRIMARY KEY,
+                    dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                    price CURRENCY NOT NULL,
+                    ownable_id INTEGER NOT NULL,
+                    amount CURRENCY NOT NULL,
+                    FOREIGN KEY (ownable_id) REFERENCES ownables(rowid)
+                )
+                ''')
+    _add_column_if_not_exists(cursor, '''
+                -- there is a not null constraint for new values that is watched by triggers
+                ALTER TABLE transactions ADD COLUMN buyer_id INTEGER REFERENCES users(rowid)
+                ''')
+    _add_column_if_not_exists(cursor, '''
+                -- there is a not null constraint for new values that is watched by triggers
+                ALTER TABLE transactions ADD COLUMN seller_id INTEGER REFERENCES users(rowid)
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS keys(
+                    rowid INTEGER PRIMARY KEY,
+                    key STRING UNIQUE NOT NULL,
+                    used_by_user_id INTEGER,
+                    FOREIGN KEY (used_by_user_id) REFERENCES users(rowid)
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS news(
+                    rowid INTEGER PRIMARY KEY,
+                    dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                    title VARCHAR(50) NOT NULL
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS banks(
+                    rowid INTEGER PRIMARY KEY,
+                    user_id NOT NULL REFERENCES users(rowid)
+                )
+                ''')
+    cursor.execute('''
+                CREATE TABLE IF NOT EXISTS global_control_values(
+                    rowid INTEGER PRIMARY KEY,
+                    dt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                    value_name VARCHAR NOT NULL,
+                    value FLOAT NOT NULL,
+                    UNIQUE (value_name, dt)
+                )
+                ''')
+    _add_column_if_not_exists(cursor, '''
+                -- there is a not null constraint for new values that is watched by triggers
+                ALTER TABLE users ADD COLUMN salt BLOB NOT NULL DEFAULT 'orderer_is_a_cool_application_]{][{²$%WT§$%GV§$%SF$%&S$%FGGFHBDHJZIF254325'
+                ''')
+
+
+def _add_column_if_not_exists(cursor, query):
+    if 'ALTER TABLE' not in query.upper():
+        raise ValueError('Only alter table queries allowed.')
+    if 'ADD COLUMN' not in query.upper():
+        raise ValueError('Only add column queries allowed.')
+    try:
+        cursor.execute(query)
+    except OperationalError as e:  # if the column already exists this will happen
+        if 'duplicate column name' not in e.args[0]:
+            raise
+        else:
+            pass

debug.py

@@ -1 +1 @@
-debug = False
+debug = True

doc/documentation.py

@@ -0,0 +1,57 @@
+# the general syntax is 
+# {attribute1: value, attribute2: value, ...}
+#
+# for example a valid request to /register would look like 
+# {"email": "user123@example.org", "username": "user123", "password": "FILTERED", "preferred_language": "german"}
+# while a valid request to /events would be the empty object {}
+
+activate_key_required_attributes = ['key', 'session_id']
+activate_key_possible_attributes = ['key', 'session_id']
+
+cancel_order_required_attributes = ['order_id', 'session_id']
+cancel_order_possible_attributes = ['order_id', 'session_id']
+
+change_password_required_attributes = ['password', 'session_id']
+change_password_possible_attributes = ['password', 'session_id']
+
+depot_required_attributes = ['session_id']
+depot_possible_attributes = ['session_id']
+
+gift_required_attributes = ['amount', 'object_name', 'session_id', 'username']
+gift_possible_attributes = ['amount', 'object_name', 'session_id', 'username']
+
+leaderboard_required_attributes = []
+leaderboard_possible_attributes = []
+
+login_required_attributes = ['password', 'username']
+login_possible_attributes = ['password', 'username']
+
+missing_attributes_required_attributes = []
+missing_attributes_possible_attributes = []
+
+news_required_attributes = []
+news_possible_attributes = []
+
+old_orders_required_attributes = ['include_canceled', 'include_executed', 'limit', 'session_id']
+old_orders_possible_attributes = ['include_canceled', 'include_executed', 'limit', 'session_id']
+
+order_required_attributes = ['amount', 'buy', 'ownable', 'session_id', 'time_until_expiration']
+order_possible_attributes = ['amount', 'buy', 'ownable', 'session_id', 'time_until_expiration']
+
+orders_required_attributes = ['session_id']
+orders_possible_attributes = ['session_id']
+
+orders_on_required_attributes = ['ownable', 'session_id']
+orders_on_possible_attributes = ['ownable', 'session_id']
+
+register_required_attributes = ['password', 'username']
+register_possible_attributes = ['password', 'username']
+
+tradables_required_attributes = []
+tradables_possible_attributes = []
+
+trades_required_attributes = ['limit', 'session_id']
+trades_possible_attributes = ['limit', 'session_id']
+
+trades_on_required_attributes = ['limit', 'ownable', 'session_id']
+trades_on_possible_attributes = ['limit', 'ownable', 'session_id']

doc/generate.py

@@ -0,0 +1,35 @@
+# read server controller
+import re
+
+if __name__ == '__main__':
+    with open('server_controller.py', 'r') as file:
+        text = file.read().replace('\n', '')
+
+    text = text.split('def ')
+    del text[0]
+    documentation = '''# the general syntax is 
+# {attribute1: value, attribute2: value, ...}
+#
+# for example a valid request to /register would look like 
+# {"email": "user123@example.org", "username": "user123", "password": "FILTERED", "preferred_language": "german"}
+# while a valid request to /events would be the empty object {}
+\n'''
+    for method in sorted(text):
+        method = 'def ' + method
+        method_name = re.search(r"def (.*?)\s*\(", method)[1]
+        if method_name[0] == '_':
+            continue
+        directly_used_arguments = re.findall(r"json_request\['(.*?)'\]", method)
+        missing_attributes = re.search(r"missing_attributes\(json_request, \[((.|\n)*?)\]\)", method)
+        if missing_attributes is None:
+            print('HINT: method', method_name, 'does not require any parameters')
+            required_arguments = []
+        else:
+            required_arguments = re.findall(r"'(.*?)'", missing_attributes[0])
+        required_arguments = sorted(list(set(required_arguments)))
+        possible_arguments = sorted(list(set(directly_used_arguments + required_arguments)))
+        documentation += method_name + '_required_attributes = ' + str(required_arguments) + '\n'
+        documentation += method_name + '_possible_attributes = ' + str(possible_arguments) + '\n\n'
+    with open("doc/documentation.py", "w", newline='\n') as file:
+        file.write(documentation[:-1])
+    print(documentation)

game.py

@@ -1,3 +1,10 @@
-CURRENCY_NAME = "₭ollar"
-MINIMUM_ORDER_AMOUNT = 1
-DEFAULT_ORDER_EXPIRY = 43200
+from lib.db_log import DBLog
+
+CURRENCY_NAME = "₭ollar"
+MINIMUM_ORDER_AMOUNT = 1
+DEFAULT_ORDER_EXPIRY = 43200
+DB_NAME = 'orderer'
+COPYRIGHT_INFRINGEMENT_PROBABILITY = 0.05
+ROOT_URL = "/orderer.zip"
+
+logger = DBLog()

jobs/README.md

@@ -0,0 +1,2 @@
+# Recommended order of running jobs
+TODO

jobs/run_multiple_jobs.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+
+from util import main_wrapper
+
+
+@main_wrapper
+def main():
+    schedule = [
+        # TODO fill
+    ]
+    for job in schedule:
+        print('Starting', job.__name__, datetime.now().strftime("%H:%M:%S"))
+        job.run()
+
+
+if __name__ == '__main__':
+    main()

lib/db_log.py

@@ -0,0 +1,134 @@
+import logging
+import os
+import sqlite3 as db
+import sys
+import time
+from math import inf
+from shutil import copyfile
+from typing import Optional
+
+import git
+
+DBName = str
+
+connected_dbs = [DBName]
+
+# get current commit at start time of program
+repo = git.Repo(search_parent_directories=True)
+CURRENT_SHA = repo.head.object.hexsha
+
+
+class DBLog:
+    def __init__(self, db_name='log.db', create_if_not_exists=True):
+        if db_name in connected_dbs:
+            raise ValueError(f'There is already a connection to {db_name}.'
+                             'If you want to re-use the same connection you can get it from `db_log.connected_dbs`.'
+                             'If you want to disconnect you can call log.disconnect().')
+        self.connection: Optional[db.Connection] = None
+        self.cursor: Optional[db.Cursor] = None
+        self.db_name: Optional[DBName] = None
+
+        db_name = db_name.lower()
+        if not os.path.isfile(db_name) and not create_if_not_exists:
+            raise FileNotFoundError('There is no database with this name.')
+        creating_new_db = not os.path.isfile(db_name)
+        try:
+            db_connection = db.connect(db_name, check_same_thread=False)
+            # db_setup.create_functions(db_connection)
+            # db_setup.set_pragmas(db_connection.cursor())
+            # connection.text_factory = lambda x: x.encode('latin-1')
+        except db.Error as e:
+            print("Database error %s:" % e.args[0])
+            raise
+
+        self.connection = db_connection
+        self.cursor = self.connection.cursor()
+        self.db_name = db_name
+        if creating_new_db:
+            try:
+                if os.path.isfile('/test-db/' + db_name):
+                    print('Using test logs')
+                    copyfile('/test-db/' + db_name, db_name)
+                else:
+                    self.setup()
+            except Exception:
+                if self.connection is not None:
+                    self.connection.rollback()
+                os.remove(db_name)
+                raise
+        self.connected = True
+        self.min_level = -inf
+
+    def disconnect(self, rollback=True):
+        if rollback:
+            self.connection.rollback()
+        else:
+            self.connection.commit()
+        self.connection.close()
+        self.connected = False
+
+    def setup(self):
+        self.cursor.execute('''
+        CREATE TABLE IF NOT EXISTS entries(
+            rowid INTEGER PRIMARY KEY,
+            dt_created DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            message TEXT NOT NULL,
+            data BLOB, -- can be null
+            pid INTEGER NOT NULl,
+            message_type VARCHAR (25) NOT NULL, -- a plain text message title
+            level INTEGER NOT NULL, -- relates to logging.ERROR and similar ones
+            head_hex_sha VARCHAR-- SHA of currently checked out commit
+        )
+        ''')
+
+    def log(self,
+            message,
+            level,
+            message_type='generic',
+            data=None,
+            dt_created=None,
+            current_pid=None,
+            current_head_hex_sha=CURRENT_SHA,
+            data_serialization_method=lambda x: x):
+        if level < self.min_level:
+            return
+        if dt_created is None:
+            dt_created = round(time.time())
+        if current_pid is None:
+            current_pid = os.getpid()
+        data: str = data_serialization_method(data)
+        self.cursor.execute('''
+        INSERT INTO entries(message, data, dt_created, pid, head_hex_sha, message_type, level)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+        ''', (message, data, dt_created, current_pid, current_head_hex_sha, message_type, level))
+
+    def debug(self, message, *args, **kwargs):
+        self.log(message, logging.DEBUG, *args, **kwargs)
+
+    def info(self, message, *args, **kwargs):
+        self.log(message, logging.INFO, *args, **kwargs)
+
+    def warning(self, message, *args, **kwargs):
+        self.log(message, logging.WARNING, *args, **kwargs)
+
+    warn = warning
+
+    def error(self, message, *args, **kwargs):
+        self.log(message, logging.ERROR, *args, **kwargs)
+
+    def critical(self, message, *args, **kwargs):
+        self.log(message, logging.CRITICAL, *args, **kwargs)
+
+    fatal = critical
+
+    def exception(self, msg, *args, data=None, **kwargs):
+        if data is None:
+            data = sys.exc_info()
+        self.error(msg, *args, data=data, **kwargs)
+
+    def commit(self):
+        c = time.clock()
+        self.connection.commit()
+        delta = time.clock() - c
+        print(f'Committing log files took {delta} seconds')
+        return delta

lib/print_exc_plus.py

@@ -0,0 +1,164 @@
+import os
+import re
+import sys
+import traceback
+from itertools import islice
+from types import FrameType
+from typing import Sized, Dict, Tuple
+
+from lib.threading_timer_decorator import exit_after
+
+try:
+    import numpy
+except ImportError:
+    numpy = None
+
+FORMATTING_OPTIONS = {
+    'MAX_LINE_LENGTH': 1024,
+    'SHORT_LINE_THRESHOLD': 128,
+    'MAX_NEWLINES': 20,
+}
+ID = int
+
+
+# noinspection PyPep8Naming
+def name_or_str(X):
+    try:
+        return re.search(r"<class '?(.*?)'?>", str(X))[1]
+    except TypeError:  # if not found
+        return str(X)
+
+
+@exit_after(2)
+def type_string(x):
+    if numpy is not None and isinstance(x, numpy.ndarray):
+        return name_or_str(type(x)) + str(x.shape)
+    elif isinstance(x, Sized):
+        return name_or_str(type(x)) + f'({len(x)})'
+    else:
+        return name_or_str(type(x))
+
+
+@exit_after(2)
+def to_string_with_timeout(x):
+    return str(x)
+
+
+def nth_index(iterable, value, n):
+    matches = (idx for idx, val in enumerate(iterable) if val == value)
+    return next(islice(matches, n - 1, n), None)
+
+
+def print_exc_plus():
+    """
+    Print the usual traceback information, followed by a listing of all the
+    local variables in each frame.
+    """
+    limit = FORMATTING_OPTIONS['MAX_LINE_LENGTH']
+    max_newlines = FORMATTING_OPTIONS['MAX_NEWLINES']
+    tb = sys.exc_info()[2]
+    if numpy is not None:
+        options = numpy.get_printoptions()
+        numpy.set_printoptions(precision=2, edgeitems=2, floatmode='maxprec', threshold=20, linewidth=120)
+    else:
+        options = {}
+    stack = []
+    long_printed_objs: Dict[ID, Tuple[str, FrameType]] = {}
+
+    while tb:
+        stack.append(tb.tb_frame)
+        tb = tb.tb_next
+    for frame in stack:
+        if frame is not stack[0]:
+            print('-' * 40)
+        try:
+            print("Frame %s in %s at line %s" % (frame.f_code.co_name,
+                                                 os.path.relpath(frame.f_code.co_filename),
+                                                 frame.f_lineno))
+        except ValueError:  # if path is not relative
+            print("Frame %s in %s at line %s" % (frame.f_code.co_name,
+                                                 frame.f_code.co_filename,
+                                                 frame.f_lineno))
+        for key, value in frame.f_locals.items():
+            # We have to be careful not to cause a new error in our error
+            # printer! Calling str() on an unknown object could cause an
+            # error we don't want.
+
+            # noinspection PyBroadException
+            try:
+                key_string = to_string_with_timeout(key)
+            except KeyboardInterrupt:
+                key_string = "<TIMEOUT WHILE PRINTING KEY>"
+            except Exception:
+                key_string = "<ERROR WHILE PRINTING KEY>"
+
+            # noinspection PyBroadException
+            try:
+                type_as_string = type_string(value)
+            except KeyboardInterrupt:
+                type_as_string = "<TIMEOUT WHILE PRINTING TYPE>"
+            except Exception as e:
+                # noinspection PyBroadException
+                try:
+                    type_as_string = f"<{type(e).__name__} WHILE PRINTING TYPE>"
+                except Exception:
+                    type_as_string = "<ERROR WHILE PRINTING TYPE>"
+
+            if id(value) in long_printed_objs:
+                prev_key_string, prev_frame = long_printed_objs[id(value)]
+                if prev_frame is frame:
+                    print("\t%s is the same as '%s'" %
+                          (key_string + ' : ' + type_as_string,
+                           prev_key_string))
+                else:
+                    print("\t%s is the same as '%s' in frame %s in %s at line %s." %
+                          (key_string + ' : ' + type_as_string,
+                           prev_key_string,
+                           prev_frame.f_code.co_name,
+                           os.path.relpath(prev_frame.f_code.co_filename),
+                           prev_frame.f_lineno))
+                continue
+
+            # noinspection PyBroadException
+            try:
+                value_string = to_string_with_timeout(value)
+            except KeyboardInterrupt:
+                value_string = "<TIMEOUT WHILE PRINTING VALUE>"
+            except Exception:
+                value_string = "<ERROR WHILE PRINTING VALUE>"
+            line: str = '\t' + key_string + ' : ' + type_as_string + ' = ' + value_string
+            if limit is not None and len(line) > limit:
+                line = line[:limit - 1] + '...'
+            if max_newlines is not None and line.count('\n') > max_newlines:
+                line = line[:nth_index(line, '\n', max_newlines)].strip() + '... (' + str(
+                    line[nth_index(line, '\n', max_newlines):].count('\n')) + ' more lines)'
+            if len(line) > FORMATTING_OPTIONS['SHORT_LINE_THRESHOLD']:
+                long_printed_objs[id(value)] = key_string, frame
+            print(line)
+
+    traceback.print_exc()
+    if numpy is not None:
+        numpy.set_printoptions(**options)
+
+
+def main():
+    def fun1(c, d, e):
+        return fun2(c, d + e)
+
+    def fun2(g, h):
+        raise RuntimeError
+
+    def fun3(z):
+        return numpy.zeros(shape=z)
+
+    try:
+        import numpy as np
+        fun1(numpy.random.normal(size=(3, 4, 5, 6)), '12321', '123')
+        data = '???' * 100
+        fun3(data)
+    except:
+        print_exc_plus()
+
+
+if __name__ == '__main__':
+    main()

lib/progress_bar.py

@@ -0,0 +1,194 @@
+import functools
+import math
+import time
+from math import floor
+from typing import Iterable, Sized, Iterator
+
+
+class ProgressBar(Sized, Iterable):
+    def __iter__(self) -> Iterator:
+        self.check_if_num_steps_defined()
+        self.current_iteration = -1  # start counting at the end of the first epoch
+        self.current_iterator = iter(self._backend)
+        self.start_time = time.clock()
+        return self
+
+    def __init__(self,
+                 num_steps=None,
+                 prefix='',
+                 suffix='',
+                 line_length=75,
+                 empty_char='-',
+                 fill_char='#',
+                 print_eta=True,
+                 decimals=1):
+        self.decimals = decimals
+        self.line_length = line_length
+        self.suffix = suffix
+        self.empty_char = empty_char
+        self.prefix = prefix
+        self.fill_char = fill_char
+        self.print_eta = print_eta
+        self.current_iteration = 0
+        self.last_printed_value = None
+        self.current_iterator = None
+        self.start_time = time.clock()
+
+        try:
+            self._backend = range(num_steps)
+        except TypeError:
+            if isinstance(num_steps, Sized):
+                if isinstance(num_steps, Iterable):
+                    self._backend = num_steps
+                else:
+                    self._backend = range(len(num_steps))
+            elif num_steps is None:
+                self._backend = None
+            else:
+                raise
+
+        assert num_steps is None or isinstance(self._backend, (Iterable, Sized))
+
+    def set_num_steps(self, num_steps):
+        try:
+            self._backend = range(num_steps)
+        except TypeError:
+            if isinstance(num_steps, Sized):
+                if isinstance(num_steps, Iterable):
+                    self._backend = num_steps
+                else:
+                    self._backend = range(len(num_steps))
+            elif num_steps is None:
+                self._backend = None
+            else:
+                raise
+
+        assert num_steps is None or isinstance(self._backend, (Iterable, Sized))
+
+    def __len__(self):
+        return len(self._backend)
+
+    def __next__(self):
+        self.print_progress()
+        try:
+            result = next(self.current_iterator)
+            self.increment_iteration()
+            self.print_progress()
+            return result
+        except StopIteration:
+            self.increment_iteration()
+            self.print_progress()
+            raise
+
+    def step(self, num_iterations=1):
+        self.current_iteration += num_iterations
+        self.print_progress()
+
+    def print_progress(self, iteration=None):
+        """
+        Call in a loop to create terminal progress bar
+        @params:
+            iteration   - Optional  : current iteration (Int)
+        """
+        if iteration is not None:
+            self.current_iteration = iteration
+        try:
+            progress = self.current_iteration / len(self)
+        except ZeroDivisionError:
+            progress = 1
+        if self.current_iteration == 0:
+            self.start_time = time.clock()
+        if self.print_eta and progress > 0:
+            time_spent = (time.clock() - self.start_time)
+            eta = time_spent / progress * (1 - progress)
+            if progress == 1:
+                eta = f' T = {int(time_spent / 60):02d}:{round(time_spent % 60):02d}'
+            else:
+                eta = f' ETA {int(eta / 60):02d}:{round(eta % 60):02d}'
+        else:
+            eta = ''
+        percent = ("{0:" + str(4 + self.decimals) + "." + str(self.decimals) + "f}").format(100 * progress)
+        bar_length = self.line_length - len(self.prefix) - len(self.suffix) - len(eta) - 4 - 6
+        try:
+            filled_length = int(bar_length * self.current_iteration // len(self))
+        except ZeroDivisionError:
+            filled_length = bar_length
+        if math.isclose(bar_length * progress, filled_length):
+            overflow = 0
+        else:
+            overflow = bar_length * progress - filled_length
+            overflow *= 10
+            overflow = floor(overflow)
+        assert overflow in range(10), overflow
+        if overflow > 0:
+            bar = self.fill_char * filled_length + str(overflow) + self.empty_char * (bar_length - filled_length - 1)
+        else:
+            bar = self.fill_char * filled_length + self.empty_char * (bar_length - filled_length)
+
+        print_value = '\r{0} |{1}| {2}% {4}{3}'.format(self.prefix, bar, percent, eta, self.suffix)
+        if self.current_iteration == len(self):
+            print_value += '\n'  # Print New Line on Complete
+        if self.last_printed_value == print_value:
+            return
+        self.last_printed_value = print_value
+        print(print_value, end='')
+
+    def increment_iteration(self):
+        self.current_iteration += 1
+        if self.current_iteration > len(self):  # catches the special case at the end of the bar
+            self.current_iteration %= len(self)
+
+    def monitor(self, func=None):
+        """ Decorates the given function func to print a progress bar before and after each call. """
+        if func is None:
+            # Partial application, to be able to specify extra keyword
+            # arguments in decorators
+            return functools.partial(self.monitor)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            self.check_if_num_steps_defined()
+            self.print_progress()
+            result = func(*args, **kwargs)
+            self.increment_iteration()
+            self.print_progress()
+            return result
+
+        return wrapper
+
+    def check_if_num_steps_defined(self):
+        if self._backend is None:
+            raise RuntimeError('You need to specify the number of iterations before starting to iterate. '
+                               'You can either pass it to the constructor or use the method `set_num_steps`.')
+
+
+if __name__ == '__main__':
+    # Einfach beim iterieren verwenden
+    for x in ProgressBar([0.5, 2, 0.5]):
+        time.sleep(x)
+
+    # manuell aufrufen
+    data = [1, 5, 5, 6, 12, 3, 4, 5]
+    y = 0
+    p = ProgressBar(len(data))
+    for x in data:
+        p.print_progress()
+        time.sleep(0.2)
+        y += x
+        p.current_iteration += 1
+        p.print_progress()
+
+    print(y)
+
+    # oder einfach bei jedem funktionsaufruf den balken printen
+    p = ProgressBar()
+
+
+    @p.monitor
+    def heavy_computation(t=0.25):
+        time.sleep(t)
+
+
+    p.set_num_steps(10)  # 10 steps pro balken
+    for _ in range(20):  # zeichnet 2 balken
+        heavy_computation(0.25)

lib/stack_tracer.py

@@ -0,0 +1,106 @@
+"""Stack tracer for multi-threaded applications.
+
+
+Usage:
+
+import stacktracer
+stacktracer.start_trace("trace.html",interval=5,auto=True) # Set auto flag to always update file!
+....
+stacktracer.stop_trace()
+"""
+
+import sys
+import traceback
+
+from pygments import highlight
+from pygments.formatters import HtmlFormatter
+from pygments.lexers import PythonLexer
+
+
+# Taken from http://bzimmer.ziclix.com/2008/12/17/python-thread-dumps/
+
+def stacktraces():
+    code = []
+    for threadId, stack in sys._current_frames().items():
+        code.append("\n# ThreadID: %s" % threadId)
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
+            if line:
+                code.append("  %s" % (line.strip()))
+
+    return highlight("\n".join(code), PythonLexer(), HtmlFormatter(
+        full=False,
+        # style="native",
+        noclasses=True,
+    ))
+
+
+# This part was made by nagylzs
+import os
+import time
+import threading
+
+
+class TraceDumper(threading.Thread):
+    """Dump stack traces into a given file periodically."""
+
+    def __init__(self, fpath, interval, auto):
+        """
+        @param fpath: File path to output HTML (stack trace file)
+        @param auto: Set flag (True) to update trace continuously.
+            Clear flag (False) to update only if file not exists.
+            (Then delete the file to force update.)
+        @param interval: In seconds: how often to update the trace file.
+        """
+        assert (interval > 0.1)
+        self.auto = auto
+        self.interval = interval
+        self.fpath = os.path.abspath(fpath)
+        self.stop_requested = threading.Event()
+        threading.Thread.__init__(self)
+
+    def run(self):
+        while not self.stop_requested.isSet():
+            time.sleep(self.interval)
+            if self.auto or not os.path.isfile(self.fpath):
+                self.stacktraces()
+
+    def stop(self):
+        self.stop_requested.set()
+        self.join()
+        try:
+            if os.path.isfile(self.fpath):
+                os.unlink(self.fpath)
+        except:
+            pass
+
+    def stacktraces(self):
+        fout = open(self.fpath, "w+")
+        try:
+            fout.write(stacktraces())
+        finally:
+            fout.close()
+
+
+_tracer = None
+
+
+def trace_start(fpath, interval=5, auto=True):
+    """Start tracing into the given file."""
+    global _tracer
+    if _tracer is None:
+        _tracer = TraceDumper(fpath, interval, auto)
+        _tracer.setDaemon(True)
+        _tracer.start()
+    else:
+        raise Exception("Already tracing to %s" % _tracer.fpath)
+
+
+def trace_stop():
+    """Stop tracing."""
+    global _tracer
+    if _tracer is None:
+        raise Exception("Not tracing, cannot stop.")
+    else:
+        _trace.stop()
+        _trace = None

lib/threading_timer_decorator.py

@@ -0,0 +1,106 @@
+from __future__ import print_function
+
+import sys
+import threading
+from time import sleep
+
+try:
+    import thread
+except ImportError:
+    import _thread as thread
+
+try:  # use code that works the same in Python 2 and 3
+    range, _print = xrange, print
+
+
+    def print(*args, **kwargs):
+        flush = kwargs.pop('flush', False)
+        _print(*args, **kwargs)
+        if flush:
+            kwargs.get('file', sys.stdout).flush()
+except NameError:
+    pass
+
+
+def cdquit(fn_name):
+    # print to stderr, unbuffered in Python 2.
+    print('{0} took too long'.format(fn_name), file=sys.stderr)
+    sys.stderr.flush()  # Python 3 stderr is likely buffered.
+    thread.interrupt_main()  # raises KeyboardInterrupt
+
+
+def exit_after(s):
+    '''
+    use as decorator to exit process if
+    function takes longer than s seconds
+    '''
+
+    def outer(fn):
+        def inner(*args, **kwargs):
+            timer = threading.Timer(s, cdquit, args=[fn.__name__])
+            timer.start()
+            try:
+                result = fn(*args, **kwargs)
+            finally:
+                timer.cancel()
+            return result
+
+        return inner
+
+    return outer
+
+
+def call_method_with_timeout(method, timeout, *args, **kwargs):
+    return exit_after(timeout)(method)(*args, **kwargs)
+
+
+@exit_after(1)
+def a():
+    print('a')
+
+
+@exit_after(2)
+def b():
+    print('b')
+    sleep(1)
+
+
+@exit_after(3)
+def c():
+    print('c')
+    sleep(2)
+
+
+@exit_after(4)
+def d():
+    print('d started')
+    for i in range(10):
+        sleep(1)
+        print(i)
+
+
+@exit_after(5)
+def countdown(n):
+    print('countdown started', flush=True)
+    for i in range(n, -1, -1):
+        print(i, end=', ', flush=True)
+        sleep(1)
+    print('countdown finished')
+
+
+def main():
+    a()
+    b()
+    c()
+    try:
+        d()
+    except KeyboardInterrupt as error:
+        print('d should not have finished, printing error as expected:')
+        print(error)
+    countdown(3)
+    countdown(10)
+    print('This should not print!!!')
+
+
+if __name__ == '__main__':
+    main()

lib/tuned_cache.py

@@ -0,0 +1,136 @@
+import functools
+import sys
+from copy import deepcopy
+
+assert 'joblib' not in sys.modules, 'Import tuned cache before joblib'
+
+# noinspection PyProtectedMember,PyPep8
+import joblib
+# noinspection PyProtectedMember,PyPep8
+from joblib._compat import PY3_OR_LATER
+# noinspection PyProtectedMember,PyPep8
+from joblib.func_inspect import _clean_win_chars
+# noinspection PyProtectedMember,PyPep8
+from joblib.memory import MemorizedFunc, _FUNCTION_HASHES, NotMemorizedFunc, Memory
+
+_FUNC_NAMES = {}
+
+
+# noinspection SpellCheckingInspection
+class TunedMemory(Memory):
+    def cache(self, func=None, ignore=None, verbose=None, mmap_mode=False):
+        """ Decorates the given function func to only compute its return
+            value for input arguments not cached on disk.
+
+            Parameters
+            ----------
+            func: callable, optional
+                The function to be decorated
+            ignore: list of strings
+                A list of arguments name to ignore in the hashing
+            verbose: integer, optional
+                The verbosity mode of the function. By default that
+                of the memory object is used.
+            mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
+                The memmapping mode used when loading from cache
+                numpy arrays. See numpy.load for the meaning of the
+                arguments. By default that of the memory object is used.
+
+            Returns
+            -------
+            decorated_func: MemorizedFunc object
+                The returned object is a MemorizedFunc object, that is
+                callable (behaves like a function), but offers extra
+                methods for cache lookup and management. See the
+                documentation for :class:`joblib.memory.MemorizedFunc`.
+        """
+        if func is None:
+            # Partial application, to be able to specify extra keyword
+            # arguments in decorators
+            return functools.partial(self.cache, ignore=ignore,
+                                     verbose=verbose, mmap_mode=mmap_mode)
+        if self.store_backend is None:
+            return NotMemorizedFunc(func)
+        if verbose is None:
+            verbose = self._verbose
+        if mmap_mode is False:
+            mmap_mode = self.mmap_mode
+        if isinstance(func, TunedMemorizedFunc):
+            func = func.func
+        return TunedMemorizedFunc(func, location=self.store_backend,
+                                  backend=self.backend,
+                                  ignore=ignore, mmap_mode=mmap_mode,
+                                  compress=self.compress,
+                                  verbose=verbose, timestamp=self.timestamp)
+
+
+class TunedMemorizedFunc(MemorizedFunc):
+    def __call__(self, *args, **kwargs):
+        # Also store in the in-memory store of function hashes
+        if self.func not in _FUNCTION_HASHES:
+            if PY3_OR_LATER:
+                is_named_callable = (hasattr(self.func, '__name__') and
+                                     self.func.__name__ != '<lambda>')
+            else:
+                is_named_callable = (hasattr(self.func, 'func_name') and
+                                     self.func.func_name != '<lambda>')
+            if is_named_callable:
+                # Don't do this for lambda functions or strange callable
+                # objects, as it ends up being too fragile
+                func_hash = self._hash_func()
+                try:
+                    _FUNCTION_HASHES[self.func] = func_hash
+                except TypeError:
+                    # Some callable are not hashable
+                    pass
+
+        # return same result as before
+        return MemorizedFunc.__call__(self, *args, **kwargs)
+
+
+old_get_func_name = joblib.func_inspect.get_func_name
+
+
+def tuned_get_func_name(func, resolv_alias=True, win_characters=True):
+    if (func, resolv_alias, win_characters) not in _FUNC_NAMES:
+        _FUNC_NAMES[(func, resolv_alias, win_characters)] = old_get_func_name(func, resolv_alias, win_characters)
+
+        if len(_FUNC_NAMES) > 1000:
+            # keep cache small and fast
+            for idx, k in enumerate(_FUNC_NAMES.keys()):
+                if idx % 2:
+                    del _FUNC_NAMES[k]
+        # print('cache size ', len(_FUNC_NAMES))
+
+    return deepcopy(_FUNC_NAMES[(func, resolv_alias, win_characters)])
+
+
+joblib.func_inspect.get_func_name = tuned_get_func_name
+joblib.memory.get_func_name = tuned_get_func_name
+
+
+def main():
+    class A:
+        test_cache = TunedMemory('.cache/test_cache', verbose=1)
+
+        def __init__(self, a):
+            self.a = a
+            self.compute = self.test_cache.cache(self.compute)
+
+        def compute(self):
+            return self.a + 1
+
+    a1, a2 = A(2), A(2)
+    print(a1.compute())
+    print('---')
+    print(a2.compute())
+    print('---')
+    a1.a = 3
+    print(a1.compute())
+    print('---')
+    print(a2.compute())
+    print('---')
+
+
+if __name__ == '__main__':
+    main()

model.py

@@ -1,1182 +1,1225 @@
-import random
-import re
-import sqlite3 as db
-import sys
-import uuid
-from math import floor
-
-from passlib.handlers.sha2_crypt import sha256_crypt
-
-import db_setup
-import trading_bot
-from debug import debug
-from game import CURRENCY_NAME
-from util import random_chars, salt
-
-# connection: db.Connection = None
-# cursor: db.Cursor = None
-connection = None  # no type annotations in python 3.5
-cursor = None  # no type annotations in python 3.5
-db_name = None
-
-
-def query_save_name():
-    global db_name
-    if debug:
-        db_name = 'test.db'
-        return
-    while True:
-        save_name = input('Name of the savegame: ')
-        if re.match(r"[A-Za-z0-9.-]{0,50}", save_name):
-            db_name = save_name + '.db'
-            return
-        else:
-            print('Must match "[A-Za-z0-9.-]{0,50}"')
-
-
-def connect(reconnect=False):
-    global connection
-    global cursor
-    global db_name
-    if reconnect:
-        connection.commit()
-        connection.close()
-        cursor = None
-        connection = None
-        db_name = None
-
-    if connection is None or cursor is None:
-        query_save_name()
-
-        try:
-            connection = db.connect(db_name)
-            # connection.text_factory = lambda x: unicode(x, 'utf-8', 'ignore')
-
-            cursor = connection.cursor()
-
-        except db.Error as e:
-            print("Database error %s:" % e.args[0])
-            sys.exit(1)
-
-        # finally:
-        #     if con is not None:
-        #         con.close()
-
-
-def setup():
-    connect()
-
-    db_setup.setup(cursor)
-
-    connection.commit()
-
-
-def used_key_count():
-    connect()
-
-    cursor.execute('''
-        SELECT COUNT(*) -- rarely executed, no index needed, O(n) query
-        FROM keys
-        WHERE used_by_user_id IS NOT NULL
-        ''')
-
-    return cursor.fetchone()[0]
-
-
-def login(username, password):
-    connect()
-
-    # do not allow login as bank or with empty password
-    if username == 'bank' and not debug:
-        return None
-    if password == '' and not debug:
-        return None
-
-    cursor.execute('''
-                SELECT rowid, password
-                FROM users
-                WHERE username = ?
-                ''', (username,))
-    data = cursor.fetchone()
-    if not data:
-        return None
-    hashed_password = data[1]
-    user_id = data[0]
-    # if a ValueError occurs here, then most likely a password that was stored as plain text
-    if sha256_crypt.verify(password + salt, hashed_password):
-        return new_session(user_id)
-    else:
-        return None
-
-
-def register(username, password, game_key):
-    connect()
-    if username == '':
-        return False
-    if password == '':
-        return False
-    cursor.execute('''
-                INSERT INTO users 
-                (username, password)
-                VALUES (? , ?)
-                ''', (username, password))
-    if game_key != '':
-        if valid_key(game_key):
-            activate_key(game_key, get_user_id_by_name(username))
-    return True
-
-
-def own(user_id, ownable_name, amount=0):
-    if not isinstance(ownable_name, str):
-        return AssertionError('A name must be a string.')
-
-    cursor.execute('''
-                INSERT OR IGNORE INTO ownership (user_id, ownable_id, amount)
-                SELECT ?, (SELECT rowid FROM ownables WHERE name = ?), ?
-                ''', (user_id, ownable_name, amount))
-
-
-def send_ownable(from_user_id, to_user_id, ownable_id, amount):
-    connect()
-
-    if amount < 0:
-        raise AssertionError('Can not send negative amount')
-
-    cursor.execute('''
-                UPDATE ownership
-                SET amount = amount - ?
-                WHERE user_id = ?
-                AND ownable_id = ?
-                ''', (amount, from_user_id, ownable_id,))
-
-    own(to_user_id,ownable_name_by_id(ownable_id))
-
-    cursor.execute('''
-                UPDATE ownership
-                SET amount = amount + ?
-                WHERE user_id = ?
-                AND ownable_id = ?
-                ''', (amount, to_user_id, ownable_id,))
-    return True
-
-
-def valid_key(key):
-    connect()
-
-    cursor.execute('''
-                SELECT key
-                FROM keys
-                WHERE used_by_user_id IS NULL
-                AND key = ?
-                ''', (key,))
-
-    if cursor.fetchone():
-        return True
-    else:
-        return False
-
-
-def new_session(user_id):
-    connect()
-
-    session_id = str(uuid.uuid4())
-
-    cursor.execute('''
-                INSERT INTO SESSIONS 
-                (user_id, session_id)
-                VALUES (? , ?)
-                ''', (user_id, session_id))
-
-    return session_id
-
-
-def save_key(key):
-    connect()
-
-    cursor.execute('''
-                INSERT INTO keys 
-                (key)
-                VALUES (?)
-                ''', (key,))
-
-
-def drop_old_sessions():
-    connect()
-
-    cursor.execute(''' -- no need to optimize this very well
-                DELETE FROM sessions
-                WHERE 
-                    (SELECT COUNT(*) as newer
-                     FROM sessions s2
-                     WHERE user_id = s2.user_id
-                     AND rowid < s2.rowid) >= 10
-                ''')
-
-
-def user_exists(username):
-    connect()
-
-    cursor.execute('''
-                SELECT rowid
-                FROM users
-                WHERE username = ?
-                ''', (username,))
-
-    if cursor.fetchone():
-        return True
-    else:
-        return False
-
-
-def get_user_id_by_session_id(session_id):
-    connect()
-
-    cursor.execute('''
-        SELECT users.rowid
-        FROM sessions, users
-        WHERE sessions.session_id = ?
-        AND users.rowid = sessions.user_id
-        ''', (session_id,))
-
-    ids = cursor.fetchone()
-    if not ids:
-        return False
-    return ids[0]
-
-
-def get_user_id_by_name(username):
-    connect()
-
-    cursor.execute('''
-        SELECT users.rowid
-        FROM users
-        WHERE username = ?
-        ''', (username,))
-
-    return cursor.fetchone()[0]
-
-
-def get_user_ownership(user_id):
-    connect()
-
-    cursor.execute('''
-        SELECT 
-            ownables.name, 
-            ownership.amount, 
-            COALESCE (
-            CASE -- sum score for each of the users ownables
-            WHEN ownership.ownable_id = ? THEN 1
-            ELSE (SELECT price 
-                  FROM transactions
-                  WHERE ownable_id = ownership.ownable_id 
-                  ORDER BY rowid DESC -- equivalent to ordering by dt
-                  LIMIT 1)
-            END, 0) AS price, 
-            (SELECT MAX("limit") 
-             FROM orders, ownership o2
-             WHERE o2.rowid = orders.ownership_id
-             AND o2.ownable_id = ownership.ownable_id
-             AND buy
-             AND NOT stop_loss) AS bid, 
-            (SELECT MIN("limit") 
-             FROM orders, ownership o2
-             WHERE o2.rowid = orders.ownership_id
-             AND o2.ownable_id = ownership.ownable_id
-             AND NOT buy
-             AND NOT stop_loss) AS ask
-        FROM ownership, ownables
-        WHERE user_id = ?
-        AND (ownership.amount > 0 OR ownership.ownable_id = ?)
-        AND ownership.ownable_id = ownables.rowid
-        ORDER BY ownables.rowid ASC
-        ''', (currency_id(), user_id, currency_id(),))
-
-    return cursor.fetchall()
-
-
-def activate_key(key, user_id):
-    connect()
-    cursor.execute('''
-                UPDATE keys
-                SET used_by_user_id = ?
-                WHERE used_by_user_id IS NULL
-                AND key = ?
-                ''', (user_id, key,))
-
-    send_ownable(bank_id(), user_id, currency_id(), 1000)
-
-
-def bank_id():
-    connect()
-
-    cursor.execute('''
-        SELECT users.rowid
-        FROM users
-        WHERE username = 'bank'
-        ''')
-
-    return cursor.fetchone()[0]
-
-
-def valid_session_id(session_id):
-    connect()
-
-    cursor.execute('''
-                SELECT rowid
-                FROM sessions
-                WHERE session_id = ?
-                ''', (session_id,))
-
-    if cursor.fetchone():
-        return True
-    else:
-        return False
-
-
-def get_user_orders(user_id):
-    connect()
-
-    cursor.execute('''
-        SELECT 
-            CASE 
-                WHEN orders.buy  THEN 'Buy'
-                ELSE 'Sell'
-            END,
-            ownables.name, 
-            (orders.ordered_amount - orders.executed_amount) || '/' || orders.ordered_amount, 
-            orders."limit", 
-            CASE 
-                WHEN orders."limit" IS NULL THEN NULL 
-                WHEN orders.stop_loss THEN 'Yes'
-                ELSE 'No'
-            END, 
-            datetime(orders.expiry_dt, 'localtime'),
-            orders.rowid
-        FROM orders, ownables, ownership
-        WHERE ownership.user_id = ?
-        AND ownership.ownable_id = ownables.rowid
-        AND orders.ownership_id = ownership.rowid
-        ORDER BY ownables.name ASC, orders.stop_loss ASC, orders.buy DESC, orders."limit" ASC
-        ''', (user_id,))
-
-    return cursor.fetchall()
-
-
-def get_ownable_orders(user_id, ownable_id):
-    connect()
-
-    cursor.execute('''
-        SELECT 
-            CASE 
-                WHEN ownership.user_id = ? THEN 'X'
-                ELSE NULL
-            END,
-            CASE 
-                WHEN orders.buy THEN 'Buy'
-                ELSE 'Sell'
-            END,
-            ownables.name, 
-            orders.ordered_amount - orders.executed_amount, 
-            orders."limit", 
-            datetime(orders.expiry_dt, 'localtime'),
-            orders.rowid
-        FROM orders, ownables, ownership
-        WHERE ownership.ownable_id = ?
-        AND ownership.ownable_id = ownables.rowid
-        AND orders.ownership_id = ownership.rowid
-        AND (orders.stop_loss IS NULL OR NOT orders.stop_loss)
-        ORDER BY ownables.name ASC, orders.stop_loss ASC, orders.buy DESC, orders."limit" ASC
-        ''', (user_id, ownable_id,))
-
-    return cursor.fetchall()
-
-
-def sell_ordered_amount(user_id, ownable_id):
-    connect()
-
-    cursor.execute('''
-                SELECT COALESCE(SUM(orders.ordered_amount - orders.executed_amount),0)
-                FROM orders, ownership
-                WHERE ownership.rowid = orders.ownership_id
-                AND ownership.user_id = ?
-                AND ownership.ownable_id = ?
-                AND NOT orders.buy
-                ''', (user_id, ownable_id))
-
-    return cursor.fetchone()[0]
-
-
-def available_amount(user_id, ownable_id):
-    connect()
-
-    cursor.execute('''
-                SELECT amount
-                FROM ownership
-                WHERE user_id = ?
-                AND ownable_id = ?
-                ''', (user_id, ownable_id))
-
-    return cursor.fetchone()[0] - sell_ordered_amount(user_id, ownable_id)
-
-
-def user_has_at_least_available(amount, user_id, ownable_id):
-    connect()
-
-    if not isinstance(amount, float) and not isinstance(amount, int):
-        # comparison of float with strings does not work so well in sql
-        raise AssertionError()
-
-    cursor.execute('''
-                SELECT rowid
-                FROM ownership
-                WHERE user_id = ?
-                AND ownable_id = ?
-                AND amount - ? >= ?
-                ''', (user_id, ownable_id, sell_ordered_amount(user_id, ownable_id), amount))
-
-    if cursor.fetchone():
-        return True
-    else:
-        return False
-
-
-def news():
-    connect()
-
-    cursor.execute('''
-        SELECT dt, title FROM
-            (SELECT *, rowid 
-            FROM news
-            ORDER BY rowid DESC -- equivalent to order by dt
-            LIMIT 20) n
-        ORDER BY rowid ASC -- equivalent to order by dt
-        ''')
-
-    return cursor.fetchall()
-
-
-def ownable_name_exists(name):
-    connect()
-
-    cursor.execute('''
-                SELECT rowid
-                FROM ownables
-                WHERE name = ?
-                ''', (name,))
-
-    if cursor.fetchone():
-        return True
-    else:
-        return False
-
-
-def new_stock(expiry, name=None):
-    connect()
-
-    while name is None:
-        name = random_chars(6)
-        if ownable_name_exists(name):
-            name = None
-
-    cursor.execute('''
-        INSERT INTO ownables(name)
-        VALUES (?)
-        ''', (name,))
-
-    new_news('A new stock can now be bought: ' + name)
-    if random.getrandbits(1):
-        new_news('Experts expect the price of ' + name + ' to fall')
-    else:
-        new_news('Experts expect the price of ' + name + ' to rise')
-
-    amount = random.randrange(100, 10000)
-    price = random.randrange(10000, 20000) / amount
-    ownable_id = ownable_id_by_name(name)
-    own(bank_id(), name, amount)
-    bank_order(False,
-               ownable_id,
-               price,
-               amount,
-               expiry)
-    return name
-
-
-def ownable_id_by_name(ownable_name):
-    connect()
-
-    cursor.execute('''
-        SELECT rowid
-        FROM ownables
-        WHERE name = ?
-        ''', (ownable_name,))
-
-    return cursor.fetchone()[0]
-
-
-def get_ownership_id(ownable_id, user_id):
-    connect()
-
-    cursor.execute('''
-        SELECT rowid
-        FROM ownership
-        WHERE ownable_id = ?
-        AND user_id = ?
-        ''', (ownable_id, user_id,))
-
-    return cursor.fetchone()[0]
-
-
-def currency_id():
-    connect()
-
-    cursor.execute('''
-        SELECT rowid
-        FROM ownables
-        WHERE name = ?
-        ''', (CURRENCY_NAME,))
-
-    return cursor.fetchone()[0]
-
-
-def user_money(user_id):
-    connect()
-
-    cursor.execute('''
-        SELECT amount
-        FROM ownership
-        WHERE user_id = ?
-        AND ownable_id = ?
-        ''', (user_id, currency_id()))
-
-    return cursor.fetchone()[0]
-
-
-def delete_order(order_id, new_order_status):
-    connect()
-
-    cursor.execute('''
-        INSERT INTO order_history
-        (ownership_id, buy, "limit", ordered_amount, executed_amount, expiry_dt, status, order_id)
-        SELECT 
-            ownership_id, 
-            buy, 
-            "limit", 
-            ordered_amount, 
-            executed_amount, 
-            expiry_dt, 
-            ?, 
-            rowid
-        FROM orders
-        WHERE rowid = ?
-        ''', (new_order_status, order_id,))
-
-    cursor.execute('''
-        DELETE FROM orders
-        WHERE rowid = ?
-        ''', (order_id,))
-
-
-def current_value(ownable_id):
-    connect()
-
-    if ownable_id == currency_id():
-        return 1
-
-    cursor.execute('''SELECT price 
-                      FROM transactions
-                      WHERE ownable_id = ?
-                      ORDER BY rowid DESC -- equivalent to order by dt 
-                      LIMIT 1
-        ''', (ownable_id,))
-    return cursor.fetchone()[0]
-
-
-def execute_orders(ownable_id):
-    connect()
-    orders_traded = False
-    while True:
-        # find order to execute
-        cursor.execute('''
-            -- two best orders
-            SELECT * FROM (
-                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
-                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
-                WHERE buy_order.buy AND NOT sell_order.buy
-                AND buyer.rowid = buy_order.ownership_id
-                AND seller.rowid = sell_order.ownership_id
-                AND buyer.ownable_id = ?
-                AND seller.ownable_id = ?
-                AND buy_order."limit" IS NULL
-                AND sell_order."limit" IS NULL
-                ORDER BY buy_order.rowid ASC,
-                         sell_order.rowid ASC
-                LIMIT 1)
-            UNION ALL -- best buy orders
-            SELECT * FROM (
-                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
-                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
-                WHERE buy_order.buy AND NOT sell_order.buy
-                AND buyer.rowid = buy_order.ownership_id
-                AND seller.rowid = sell_order.ownership_id
-                AND buyer.ownable_id = ?
-                AND seller.ownable_id = ?
-                AND buy_order."limit" IS NULL
-                AND sell_order."limit" IS NOT NULL
-                AND NOT sell_order.stop_loss
-                ORDER BY sell_order."limit" ASC,
-                         buy_order.rowid ASC,
-                         sell_order.rowid ASC
-                LIMIT 1)
-            UNION ALL -- best sell orders
-            SELECT * FROM (
-                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
-                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
-                WHERE buy_order.buy AND NOT sell_order.buy
-                AND buyer.rowid = buy_order.ownership_id
-                AND seller.rowid = sell_order.ownership_id
-                AND buyer.ownable_id = ?
-                AND seller.ownable_id = ?
-                AND buy_order."limit" IS NOT NULL
-                AND NOT buy_order.stop_loss
-                AND sell_order."limit" IS NULL
-                ORDER BY buy_order."limit" DESC,
-                         buy_order.rowid ASC,
-                         sell_order.rowid ASC
-                LIMIT 1)
-            UNION ALL -- both limit orders
-            SELECT * FROM (
-                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
-                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
-                WHERE buy_order.buy AND NOT sell_order.buy
-                AND buyer.rowid = buy_order.ownership_id
-                AND seller.rowid = sell_order.ownership_id
-                AND buyer.ownable_id = ?
-                AND seller.ownable_id = ?
-                AND buy_order."limit" IS NOT NULL
-                AND sell_order."limit" IS NOT NULL
-                AND sell_order."limit" <= buy_order."limit"
-                AND NOT sell_order.stop_loss
-                AND NOT buy_order.stop_loss
-                ORDER BY buy_order."limit" DESC,
-                         sell_order."limit" ASC,
-                         buy_order.rowid ASC,
-                         sell_order.rowid ASC
-                LIMIT 1)
-            LIMIT 1
-            ''', tuple(ownable_id for _ in range(8)))
-
-        matching_orders = cursor.fetchone()
-        # return type: (ownership_id,buy,limit,stop_loss,ordered_amount,executed_amount,expiry_dt,
-        #               ownership_id,buy,limit,stop_loss,ordered_amount,executed_amount,expiry_dt,
-        #               user_id,user_id,rowid,rowid)
-
-        if not matching_orders:
-            if not orders_traded:
-                break
-            # check if the trading bot has any new offers to make
-            new_order_was_placed = trading_bot.notify_order_traded(ownable_id)
-            if new_order_was_placed:
-                orders_traded = False
-                continue
-            else:
-                break
-
-        buy_ownership_id, _, buy_limit, _, buy_order_amount, buy_executed_amount, buy_expiry_dt, \
-            sell_ownership_id, _, sell_limit, _, sell_order_amount, sell_executed_amount, sell_expiry_dt, \
-            buyer_id, seller_id, buy_order_id, sell_order_id \
-            = matching_orders
-
-        if buy_limit is None and sell_limit is None:
-            price = current_value(ownable_id)
-        elif buy_limit is None:
-            price = sell_limit
-        elif sell_limit is None:
-            price = buy_limit
-        else:  # both not NULL
-            # the price of the older order is used, just like in the real exchange
-            if buy_order_id < sell_order_id:
-                price = buy_limit
-            else:
-                price = sell_limit
-
-        buyer_money = user_money(buyer_id)
-
-        def _my_division(x, y):
-            try:
-                return floor(x / y)
-            except ZeroDivisionError:
-                return float('Inf')
-
-        amount = min(buy_order_amount - buy_executed_amount,
-                     sell_order_amount - sell_executed_amount,
-                     _my_division(buyer_money, price))
-
-        if amount == 0:  # probable because buyer has not enough money
-            delete_order(buy_order_id, 'Unable to pay')
-            continue
-
-        buy_order_finished = (buy_order_amount - buy_executed_amount - amount <= 0) or (
-                buyer_money - amount * price < price)
-        sell_order_finished = (sell_order_amount - sell_executed_amount - amount <= 0)
-
-        if price < 0 or amount <= 0:  # price of 0 is possible though unlikely
-            return AssertionError()
-
-        # actually execute the order, but the bank does not send or receive anything
-        send_ownable(buyer_id, seller_id, currency_id(), price * amount)
-        send_ownable(seller_id, buyer_id, ownable_id, amount)
-
-        # update order execution state
-        cursor.execute('''
-            UPDATE orders 
-            SET executed_amount = executed_amount + ?
-            WHERE rowid = ?
-            OR rowid = ?
-            ''', (amount, buy_order_id, sell_order_id))
-
-        if buy_order_finished:
-            delete_order(buy_order_id, 'Executed')
-            orders_traded = True
-        if sell_order_finished:
-            delete_order(sell_order_id, 'Executed')
-            orders_traded = True
-
-        if seller_id != buyer_id:  # prevent showing self-transactions
-            cursor.execute('''
-                INSERT INTO transactions
-                (price, ownable_id, amount, buyer_id, seller_id)
-                VALUES(?, ?, ?, ?, ?)
-                ''', (price, ownable_id, amount, buyer_id, seller_id))
-
-        # trigger stop-loss orders
-        if buyer_id != seller_id:
-            cursor.execute('''
-                UPDATE orders
-                SET stop_loss = NULL,
-                    "limit" = NULL
-                WHERE stop_loss IS NOT NULL
-                AND stop_loss
-                AND ? IN (SELECT ownable_id FROM ownership WHERE rowid = ownership_id)
-                AND ((buy AND "limit" < ?) OR (NOT buy AND "limit" > ?))
-                ''', (ownable_id, price, price,))
-
-
-def ownable_id_by_ownership_id(ownership_id):
-    connect()
-
-    cursor.execute('''
-        SELECT ownable_id
-        FROM ownership
-        WHERE rowid = ?
-        ''', (ownership_id,))
-
-    return cursor.fetchone()[0]
-
-
-def ownable_name_by_id(ownable_id):
-    connect()
-
-    cursor.execute('''
-        SELECT name
-        FROM ownables
-        WHERE rowid = ?
-        ''', (ownable_id,))
-
-    return cursor.fetchone()[0]
-
-
-def bank_order(buy, ownable_id, limit, amount, expiry):
-    if not limit:
-        raise AssertionError('The bank does not give away anything.')
-    place_order(buy,
-                get_ownership_id(ownable_id, bank_id()),
-                limit,
-                False,
-                amount,
-                expiry)
-    ownable_name = ownable_name_by_id(ownable_id)
-    new_news('External investors are selling ' + ownable_name + ' atm')
-
-
-def current_db_time():  # might differ from datetime.datetime.now() for time zone reasons
-    connect()
-
-    cursor.execute('''
-        SELECT datetime('now')
-        ''')
-
-    return cursor.fetchone()[0]
-
-
-def place_order(buy, ownership_id, limit, stop_loss, amount, expiry):
-    connect()
-    cursor.execute('''
-                INSERT INTO orders 
-                (buy, ownership_id, "limit", stop_loss, ordered_amount, expiry_dt)
-                VALUES (?, ?, ?, ?, ?, ?)
-                ''', (buy, ownership_id, limit, stop_loss, amount, expiry))
-
-    execute_orders(ownable_id_by_ownership_id(ownership_id))
-    return True
-
-
-def trades_on(ownable_id, limit):
-    connect()
-
-    cursor.execute('''
-        SELECT datetime(dt,'localtime'), amount, price
-        FROM transactions
-        WHERE ownable_id = ?
-        ORDER BY rowid DESC -- equivalent to order by dt
-        LIMIT ?
-        ''', (ownable_id, limit,))
-
-    return cursor.fetchall()
-
-
-def trades(user_id, limit):
-    connect()
-    cursor.execute('''
-        SELECT 
-            (CASE WHEN seller_id = ? THEN 'Sell' ELSE 'Buy' END), 
-            (SELECT name FROM ownables WHERE rowid = transactions.ownable_id), 
-            amount, 
-            price,
-            datetime(dt,'localtime')
-        FROM transactions
-        WHERE seller_id = ? OR buyer_id = ?
-        ORDER BY rowid DESC -- equivalent to order by dt
-        LIMIT ?
-        ''', (user_id, user_id, user_id, limit,))
-
-    return cursor.fetchall()
-
-
-def drop_expired_orders():
-    connect()
-
-    cursor.execute('''
-        SELECT rowid, ownership_id, * FROM orders 
-        WHERE expiry_dt < DATETIME('now')
-        ''')
-
-    data = cursor.fetchall()
-    for order in data:
-        order_id = order[0]
-        delete_order(order_id, 'Expired')
-
-    return data
-
-
-def generate_keys(count=1):
-    # source https://stackoverflow.com/questions/17049308/python-3-3-serial-key-generator-list-problems
-
-    for i in range(count):
-        key = '-'.join(random_chars(5) for _ in range(5))
-        save_key(key)
-        print(key)
-
-
-def user_has_order_with_id(session_id, order_id):
-    connect()
-
-    cursor.execute('''
-                SELECT orders.rowid
-                FROM orders, ownership, sessions
-                WHERE orders.rowid = ?
-                AND sessions.session_id = ?
-                AND sessions.user_id = ownership.user_id
-                AND ownership.rowid = orders.ownership_id
-                ''', (order_id, session_id,))
-
-    if cursor.fetchone():
-        return True
-    else:
-        return False
-
-
-def leaderboard():
-    connect()
-
-    cursor.execute('''
-        SELECT * 
-        FROM ( -- one score for each user
-            SELECT 
-                username, 
-                SUM(CASE -- sum score for each of the users ownables
-                    WHEN ownership.ownable_id = ? THEN ownership.amount
-                    ELSE ownership.amount * (SELECT price 
-                                             FROM transactions
-                                             WHERE ownable_id = ownership.ownable_id 
-                                             ORDER BY rowid DESC -- equivalent to ordering by dt
-                                             LIMIT 1)
-                    END
-                ) score
-            FROM users, ownership
-            WHERE ownership.user_id = users.rowid
-            AND users.username != 'bank'
-            GROUP BY users.rowid
-        ) AS scores
-        ORDER BY score DESC
-        LIMIT 50
-        ''', (currency_id(),))
-
-    return cursor.fetchall()
-
-
-def user_wealth(user_id):
-    connect()
-
-    cursor.execute('''
-        SELECT SUM(
-            CASE -- sum score for each of the users ownables
-            WHEN ownership.ownable_id = ? THEN ownership.amount
-            ELSE ownership.amount * (SELECT price 
-                                     FROM transactions
-                                     WHERE ownable_id = ownership.ownable_id 
-                                     ORDER BY rowid DESC -- equivalent to ordering by dt
-                                     LIMIT 1)
-            END
-        ) score
-        FROM ownership
-        WHERE ownership.user_id = ?
-        ''', (currency_id(), user_id,))
-
-    return cursor.fetchone()[0]
-
-
-def change_password(session_id, password):
-    connect()
-
-    cursor.execute('''
-                UPDATE users
-                SET password = ?
-                WHERE rowid = (SELECT user_id FROM sessions WHERE sessions.session_id = ?)
-                ''', (password, session_id,))
-
-
-def sign_out_user(session_id):
-    connect()
-
-    cursor.execute('''
-        DELETE FROM sessions
-        WHERE user_id = (SELECT user_id FROM sessions s2 WHERE s2.session_id = ?)
-        ''', (session_id,))
-
-
-def delete_user(user_id):
-    connect()
-
-    cursor.execute('''
-        DELETE FROM sessions
-        WHERE user_id = ?
-        ''', (user_id,))
-
-    cursor.execute('''
-        DELETE FROM orders
-        WHERE ownership_id IN (
-            SELECT rowid FROM ownership WHERE user_id = ?)
-        ''', (user_id,))
-
-    cursor.execute('''
-        DELETE FROM ownership
-        WHERE user_id = ?
-        ''', (user_id,))
-
-    cursor.execute('''
-        DELETE FROM keys
-        WHERE used_by_user_id = ?
-        ''', (user_id,))
-
-    cursor.execute('''
-        INSERT INTO news(title)
-        VALUES ((SELECT username FROM users WHERE rowid = ?) || ' retired.')
-        ''', (user_id,))
-
-    cursor.execute('''
-        DELETE FROM users
-        WHERE rowid = ?
-        ''', (user_id,))
-
-
-def delete_ownable(ownable_id):
-    connect()
-
-    cursor.execute('''
-        DELETE FROM transactions
-        WHERE ownable_id = ?
-        ''', (ownable_id,))
-
-    cursor.execute('''
-        DELETE FROM orders
-        WHERE ownership_id IN (
-            SELECT rowid FROM ownership WHERE ownable_id = ?)
-        ''', (ownable_id,))
-
-    cursor.execute('''
-        DELETE FROM orders_history
-        WHERE ownership_id IN (
-            SELECT rowid FROM ownership WHERE ownable_id = ?)
-        ''', (ownable_id,))
-
-    # only delete empty ownerships
-    cursor.execute('''
-        DELETE FROM ownership
-        WHERE ownable_id = ?
-        AND amount = 0
-        ''', (ownable_id,))
-
-    cursor.execute('''
-        INSERT INTO news(title)
-        VALUES ((SELECT name FROM ownables WHERE rowid = ?) || ' can not be traded any more.')
-        ''', (ownable_id,))
-
-    cursor.execute('''
-        DELETE FROM ownables
-        WHERE rowid = ?
-        ''', (ownable_id,))
-
-
-def hash_all_users_passwords():
-    connect()
-
-    cursor.execute('''
-        SELECT rowid, password 
-        FROM users
-        ''')
-
-    users = cursor.fetchall()
-
-    for user in users:
-        user_id = user[0]
-        pw = user[1]
-        valid_hash = True
-        try:
-            sha256_crypt.verify('password' + salt, pw)
-        except ValueError:
-            valid_hash = False
-        if valid_hash:
-            raise AssertionError('There is already a hashed password in the database! Be careful what you are doing!')
-        pw = sha256_crypt.encrypt(pw + salt)
-        cursor.execute('''
-            UPDATE users
-            SET password = ?
-            WHERE rowid = ?
-            ''', (pw, user_id,))
-
-
-def new_news(message):
-    connect()
-    cursor.execute('''
-        INSERT INTO news(title)
-        VALUES (?)
-        ''', (message,))
-
-
-def abs_spread(ownable_id):
-    connect()
-
-    cursor.execute('''
-        SELECT 
-            (SELECT MAX("limit") 
-             FROM orders, ownership
-             WHERE ownership.rowid = orders.ownership_id
-             AND ownership.ownable_id = ?
-             AND buy
-             AND NOT stop_loss) AS bid, 
-            (SELECT MIN("limit") 
-             FROM orders, ownership
-             WHERE ownership.rowid = orders.ownership_id
-             AND ownership.ownable_id = ?
-             AND NOT buy
-             AND NOT stop_loss) AS ask
-        ''', (ownable_id, ownable_id,))
-
-    return cursor.fetchone()
-
-
-def ownables():
-    connect()
-
-    cursor.execute('''
-        SELECT name, course,
-            (SELECT SUM(amount)
-            FROM ownership
-            WHERE ownership.ownable_id = ownables_with_course.rowid) market_size
-        FROM (SELECT
-                name, ownables.rowid,
-                CASE WHEN ownables.rowid = ? 
-                THEN 1
-                ELSE (SELECT price
-                      FROM transactions
-                      WHERE ownable_id = ownables.rowid
-                      ORDER BY rowid DESC -- equivalent to ordering by dt
-                      LIMIT 1) END course
-             FROM ownables) ownables_with_course
-        ''', (currency_id(),))
-
-    data = cursor.fetchall()
-
-    for idx in range(len(data)):
-        # compute market cap
-        row = data[idx]
-        if row[1] is None:
-            market_cap = None
-        elif row[2] is None:
-            market_cap = None
-        else:
-            market_cap = row[1] * row[2]
-        data[idx] = (row[0], row[1], market_cap)
-
-    return data
-
-
-def reset_bank():
-    connect()
-    cursor.execute('''
-        DELETE FROM ownership 
-        WHERE user_id = ?
-        ''', (bank_id(),))
-
-
-def cleanup():
-    if connection is not None:
-        connection.commit()
-        connection.close()
-
-
-def ownable_ids():
-    connect()
-
-    cursor.execute('''
-        SELECT rowid FROM ownables
-        ''')
-
-    return [ownable_id[0] for ownable_id in cursor.fetchall()]
-
-
-def get_old_orders(user_id, include_executed, include_canceled, limit):
-    connect()
-    cursor.execute('''
-        SELECT 
-            (CASE WHEN order_history.buy THEN 'Buy' ELSE 'Sell' END),
-            ownables.name,
-            (order_history.ordered_amount - order_history.executed_amount) || '/' || order_history.ordered_amount,
-            order_history."limit",
-            order_history.expiry_dt,
-            order_history.order_id,
-            order_history.status
-        FROM order_history, ownership, ownables
-        WHERE ownership.user_id = ?
-        AND ownership.rowid = order_history.ownership_id
-        AND ownables.rowid = ownership.ownable_id
-        AND (
-             (order_history.status = 'Executed' AND ?)
-             OR 
-             ((order_history.status = 'Expired' OR order_history.status = 'Canceled') AND ?)
-            )
-        ORDER BY order_history.rowid DESC -- equivalent to ordering by creation time
-        LIMIT ?
-        ''', (user_id, include_executed, include_canceled, limit))
-
-    return cursor.fetchall()
+import json
+import os
+import random
+import re
+import sqlite3 as db
+import uuid
+from logging import INFO
+from math import floor
+from shutil import copyfile
+from typing import Optional, Dict
+
+from passlib.handlers.sha2_crypt import sha256_crypt
+
+import db_setup
+import trading_bot
+from game import CURRENCY_NAME, logger, DB_NAME
+from util import random_chars
+
+DBName = str
+connections: Dict[DBName, db.Connection] = {}
+current_connection: Optional[db.Connection] = None
+current_cursor: Optional[db.Cursor] = None
+current_db_name: Optional[DBName] = None
+current_user_id: Optional[int] = None
+
+
+def execute(sql, parameters=()):
+    if not re.search(r"(?i)\s*SELECT", sql):
+        logger.info(sql, 'sql_query', data=json.dumps(parameters))
+    return current_cursor.execute(sql, parameters)
+
+
+def valid_db_name(name):
+    return re.match(r"[a-z0-9.-]{0,20}", name)
+
+
+def query_save_name():
+    while True:
+        # save_name = input('Name of the database (You can also enter a new filename here): ')
+        save_name = DB_NAME
+        if valid_db_name(save_name):
+            return save_name
+        else:
+            print('Must match "[a-z0-9.-]{0,20}"')
+
+
+def connect(db_name=None, create_if_not_exists=False):
+    """
+    connects to the database with the given name, if it exists
+    if the database does not exist an exception is raised
+        (unless create_if_not_exists is true, then the database is created)
+    if there is already a connection to this database, that connection is used
+    :return: the connection and the connections' cursor
+    """
+    if db_name is None:
+        db_name = query_save_name()
+    if not db_name.endswith('.db'):
+        db_name += '.db'
+    db_name = db_name.lower()
+    if not os.path.isfile(db_name) and not create_if_not_exists:
+        raise FileNotFoundError('There is no database with this name.')
+    creating_new_db = not os.path.isfile(db_name)
+    if db_name not in connections:
+        try:
+            db_connection = db.connect(db_name, check_same_thread=False)
+            db_setup.create_functions(db_connection)
+            db_setup.set_pragmas(db_connection.cursor())
+            # connection.text_factory = lambda x: x.encode('latin-1')
+        except db.Error as e:
+            print("Database error %s:" % e.args[0])
+            raise
+        connections[db_name] = db_connection
+    global current_connection
+    global current_db_name
+    global current_cursor
+    current_connection = connections[db_name]
+    current_cursor = connections[db_name].cursor()
+    current_db_name = db_name
+    if creating_new_db:
+        try:
+            if os.path.isfile('/test-db/' + db_name):
+                print('Using test database containing fake data')
+                copyfile('/test-db/' + db_name, db_name)
+            else:
+                logger.log('Creating database', INFO, 'database_creation')
+                logger.commit()
+                setup()
+        except Exception:
+            if current_connection is not None:
+                current_connection.rollback()
+            if db_name in connections:
+                disconnect(db_name, rollback=True)
+            os.remove(db_name)
+            current_connection = None
+            current_cursor = None
+            current_db_name = None
+            raise
+
+
+def disconnect(connection_name, rollback=True):
+    global connections
+    if connection_name not in connections:
+        raise ValueError('Invalid connection')
+    if rollback:
+        connections[connection_name].rollback()
+    else:
+        connections[connection_name].commit()
+    connections[connection_name].close()
+    del connections[connection_name]
+
+
+def setup():
+    db_setup.setup(current_cursor)
+
+
+def used_key_count():
+    connect()
+
+    execute('''
+        SELECT COUNT(*) -- rarely executed, no index needed, O(n) query
+        FROM keys
+        WHERE used_by_user_id IS NOT NULL
+        ''')
+
+    return current_cursor.fetchone()[0]
+
+
+def login(username, password):
+    execute('''
+                SELECT rowid, password, salt
+                FROM users
+                WHERE username = ?
+                ''', (username,))
+    data = current_cursor.fetchone()
+    if not data:
+        return None
+    user_id, hashed_password, salt = data
+    # if a ValueError occurs here, then most likely a password that was stored as plain text
+    if sha256_crypt.verify(password + salt, hashed_password):
+        return new_session(user_id)
+    else:
+        return None
+
+
+def register(username, password, game_key):
+    salt = str(uuid.uuid4())
+    hashed_password = sha256_crypt.using(rounds=100000).encrypt(str(password) + salt)
+    connect()
+    if username == '':
+        return False
+    if password == '':
+        return False
+    execute('''
+                INSERT INTO users 
+                (username, password, salt)
+                VALUES (? , ?, ?)
+                ''', (username, hashed_password, salt))
+    if game_key != '':
+        if valid_key(game_key):
+            activate_key(game_key, get_user_id_by_name(username))
+    return True
+
+
+def own(user_id, ownable_name, amount=0):
+    if not isinstance(ownable_name, str):
+        return AssertionError('A name must be a string.')
+
+    execute('''
+                INSERT OR IGNORE INTO ownership (user_id, ownable_id, amount)
+                SELECT ?, (SELECT rowid FROM ownables WHERE name = ?), ?
+                ''', (user_id, ownable_name, amount))
+
+
+def send_ownable(from_user_id, to_user_id, ownable_id, amount):
+    connect()
+
+    if amount < 0:
+        raise AssertionError('Can not send negative amount')
+
+    execute('''
+                UPDATE ownership
+                SET amount = amount - ?
+                WHERE user_id = ?
+                AND ownable_id = ?
+                ''', (amount, from_user_id, ownable_id,))
+
+    own(to_user_id, ownable_name_by_id(ownable_id))
+
+    execute('''
+                UPDATE ownership
+                SET amount = amount + ?
+                WHERE user_id = ?
+                AND ownable_id = ?
+                ''', (amount, to_user_id, ownable_id,))
+    return True
+
+
+def valid_key(key):
+    connect()
+
+    execute('''
+                SELECT key
+                FROM keys
+                WHERE used_by_user_id IS NULL
+                AND key = ?
+                ''', (key,))
+
+    if current_cursor.fetchone():
+        return True
+    else:
+        return False
+
+
+def new_session(user_id):
+    connect()
+
+    session_id = str(uuid.uuid4())
+
+    execute('''
+                INSERT INTO SESSIONS 
+                (user_id, session_id)
+                VALUES (? , ?)
+                ''', (user_id, session_id))
+
+    return session_id
+
+
+def save_key(key):
+    connect()
+
+    execute('''
+                INSERT INTO keys 
+                (key)
+                VALUES (?)
+                ''', (key,))
+
+
+def drop_old_sessions():
+    connect()
+
+    execute(''' -- no need to optimize this very well
+                DELETE FROM sessions
+                WHERE 
+                    (SELECT COUNT(*) as newer
+                     FROM sessions s2
+                     WHERE user_id = s2.user_id
+                     AND rowid < s2.rowid) >= 10
+                ''')
+
+
+def user_exists(username):
+    connect()
+
+    execute('''
+                SELECT rowid
+                FROM users
+                WHERE username = ?
+                ''', (username,))
+
+    if current_cursor.fetchone():
+        return True
+    else:
+        return False
+
+
+def get_user_id_by_session_id(session_id):
+    connect()
+
+    execute('''
+        SELECT users.rowid
+        FROM sessions, users
+        WHERE sessions.session_id = ?
+        AND users.rowid = sessions.user_id
+        ''', (session_id,))
+
+    ids = current_cursor.fetchone()
+    if not ids:
+        return False
+    return ids[0]
+
+
+def get_user_id_by_name(username):
+    connect()
+
+    execute('''
+        SELECT users.rowid
+        FROM users
+        WHERE username = ?
+        ''', (username,))
+
+    return current_cursor.fetchone()[0]
+
+
+def get_user_ownership(user_id):
+    connect()
+
+    execute('''
+        SELECT 
+            ownables.name, 
+            ownership.amount, 
+            COALESCE (
+            CASE -- sum score for each of the users ownables
+            WHEN ownership.ownable_id = ? THEN 1
+            ELSE (SELECT price 
+                  FROM transactions
+                  WHERE ownable_id = ownership.ownable_id 
+                  ORDER BY rowid DESC -- equivalent to ordering by dt
+                  LIMIT 1)
+            END, 0) AS price, 
+            (SELECT MAX("limit") 
+             FROM orders, ownership o2
+             WHERE o2.rowid = orders.ownership_id
+             AND o2.ownable_id = ownership.ownable_id
+             AND buy
+             AND NOT stop_loss) AS bid, 
+            (SELECT MIN("limit") 
+             FROM orders, ownership o2
+             WHERE o2.rowid = orders.ownership_id
+             AND o2.ownable_id = ownership.ownable_id
+             AND NOT buy
+             AND NOT stop_loss) AS ask
+        FROM ownership, ownables
+        WHERE user_id = ?
+        AND (ownership.amount > 0 OR ownership.ownable_id = ?)
+        AND ownership.ownable_id = ownables.rowid
+        ORDER BY ownables.rowid ASC
+        ''', (currency_id(), user_id, currency_id(),))
+
+    return current_cursor.fetchall()
+
+
+def activate_key(key, user_id):
+    connect()
+    execute('''
+                UPDATE keys
+                SET used_by_user_id = ?
+                WHERE used_by_user_id IS NULL
+                AND key = ?
+                ''', (user_id, key,))
+
+    send_ownable(bank_id(), user_id, currency_id(), 1000)
+
+
+def bank_id():
+    connect()
+
+    execute('''
+        SELECT users.rowid
+        FROM users
+        WHERE username = 'bank'
+        ''')
+
+    return current_cursor.fetchone()[0]
+
+
+def valid_session_id(session_id):
+    connect()
+
+    execute('''
+                SELECT rowid
+                FROM sessions
+                WHERE session_id = ?
+                ''', (session_id,))
+
+    if current_cursor.fetchone():
+        return True
+    else:
+        return False
+
+
+def get_user_orders(user_id):
+    connect()
+
+    execute('''
+        SELECT 
+            CASE 
+                WHEN orders.buy  THEN 'Buy'
+                ELSE 'Sell'
+            END,
+            ownables.name, 
+            (orders.ordered_amount - orders.executed_amount) || '/' || orders.ordered_amount, 
+            orders."limit", 
+            CASE 
+                WHEN orders."limit" IS NULL THEN NULL 
+                WHEN orders.stop_loss THEN 'Yes'
+                ELSE 'No'
+            END, 
+            datetime(orders.expiry_dt, 'localtime'),
+            orders.rowid
+        FROM orders, ownables, ownership
+        WHERE ownership.user_id = ?
+        AND ownership.ownable_id = ownables.rowid
+        AND orders.ownership_id = ownership.rowid
+        ORDER BY ownables.name ASC, orders.stop_loss ASC, orders.buy DESC, orders."limit" ASC
+        ''', (user_id,))
+
+    return current_cursor.fetchall()
+
+
+def get_ownable_orders(user_id, ownable_id):
+    connect()
+
+    execute('''
+        SELECT 
+            CASE 
+                WHEN ownership.user_id = ? THEN 'X'
+                ELSE NULL
+            END,
+            CASE 
+                WHEN orders.buy THEN 'Buy'
+                ELSE 'Sell'
+            END,
+            ownables.name, 
+            orders.ordered_amount - orders.executed_amount, 
+            orders."limit", 
+            datetime(orders.expiry_dt, 'localtime'),
+            orders.rowid
+        FROM orders, ownables, ownership
+        WHERE ownership.ownable_id = ?
+        AND ownership.ownable_id = ownables.rowid
+        AND orders.ownership_id = ownership.rowid
+        AND (orders.stop_loss IS NULL OR NOT orders.stop_loss)
+        ORDER BY ownables.name ASC, orders.stop_loss ASC, orders.buy DESC, orders."limit" ASC
+        ''', (user_id, ownable_id,))
+
+    return current_cursor.fetchall()
+
+
+def sell_ordered_amount(user_id, ownable_id):
+    connect()
+
+    execute('''
+                SELECT COALESCE(SUM(orders.ordered_amount - orders.executed_amount),0)
+                FROM orders, ownership
+                WHERE ownership.rowid = orders.ownership_id
+                AND ownership.user_id = ?
+                AND ownership.ownable_id = ?
+                AND NOT orders.buy
+                ''', (user_id, ownable_id))
+
+    return current_cursor.fetchone()[0]
+
+
+def available_amount(user_id, ownable_id):
+    connect()
+
+    execute('''
+                SELECT amount
+                FROM ownership
+                WHERE user_id = ?
+                AND ownable_id = ?
+                ''', (user_id, ownable_id))
+
+    return current_cursor.fetchone()[0] - sell_ordered_amount(user_id, ownable_id)
+
+
+def user_has_at_least_available(amount, user_id, ownable_id):
+    connect()
+
+    if not isinstance(amount, float) and not isinstance(amount, int):
+        # comparison of float with strings does not work so well in sql
+        raise AssertionError()
+
+    execute('''
+                SELECT rowid
+                FROM ownership
+                WHERE user_id = ?
+                AND ownable_id = ?
+                AND amount - ? >= ?
+                ''', (user_id, ownable_id, sell_ordered_amount(user_id, ownable_id), amount))
+
+    if current_cursor.fetchone():
+        return True
+    else:
+        return False
+
+
+def news():
+    connect()
+
+    execute('''
+        SELECT dt, title FROM
+            (SELECT *, rowid 
+            FROM news
+            ORDER BY rowid DESC -- equivalent to order by dt
+            LIMIT 20) n
+        ORDER BY rowid ASC -- equivalent to order by dt
+        ''')
+
+    return current_cursor.fetchall()
+
+
+def ownable_name_exists(name):
+    connect()
+
+    execute('''
+                SELECT rowid
+                FROM ownables
+                WHERE name = ?
+                ''', (name,))
+
+    if current_cursor.fetchone():
+        return True
+    else:
+        return False
+
+
+def new_stock(expiry, name=None):
+    connect()
+
+    while name is None:
+        name = random_chars(6)
+        if ownable_name_exists(name):
+            name = None
+
+    execute('''
+        INSERT INTO ownables(name)
+        VALUES (?)
+        ''', (name,))
+
+    new_news('A new stock can now be bought: ' + name)
+    if random.getrandbits(1):
+        new_news('Experts expect the price of ' + name + ' to fall')
+    else:
+        new_news('Experts expect the price of ' + name + ' to rise')
+
+    amount = random.randrange(100, 10000)
+    price = random.randrange(10000, 20000) / amount
+    ownable_id = ownable_id_by_name(name)
+    own(bank_id(), name, amount)
+    bank_order(False,
+               ownable_id,
+               price,
+               amount,
+               expiry)
+    return name
+
+
+def ownable_id_by_name(ownable_name):
+    connect()
+
+    execute('''
+        SELECT rowid
+        FROM ownables
+        WHERE name = ?
+        ''', (ownable_name,))
+
+    return current_cursor.fetchone()[0]
+
+
+def get_ownership_id(ownable_id, user_id):
+    connect()
+
+    execute('''
+        SELECT rowid
+        FROM ownership
+        WHERE ownable_id = ?
+        AND user_id = ?
+        ''', (ownable_id, user_id,))
+
+    return current_cursor.fetchone()[0]
+
+
+def currency_id():
+    connect()
+
+    execute('''
+        SELECT rowid
+        FROM ownables
+        WHERE name = ?
+        ''', (CURRENCY_NAME,))
+
+    return current_cursor.fetchone()[0]
+
+
+def user_money(user_id):
+    connect()
+
+    execute('''
+        SELECT amount
+        FROM ownership
+        WHERE user_id = ?
+        AND ownable_id = ?
+        ''', (user_id, currency_id()))
+
+    return current_cursor.fetchone()[0]
+
+
+def delete_order(order_id, new_order_status):
+    connect()
+
+    execute('''
+        INSERT INTO order_history
+        (ownership_id, buy, "limit", ordered_amount, executed_amount, expiry_dt, status, order_id)
+        SELECT 
+            ownership_id, 
+            buy, 
+            "limit", 
+            ordered_amount, 
+            executed_amount, 
+            expiry_dt, 
+            ?, 
+            rowid
+        FROM orders
+        WHERE rowid = ?
+        ''', (new_order_status, order_id,))
+
+    execute('''
+        DELETE FROM orders
+        WHERE rowid = ?
+        ''', (order_id,))
+
+
+def current_value(ownable_id):
+    connect()
+
+    if ownable_id == currency_id():
+        return 1
+
+    execute('''SELECT price 
+                      FROM transactions
+                      WHERE ownable_id = ?
+                      ORDER BY rowid DESC -- equivalent to order by dt 
+                      LIMIT 1
+        ''', (ownable_id,))
+    return current_cursor.fetchone()[0]
+
+
+def execute_orders(ownable_id):
+    connect()
+    orders_traded = False
+    while True:
+        # find order to execute
+        execute('''
+            -- two best orders
+            SELECT * FROM (
+                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
+                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
+                WHERE buy_order.buy AND NOT sell_order.buy
+                AND buyer.rowid = buy_order.ownership_id
+                AND seller.rowid = sell_order.ownership_id
+                AND buyer.ownable_id = ?
+                AND seller.ownable_id = ?
+                AND buy_order."limit" IS NULL
+                AND sell_order."limit" IS NULL
+                ORDER BY buy_order.rowid ASC,
+                         sell_order.rowid ASC
+                LIMIT 1)
+            UNION ALL -- best buy orders
+            SELECT * FROM (
+                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
+                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
+                WHERE buy_order.buy AND NOT sell_order.buy
+                AND buyer.rowid = buy_order.ownership_id
+                AND seller.rowid = sell_order.ownership_id
+                AND buyer.ownable_id = ?
+                AND seller.ownable_id = ?
+                AND buy_order."limit" IS NULL
+                AND sell_order."limit" IS NOT NULL
+                AND NOT sell_order.stop_loss
+                ORDER BY sell_order."limit" ASC,
+                         buy_order.rowid ASC,
+                         sell_order.rowid ASC
+                LIMIT 1)
+            UNION ALL -- best sell orders
+            SELECT * FROM (
+                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
+                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
+                WHERE buy_order.buy AND NOT sell_order.buy
+                AND buyer.rowid = buy_order.ownership_id
+                AND seller.rowid = sell_order.ownership_id
+                AND buyer.ownable_id = ?
+                AND seller.ownable_id = ?
+                AND buy_order."limit" IS NOT NULL
+                AND NOT buy_order.stop_loss
+                AND sell_order."limit" IS NULL
+                ORDER BY buy_order."limit" DESC,
+                         buy_order.rowid ASC,
+                         sell_order.rowid ASC
+                LIMIT 1)
+            UNION ALL -- both limit orders
+            SELECT * FROM (
+                SELECT buy_order.*, sell_order.*, buyer.user_id, seller.user_id, buy_order.rowid, sell_order.rowid
+                FROM orders buy_order, orders sell_order, ownership buyer, ownership seller
+                WHERE buy_order.buy AND NOT sell_order.buy
+                AND buyer.rowid = buy_order.ownership_id
+                AND seller.rowid = sell_order.ownership_id
+                AND buyer.ownable_id = ?
+                AND seller.ownable_id = ?
+                AND buy_order."limit" IS NOT NULL
+                AND sell_order."limit" IS NOT NULL
+                AND sell_order."limit" <= buy_order."limit"
+                AND NOT sell_order.stop_loss
+                AND NOT buy_order.stop_loss
+                ORDER BY buy_order."limit" DESC,
+                         sell_order."limit" ASC,
+                         buy_order.rowid ASC,
+                         sell_order.rowid ASC
+                LIMIT 1)
+            LIMIT 1
+            ''', tuple(ownable_id for _ in range(8)))
+
+        matching_orders = current_cursor.fetchone()
+        # return type: (ownership_id,buy,limit,stop_loss,ordered_amount,executed_amount,expiry_dt,
+        #               ownership_id,buy,limit,stop_loss,ordered_amount,executed_amount,expiry_dt,
+        #               user_id,user_id,rowid,rowid)
+
+        if not matching_orders:
+            if not orders_traded:
+                break
+            # check if the trading bot has any new offers to make
+            new_order_was_placed = trading_bot.notify_order_traded(ownable_id)
+            if new_order_was_placed:
+                orders_traded = False
+                continue
+            else:
+                break
+
+        buy_ownership_id, _, buy_limit, _, buy_order_amount, buy_executed_amount, buy_expiry_dt, \
+        sell_ownership_id, _, sell_limit, _, sell_order_amount, sell_executed_amount, sell_expiry_dt, \
+        buyer_id, seller_id, buy_order_id, sell_order_id \
+            = matching_orders
+
+        if buy_limit is None and sell_limit is None:
+            price = current_value(ownable_id)
+        elif buy_limit is None:
+            price = sell_limit
+        elif sell_limit is None:
+            price = buy_limit
+        else:  # both not NULL
+            # the price of the older order is used, just like in the real exchange
+            if buy_order_id < sell_order_id:
+                price = buy_limit
+            else:
+                price = sell_limit
+
+        buyer_money = user_money(buyer_id)
+
+        def _my_division(x, y):
+            try:
+                return floor(x / y)
+            except ZeroDivisionError:
+                return float('Inf')
+
+        amount = min(buy_order_amount - buy_executed_amount,
+                     sell_order_amount - sell_executed_amount,
+                     _my_division(buyer_money, price))
+
+        if amount == 0:  # probable because buyer has not enough money
+            delete_order(buy_order_id, 'Unable to pay')
+            continue
+
+        buy_order_finished = (buy_order_amount - buy_executed_amount - amount <= 0) or (
+                buyer_money - amount * price < price)
+        sell_order_finished = (sell_order_amount - sell_executed_amount - amount <= 0)
+
+        if price < 0 or amount <= 0:  # price of 0 is possible though unlikely
+            return AssertionError()
+
+        # actually execute the order, but the bank does not send or receive anything
+        send_ownable(buyer_id, seller_id, currency_id(), price * amount)
+        send_ownable(seller_id, buyer_id, ownable_id, amount)
+
+        # update order execution state
+        execute('''
+            UPDATE orders 
+            SET executed_amount = executed_amount + ?
+            WHERE rowid = ?
+            OR rowid = ?
+            ''', (amount, buy_order_id, sell_order_id))
+
+        if buy_order_finished:
+            delete_order(buy_order_id, 'Executed')
+            orders_traded = True
+        if sell_order_finished:
+            delete_order(sell_order_id, 'Executed')
+            orders_traded = True
+
+        if seller_id != buyer_id:  # prevent showing self-transactions
+            execute('''
+                INSERT INTO transactions
+                (price, ownable_id, amount, buyer_id, seller_id)
+                VALUES(?, ?, ?, ?, ?)
+                ''', (price, ownable_id, amount, buyer_id, seller_id))
+
+        # trigger stop-loss orders
+        if buyer_id != seller_id:
+            execute('''
+                UPDATE orders
+                SET stop_loss = NULL,
+                    "limit" = NULL
+                WHERE stop_loss IS NOT NULL
+                AND stop_loss
+                AND ? IN (SELECT ownable_id FROM ownership WHERE rowid = ownership_id)
+                AND ((buy AND "limit" < ?) OR (NOT buy AND "limit" > ?))
+                ''', (ownable_id, price, price,))
+
+
+def ownable_id_by_ownership_id(ownership_id):
+    connect()
+
+    execute('''
+        SELECT ownable_id
+        FROM ownership
+        WHERE rowid = ?
+        ''', (ownership_id,))
+
+    return current_cursor.fetchone()[0]
+
+
+def ownable_name_by_id(ownable_id):
+    connect()
+
+    execute('''
+        SELECT name
+        FROM ownables
+        WHERE rowid = ?
+        ''', (ownable_id,))
+
+    return current_cursor.fetchone()[0]
+
+
+def bank_order(buy, ownable_id, limit, amount, expiry):
+    if not limit:
+        raise AssertionError('The bank does not give away anything.')
+    place_order(buy,
+                get_ownership_id(ownable_id, bank_id()),
+                limit,
+                False,
+                amount,
+                expiry)
+    ownable_name = ownable_name_by_id(ownable_id)
+    new_news('External investors are selling ' + ownable_name + ' atm')
+
+
+def current_db_time():  # might differ from datetime.datetime.now() for time zone reasons
+    connect()
+
+    execute('''
+        SELECT datetime('now')
+        ''')
+
+    return current_cursor.fetchone()[0]
+
+
+def place_order(buy, ownership_id, limit, stop_loss, amount, expiry):
+    connect()
+    execute('''
+                INSERT INTO orders 
+                (buy, ownership_id, "limit", stop_loss, ordered_amount, expiry_dt)
+                VALUES (?, ?, ?, ?, ?, ?)
+                ''', (buy, ownership_id, limit, stop_loss, amount, expiry))
+
+    execute_orders(ownable_id_by_ownership_id(ownership_id))
+    return True
+
+
+def trades_on(ownable_id, limit):
+    connect()
+
+    execute('''
+        SELECT datetime(dt,'localtime'), amount, price
+        FROM transactions
+        WHERE ownable_id = ?
+        ORDER BY rowid DESC -- equivalent to order by dt
+        LIMIT ?
+        ''', (ownable_id, limit,))
+
+    return current_cursor.fetchall()
+
+
+def trades(user_id, limit):
+    connect()
+    execute('''
+        SELECT 
+            (CASE WHEN seller_id = ? THEN 'Sell' ELSE 'Buy' END), 
+            (SELECT name FROM ownables WHERE rowid = transactions.ownable_id), 
+            amount, 
+            price,
+            datetime(dt,'localtime')
+        FROM transactions
+        WHERE seller_id = ? OR buyer_id = ?
+        ORDER BY rowid DESC -- equivalent to order by dt
+        LIMIT ?
+        ''', (user_id, user_id, user_id, limit,))
+
+    return current_cursor.fetchall()
+
+
+def drop_expired_orders():
+    connect()
+
+    execute('''
+        SELECT rowid, ownership_id, * FROM orders 
+        WHERE expiry_dt < DATETIME('now')
+        ''')
+
+    data = current_cursor.fetchall()
+    for order in data:
+        order_id = order[0]
+        delete_order(order_id, 'Expired')
+
+    return data
+
+
+def generate_keys(count=1):
+    # source https://stackoverflow.com/questions/17049308/python-3-3-serial-key-generator-list-problems
+
+    for i in range(count):
+        key = '-'.join(random_chars(5) for _ in range(5))
+        save_key(key)
+        print(key)
+
+
+def user_has_order_with_id(session_id, order_id):
+    connect()
+
+    execute('''
+                SELECT orders.rowid
+                FROM orders, ownership, sessions
+                WHERE orders.rowid = ?
+                AND sessions.session_id = ?
+                AND sessions.user_id = ownership.user_id
+                AND ownership.rowid = orders.ownership_id
+                ''', (order_id, session_id,))
+
+    if current_cursor.fetchone():
+        return True
+    else:
+        return False
+
+
+def leaderboard():
+    connect()
+
+    execute('''
+        SELECT * 
+        FROM ( -- one score for each user
+            SELECT 
+                username, 
+                SUM(CASE -- sum score for each of the users ownables
+                    WHEN ownership.ownable_id = ? THEN ownership.amount
+                    ELSE ownership.amount * (SELECT price 
+                                             FROM transactions
+                                             WHERE ownable_id = ownership.ownable_id 
+                                             ORDER BY rowid DESC -- equivalent to ordering by dt
+                                             LIMIT 1)
+                    END
+                ) score
+            FROM users, ownership
+            WHERE ownership.user_id = users.rowid
+            AND users.username != 'bank'
+            GROUP BY users.rowid
+        ) AS scores
+        ORDER BY score DESC
+        LIMIT 50
+        ''', (currency_id(),))
+
+    return current_cursor.fetchall()
+
+
+def user_wealth(user_id):
+    connect()
+
+    execute('''
+        SELECT COALESCE(SUM(
+            CASE -- sum score for each of the users ownables
+            WHEN ownership.ownable_id = ? THEN ownership.amount
+            ELSE ownership.amount * (SELECT price 
+                                     FROM transactions
+                                     WHERE ownable_id = ownership.ownable_id 
+                                     ORDER BY rowid DESC -- equivalent to ordering by dt
+                                     LIMIT 1)
+            END
+        ), 0) score
+        FROM ownership
+        WHERE ownership.user_id = ?
+        ''', (currency_id(), user_id,))
+
+    return current_cursor.fetchone()[0]
+
+
+def change_password(session_id, password, salt):
+    connect()
+
+    execute('''
+                UPDATE users
+                SET password = ?, salt= ?
+                WHERE rowid = (SELECT user_id FROM sessions WHERE sessions.session_id = ?)
+                ''', (password, salt, session_id,))
+
+
+def sign_out_user(session_id):
+    connect()
+
+    execute('''
+        DELETE FROM sessions
+        WHERE user_id = (SELECT user_id FROM sessions s2 WHERE s2.session_id = ?)
+        ''', (session_id,))
+
+
+def delete_user(user_id):
+    connect()
+
+    execute('''
+        DELETE FROM sessions
+        WHERE user_id = ?
+        ''', (user_id,))
+
+    execute('''
+        DELETE FROM orders
+        WHERE ownership_id IN (
+            SELECT rowid FROM ownership WHERE user_id = ?)
+        ''', (user_id,))
+
+    execute('''
+        DELETE FROM ownership
+        WHERE user_id = ?
+        ''', (user_id,))
+
+    execute('''
+        DELETE FROM keys
+        WHERE used_by_user_id = ?
+        ''', (user_id,))
+
+    execute('''
+        INSERT INTO news(title)
+        VALUES ((SELECT username FROM users WHERE rowid = ?) || ' retired.')
+        ''', (user_id,))
+
+    execute('''
+        DELETE FROM users
+        WHERE rowid = ?
+        ''', (user_id,))
+
+
+def delete_ownable(ownable_id):
+    connect()
+
+    execute('''
+        DELETE FROM transactions
+        WHERE ownable_id = ?
+        ''', (ownable_id,))
+
+    execute('''
+        DELETE FROM orders
+        WHERE ownership_id IN (
+            SELECT rowid FROM ownership WHERE ownable_id = ?)
+        ''', (ownable_id,))
+
+    execute('''
+        DELETE FROM order_history
+        WHERE ownership_id IN (
+            SELECT rowid FROM ownership WHERE ownable_id = ?)
+        ''', (ownable_id,))
+
+    # only delete empty ownerships
+    execute('''
+        DELETE FROM ownership
+        WHERE ownable_id = ?
+        AND amount = 0
+        ''', (ownable_id,))
+
+    execute('''
+        INSERT INTO news(title)
+        VALUES ((SELECT name FROM ownables WHERE rowid = ?) || ' can not be traded any more.')
+        ''', (ownable_id,))
+
+    execute('''
+        DELETE FROM ownables
+        WHERE rowid = ?
+        ''', (ownable_id,))
+
+
+def hash_all_users_passwords():
+    connect()
+
+    execute('''
+        SELECT rowid, password, salt
+        FROM users
+        ''')
+
+    users = current_cursor.fetchall()
+
+    for user_id, pw, salt in users:
+        valid_hash = True
+        try:
+            sha256_crypt.verify('password' + salt, pw)
+        except ValueError:
+            valid_hash = False
+        if valid_hash:
+            raise AssertionError('There is already a hashed password in the database! Be careful what you are doing!')
+        pw = sha256_crypt.encrypt(pw + salt)
+        execute('''
+            UPDATE users
+            SET password = ?
+            WHERE rowid = ?
+            ''', (pw, user_id,))
+
+
+def new_news(message):
+    connect()
+    execute('''
+        INSERT INTO news(title)
+        VALUES (?)
+        ''', (message,))
+
+
+def abs_spread(ownable_id):
+    connect()
+
+    execute('''
+        SELECT 
+            (SELECT MAX("limit") 
+             FROM orders, ownership
+             WHERE ownership.rowid = orders.ownership_id
+             AND ownership.ownable_id = ?
+             AND buy
+             AND NOT stop_loss) AS bid, 
+            (SELECT MIN("limit") 
+             FROM orders, ownership
+             WHERE ownership.rowid = orders.ownership_id
+             AND ownership.ownable_id = ?
+             AND NOT buy
+             AND NOT stop_loss) AS ask
+        ''', (ownable_id, ownable_id,))
+
+    return current_cursor.fetchone()
+
+
+def ownables():
+    connect()
+
+    execute('''
+        SELECT name, course,
+            (SELECT SUM(amount)
+            FROM ownership
+            WHERE ownership.ownable_id = ownables_with_course.rowid) market_size
+        FROM (SELECT
+                name, ownables.rowid,
+                CASE WHEN ownables.rowid = ? 
+                THEN 1
+                ELSE (SELECT price
+                      FROM transactions
+                      WHERE ownable_id = ownables.rowid
+                      ORDER BY rowid DESC -- equivalent to ordering by dt
+                      LIMIT 1) END course
+             FROM ownables) ownables_with_course
+        ''', (currency_id(),))
+
+    data = current_cursor.fetchall()
+
+    for idx in range(len(data)):
+        # compute market cap
+        row = data[idx]
+        if row[1] is None:
+            market_cap = None
+        elif row[2] is None:
+            market_cap = None
+        else:
+            market_cap = row[1] * row[2]
+        data[idx] = (row[0], row[1], market_cap)
+
+    return data
+
+
+def reset_bank():
+    connect()
+    execute('''
+        DELETE FROM ownership 
+        WHERE user_id = ?
+        ''', (bank_id(),))
+
+
+def cleanup():
+    global connections
+    global current_connection
+    global current_cursor
+    global current_db_name
+    global current_user_id
+    for name in connections:
+        connections[name].rollback()
+        connections[name].close()
+    connections = []
+    current_connection = None
+    current_cursor = None
+    current_db_name = None
+    current_user_id = None
+
+
+def ownable_ids():
+    connect()
+
+    execute('''
+        SELECT rowid FROM ownables
+        ''')
+
+    return [ownable_id[0] for ownable_id in current_cursor.fetchall()]
+
+
+def get_old_orders(user_id, include_executed, include_canceled, limit):
+    connect()
+    execute('''
+        SELECT 
+            (CASE WHEN order_history.buy THEN 'Buy' ELSE 'Sell' END),
+            ownables.name,
+            (order_history.ordered_amount - order_history.executed_amount) || '/' || order_history.ordered_amount,
+            order_history."limit",
+            order_history.expiry_dt,
+            order_history.order_id,
+            order_history.status
+        FROM order_history, ownership, ownables
+        WHERE ownership.user_id = ?
+        AND ownership.rowid = order_history.ownership_id
+        AND ownables.rowid = ownership.ownable_id
+        AND (
+             (order_history.status = 'Executed' AND ?)
+             OR 
+             ((order_history.status = 'Expired' OR order_history.status = 'Canceled') AND ?)
+            )
+        ORDER BY order_history.rowid DESC -- equivalent to ordering by creation time
+        LIMIT ?
+        ''', (user_id, include_executed, include_canceled, limit))
+
+    return current_cursor.fetchall()

my_types.py

@@ -0,0 +1,8 @@
+from typing import Tuple, Dict, List
+
+MessageType = str
+Message = Dict
+UserId = int
+DbName = str
+UserIdentification = Tuple[DbName, UserId]
+MessageQueue = List[Tuple[List[UserIdentification], Message, MessageType]]

recreate_db.py

@@ -0,0 +1,17 @@
+import os
+import time
+
+import model
+from game import DB_NAME
+
+start = time.clock()
+
+db_name = DB_NAME
+try:
+    os.remove(db_name + '.db')
+except FileNotFoundError:
+    pass
+model.connect(db_name, create_if_not_exists=True)
+print('db name:', db_name)
+model.current_connection.commit()
+print('Processing time:', time.clock() - start)

routes.py

@@ -0,0 +1,25 @@
+# for example host:port/json/create_game is a valid route if using the POST method
+valid_post_routes = {
+    'login',
+    'register',
+    'depot',
+    'activate_key',
+    'order', 'orders',
+    'news',
+    'trades',
+    'trades_on',
+    'orders_on',
+    'old_orders',
+    'cancel_order',
+    'leaderboard',
+    'tradables',
+    'gift',
+    'change_password'
+}
+
+push_message_types = set()
+
+upload_filtered = set()  # TODO enable upload filter again when accuracy improves
+
+assert len(set(valid_post_routes)) == len(valid_post_routes)
+assert upload_filtered.issubset(valid_post_routes)

run_client.py

@@ -5,10 +5,13 @@ import time
 
 import client_controller
 from debug import debug
+from lib.print_exc_plus import print_exc_plus
 from util import yn_dialog
 
 
 def fake_loading_bar(msg, duration=5.):
+    if debug:
+        duration /= 10
     if len(msg) >= 60:
         raise AssertionError('Loading bar label too large')
     msg += ': '
@@ -48,7 +51,7 @@ def welcome():
                                                                       
                                                                       
                                                                       
-To display an overview of available commands type \'help\'.
+To display an overview of available commands type 'help'.
 
 ''')
 
@@ -105,6 +108,8 @@ def one_command():
             except KeyboardInterrupt:
                 print('Interrupted')
             except Exception as _:
+                if debug:
+                    print_exc_plus()
                 print('An unknown error occurred while executing a command.')
 
 

run_db_setup.py

@@ -1,4 +0,0 @@
-import model
-
-if __name__ == '__main__':
-    model.setup()

run_server.py

@@ -1,66 +1,366 @@
-import sqlite3
-import time
-
-from bottle import run, response, route, redirect
-
-import connection
-import model
-import server_controller
-import trading_bot
-from debug import debug
-from server_controller import not_found
-
-if __name__ == '__main__':
-    print('sqlite3.version', model.db.version)
-    model.connect()
-
-    valid_routes = ['login',
-                    'register',
-                    'depot',
-                    'activate_key',
-                    'order', 'orders',
-                    'news',
-                    'trades',
-                    'trades_on',
-                    'orders_on',
-                    'old_orders',
-                    'cancel_order',
-                    'leaderboard',
-                    'tradables',
-                    'gift',
-                    'change_password']
-
-
-    @route('/<path>', method='POST')
-    def process(path):
-        start = time.clock()
-        path = path.strip().lower()
-        if path not in valid_routes:
-            print('Processing time:', time.clock() - start)
-            return not_found()
-        response.content_type = 'application/json'
-        method_to_call = getattr(server_controller, path)
-        try:
-            expired_orders = model.drop_expired_orders()
-            trading_bot.notify_expired_orders(expired_orders)
-            resp = method_to_call()
-            if response.status_code == 200:
-                model.connection.commit()
-            else:
-                model.connection.rollback()
-            print('Processing time:', time.clock() - start)
-            return resp
-        except sqlite3.IntegrityError as e:
-            print(e)
-            model.connection.rollback()
-            print('Processing time:', time.clock() - start)
-            return server_controller.internal_server_error('Action violates database constraints.')
-
-
-    @route('/', method='GET')
-    def process():
-        redirect('http://koljastrohm-games.com/downloads/orderer_installer.zip')
-
-
-    run(host='0.0.0.0', port=connection.port, debug=debug)
-    model.connection.close()
+import datetime
+import errno
+import json
+import os
+import random
+import re
+import sys
+import time
+from json import JSONDecodeError
+from logging import INFO
+from threading import Thread
+from typing import Dict, Any
+
+import bottle
+# noinspection PyUnresolvedReferences
+from bottle.ext.websocket import GeventWebSocketServer
+# noinspection PyUnresolvedReferences
+from bottle.ext.websocket import websocket
+from gevent import threading
+from gevent.queue import Queue, Empty
+from gevent.threading import Lock
+from geventwebsocket import WebSocketError
+from geventwebsocket.websocket import WebSocket
+
+import connection
+import model
+import server_controller
+from connection import HttpError
+from debug import debug
+from game import ROOT_URL, COPYRIGHT_INFRINGEMENT_PROBABILITY, DB_NAME, logger
+from lib.print_exc_plus import print_exc_plus
+from lib.threading_timer_decorator import exit_after
+from routes import valid_post_routes, upload_filtered
+from util import round_to_n, rename, profile_wall_time_instead_if_profiling, LogicError
+
+FRONTEND_RELATIVE_PATH = './frontend'
+
+profile_wall_time_instead_if_profiling()
+request_lock = Lock()  # locked until the response to the request is computed
+db_commit_threads = Queue()
+if debug:
+    TIMEOUT = 600
+else:
+    TIMEOUT = 10
+
+assert all(getattr(server_controller, route) for route in valid_post_routes)
+
+
+def reset_global_variables():
+    model.current_connection = None
+    model.current_cursor = None
+    model.current_db_name = None
+    model.current_user_id = None
+    del connection.push_message_queue[:]
+    bottle.response.status = 500
+
+
+@exit_after(TIMEOUT)
+def call_controller_method_with_timeout(method, json_request: Dict[str, Any]):
+    return method(json_request)
+
+
+def _process(path, json_request):
+    start = time.clock()
+    path = path.strip().lower()
+    bottle.response.content_type = 'application/json; charset=latin-1'
+    reset_global_variables()
+    original_request = None
+    # noinspection PyBroadException
+    try:
+        json_request = json_request()
+        original_request = json_request
+        logger.log(path, INFO, message_type='handling_http_request', data=json.dumps({
+            'request': json_request,
+            'start': start,
+        }))
+        if json_request is None:
+            bottle.response.status = 400
+            resp = connection.BadRequest('Only json allowed.')
+        elif path not in valid_post_routes and not debug:
+            print('Processing time:', time.clock() - start)
+            resp = connection.NotFound('URL not available')
+        else:
+            model.connect(DB_NAME, create_if_not_exists=True)
+            if path not in valid_post_routes:
+                if not debug:
+                    raise LogicError
+                method_to_call = server_controller._mocked_route
+            else:
+                method_to_call = getattr(server_controller, path)
+            try:
+                resp = call_controller_method_with_timeout(method_to_call, json_request)
+                if isinstance(resp, HttpError):
+                    raise resp
+                raise connection.Success(resp)
+            except HttpError as e:
+                bottle.response.status = e.code
+                resp = e
+        if not isinstance(resp.body, dict):
+            raise TypeError('The response body should always be a dict')
+        if resp.code // 100 == 2 and path in upload_filtered and random.random() < COPYRIGHT_INFRINGEMENT_PROBABILITY:
+            resp = connection.UnavailableForLegalReasons('An upload filter detected a copyright infringement. '
+                                                         'If you think this is an error, please try again.')
+        bottle.response.status = resp.code
+        if model.current_connection is not None:
+            if bottle.response.status_code == 200:
+                thread = Thread(target=finish_request, args=[], kwargs={'success': True}, daemon=False)
+            else:
+                thread = Thread(target=finish_request, args=[], kwargs={'success': False}, daemon=False)
+            db_commit_threads.put(thread)
+            thread.start()
+        print('route=' + path, 't=' + str(round_to_n(time.clock() - start, 4)) + 's,',
+              'db=' + str(model.current_db_name))
+        logger.log(path, INFO, message_type='http_request_finished', data=json.dumps({
+            'request': json_request,
+            'response': resp.body,
+            'status': resp.code,
+            'start': start,
+            'end': time.clock(),
+        }))
+        return resp.body
+    except JSONDecodeError:
+        return handle_error('Unable to decode JSON', path, start, original_request)
+    except NotImplementedError:
+        return handle_error('This feature has not been fully implemented yet.', path, start, original_request)
+    except KeyboardInterrupt:
+        if time.clock() - start > TIMEOUT:
+            return handle_error('Processing timeout', path, start, original_request)
+        else:
+            raise
+    except Exception:
+        return handle_error('Unknown error', path, start, original_request)
+
+
+def finish_request(success):
+    if success:
+        model.current_connection.commit()
+        connection.push_messages_in_queue()
+    else:
+        model.current_connection.rollback()
+
+
+if __name__ == '__main__':
+    print('sqlite3.version', model.db.version)
+    if debug:
+        print('Running server in debug mode...')
+
+    print('Preparing backend API...')
+
+
+    @bottle.route('/json/<path>', method='POST')
+    def process(path):
+        with request_lock:
+            wait_for_db_commit_threads()
+            return _process(path, lambda: bottle.request.json)
+
+
+    def wait_for_db_commit_threads():
+        while len(db_commit_threads) > 0:
+            try:
+                t = db_commit_threads.get()
+            except Empty:
+                break
+            t.join()
+
+
+    print('Preparing index page...')
+
+
+    @bottle.route('/', method='GET')
+    def index():
+        if ROOT_URL != '/':
+            bottle.redirect(ROOT_URL)
+
+
+    def handle_error(message, path, start, request, status=500):
+        bottle.response.status = status
+        print_exc_plus()
+        if model.current_connection is not None:
+            model.current_connection.rollback()
+        print('route=' + str(path), 't=' + str(round_to_n(time.clock() - start, 4)) + 's,',
+              'db=' + str(model.current_db_name))
+        logger.exception(path, message_type='http_request', data=json.dumps({
+            'status': status,
+            'start': start,
+            'end': time.clock(),
+            'exception': str(sys.exc_info()),
+            'request': request,
+        }))
+        return connection.InternalServerError(message).body
+
+
+    print('Preparing websocket connections...')
+
+
+    @bottle.get('/websocket', apply=[websocket])
+    def websocket(ws: WebSocket):
+        print('websocket connection', *ws.handler.client_address, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
+        while True:
+            start = time.clock()
+            path = None
+            request_token = None
+            outer_json = None
+
+            # noinspection PyBroadException
+            try:
+                if ws.closed:
+                    connection.ws_cleanup(ws)
+                    break
+                try:
+                    msg = ws.read_message()
+                except ConnectionResetError:
+                    msg = None
+                except WebSocketError as e:
+                    if e.args[0] == 'Unexpected EOF while decoding header':
+                        msg = None
+                    else:
+                        raise
+
+                if msg is not None:  # received some message
+                    with request_lock:
+                        wait_for_db_commit_threads()
+                        msg = bytes(msg)
+                        outer_json = None
+                        outer_json = bottle.json_loads(msg)
+                        path = outer_json['route']
+                        inner_json = outer_json['body']
+                        request_token = outer_json['request_token']
+                        inner_result_json = _process(path, lambda: inner_json)
+
+                        if 'error' in inner_result_json:
+                            status_code = int(inner_result_json['error'][:3])
+                        else:
+                            status_code = 200
+
+                        if model.current_user_id is not None and status_code == 200:
+                            # if there is a user_id involved, associate it with this websocket
+                            user_id = (model.current_db_name, model.current_user_id)
+
+                            if user_id in connection.websockets_for_user:
+                                if ws not in connection.websockets_for_user[user_id]:
+                                    connection.websockets_for_user[user_id].append(ws)
+                            else:
+                                connection.websockets_for_user[user_id] = [ws]
+                            if ws in connection.users_for_websocket:
+                                if user_id not in connection.users_for_websocket[ws]:
+                                    connection.users_for_websocket[ws].append(user_id)
+                            else:
+                                connection.users_for_websocket[ws] = [user_id]
+
+                        outer_result_json = {
+                            'body': inner_result_json,
+                            'http_status_code': status_code,
+                            'request_token': request_token
+                        }
+                        outer_result_json = json.dumps(outer_result_json)
+                        if ws.closed:
+                            connection.ws_cleanup(ws)
+                            break
+                        ws.send(outer_result_json)
+                        print('websocket message',
+                              *ws.handler.client_address,
+                              datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                              status_code,
+                              len(outer_result_json))
+                else:
+                    connection.ws_cleanup(ws)
+                    break
+            except JSONDecodeError:
+                inner_result_json = handle_error('Unable to decode outer JSON', path, start, outer_json)
+                status_code = 403
+                inner_result_json['http_status_code'] = status_code
+                if request_token is not None:
+                    inner_result_json['request_token'] = request_token
+                inner_result_json = json.dumps(inner_result_json)
+                if ws.closed:
+                    connection.ws_cleanup(ws)
+                    break
+                ws.send(inner_result_json)
+                print('websocket message',
+                      *ws.handler.client_address,
+                      datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                      status_code,
+                      len(inner_result_json))
+            except Exception:
+                inner_result_json = handle_error('Unknown error', path, start, outer_json)
+                status_code = 500
+                inner_result_json['http_status_code'] = status_code
+                if request_token is not None:
+                    inner_result_json['request_token'] = request_token
+                inner_result_json = json.dumps(inner_result_json)
+                if ws.closed:
+                    connection.ws_cleanup(ws)
+                    break
+                ws.send(inner_result_json)
+                print('websocket message',
+                      *ws.handler.client_address,
+                      datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                      status_code,
+                      len(inner_result_json))
+
+
+    def _serve_static_directory(route, root, download=False):
+        method_name = ''.join(c for c in root if re.match(r'[A-Za-z]]', c))
+        assert method_name not in globals()
+
+        @bottle.route(route, method=['GET', 'OPTIONS'])
+        @rename(''.join(c for c in root if re.match(r'[A-Za-z]]', c)))
+        def serve_static_file(filename):
+            # start = time.clock()
+            # logger.log(filename, INFO, message_type='handling_http_request', data=json.dumps({
+            #     'start': start,
+            # }))
+            # try:
+            if filename == 'api.json':
+                return {'endpoint': bottle.request.urlparts[0] + '://' + bottle.request.urlparts[1] + '/json/'}
+            if download:
+                default_name = 'ytm-' + filename
+                return bottle.static_file(filename, root=root, download=default_name)
+            else:
+                return bottle.static_file(filename, root=root, download=False)
+            # finally:
+            #     logger.log(filename, INFO, message_type='http_request_finished', data=json.dumps({
+            #         'status': bottle.response.status_code,
+            #         'start': start,
+            #         'end': time.clock(),
+            #     }))
+
+
+    # frontend
+    print('Preparing frontend directories...')
+    if len(os.listdir(FRONTEND_RELATIVE_PATH)) == 0:
+        raise FileNotFoundError(errno.ENOENT, 'Frontend directory is empty:', FRONTEND_RELATIVE_PATH)
+    for subdir, dirs, files in os.walk(FRONTEND_RELATIVE_PATH):
+        # subdir now has the form   ../frontend/config
+        _serve_static_directory(
+            route=subdir.replace('\\', '/').replace(FRONTEND_RELATIVE_PATH, '') + '/<filename>',
+            root=subdir
+        )
+
+    # app
+    print('Preparing app for download...')
+    _serve_static_directory(
+        route='/app/<filename>',
+        root='../android/app/release',
+        download=True,
+    )
+
+    logger.log('Server start', INFO, 'server_start', json.dumps({
+        'host': '0.0.0.0',
+        'port': connection.PORT,
+        'debug': debug,
+    }))
+
+    # commit regularly
+    log_commit_time = logger.commit()
+    log_commit_delay = 15
+    print(f'Committing logfile transaction took {log_commit_time}s, '
+          f'scheduling to run every {log_commit_delay}s')
+    threading.Timer(log_commit_delay, logger.commit).start()
+
+    print('Running server...')
+    bottle.run(host='0.0.0.0', port=connection.PORT, debug=debug, server=GeventWebSocketServer)
+    logger.commit()
+    model.cleanup()

server_controller.py

@@ -1,296 +1,235 @@
-import json
-from datetime import timedelta, datetime
-from math import ceil, floor
-
-from bottle import request, response
-from passlib.hash import sha256_crypt
-
-import model
-from debug import debug
-from util import salt
-
-
-def missing_attributes(attributes):
-    for attr in attributes:
-        if attr not in request.json or request.json[attr] == '' or request.json[attr] is None:
-            if str(attr) == 'session_id':
-                return 'You are not signed in.'
-            return 'Missing value for attribute ' + str(attr)
-        if str(attr) == 'session_id':
-            if not model.valid_session_id(request.json['session_id']):
-                return 'You are not signed in.'
-    return False
-
-
-def login():
-    if debug:
-        missing = missing_attributes(['username'])
-    else:
-        missing = missing_attributes(['username', 'password'])
-    if missing:
-        return bad_request(missing)
-    username = request.json['username']
-    password = request.json['password']
-    session_id = model.login(username, password)
-    if session_id:
-        return {'session_id': session_id}
-    else:
-        return forbidden('Invalid login data')
-
-
-def depot():
-    missing = missing_attributes(['session_id'])
-    if missing:
-        return bad_request(missing)
-    user_id = model.get_user_id_by_session_id(request.json['session_id'])
-    return {'data': model.get_user_ownership(user_id),
-            'own_wealth': model.user_wealth(user_id)}
-
-
-def register():
-    missing = missing_attributes(['username', 'password'])
-    if missing:
-        return bad_request(missing)
-    username = request.json['username'].strip()
-    if username == '':
-        return bad_request('Username can not be empty.')
-    hashed_password = sha256_crypt.encrypt(request.json['password'] + salt)
-    if model.user_exists(username):
-        return bad_request('User already exists.')
-    game_key = ''
-    if 'game_key' in request.json:
-        game_key = request.json['game_key'].strip().upper()
-        if game_key != '' and not model.valid_key(game_key):
-            return bad_request('Game key is not valid.')
-    if model.register(username, hashed_password, game_key):
-        return {'message': "successfully registered user"}
-    else:
-        return bad_request('Registration not successful')
-
-
-def activate_key():
-    missing = missing_attributes(['key', 'session_id'])
-    if missing:
-        return bad_request(missing)
-    if model.valid_key(request.json['key']):
-        user_id = model.get_user_id_by_session_id(request.json['session_id'])
-        model.activate_key(request.json['key'], user_id)
-        return {'message': "successfully activated key"}
-    else:
-        return bad_request('Invalid key.')
-
-
-def order():
-    missing = missing_attributes(['buy', 'session_id', 'amount', 'ownable', 'time_until_expiration'])
-    if missing:
-        return bad_request(missing)
-    if not model.ownable_name_exists(request.json['ownable']):
-        return bad_request('This kind of object can not be ordered.')
-
-    buy = request.json['buy']
-    sell = not buy
-    if not isinstance(buy, bool):
-        return bad_request('`buy` must be a boolean')
-
-    session_id = request.json['session_id']
-    amount = request.json['amount']
-    try:
-        amount = int(amount)
-    except ValueError:
-        return bad_request('Invalid amount.')
-    if amount < 0:
-        return bad_request('You can not order a negative amount.')
-    if amount < 1:
-        return bad_request('The minimum order size is 1.')
-    ownable_name = request.json['ownable']
-    time_until_expiration = float(request.json['time_until_expiration'])
-    if time_until_expiration < 0:
-        return bad_request('Invalid expiration time.')
-    ownable_id = model.ownable_id_by_name(ownable_name)
-    user_id = model.get_user_id_by_session_id(session_id)
-    model.own(user_id, ownable_name)
-    ownership_id = model.get_ownership_id(ownable_id, user_id)
-
-    try:
-        if request.json['limit'] == '':
-            limit = None
-        elif request.json['limit'] is None:
-            limit = None
-        else:
-            if buy:
-                limit = floor(float(request.json['limit']) * 10000) / 10000
-            else:
-                limit = ceil(float(request.json['limit']) * 10000) / 10000
-    except ValueError:  # for example when float fails
-        return bad_request('Invalid limit.')
-    except KeyError:  # for example when limit was not specified
-        limit = None
-
-    if limit < 0:
-        return bad_request('Limit must not be negative.')
-
-    try:
-        if request.json['stop_loss'] == '':
-            stop_loss = None
-        elif request.json['stop_loss'] is None:
-            stop_loss = None
-        else:
-            stop_loss = 'stop_loss' in request.json and request.json['stop_loss']
-        if stop_loss is not None and limit is None:
-            return bad_request('Can only set stop-loss for limit orders')
-    except KeyError:  # for example when stop_loss was not specified
-        stop_loss = None
-
-    if sell:
-        if not model.user_has_at_least_available(amount, user_id, ownable_id):
-            return bad_request('You can not sell more than you own.')
-    try:
-        expiry = datetime.strptime(model.current_db_time(), '%Y-%m-%d %H:%M:%S') + \
-                 timedelta(minutes=time_until_expiration)
-    except OverflowError:
-        return bad_request('The expiration time is too far in the future.')
-    model.place_order(buy, ownership_id, limit, stop_loss, amount, expiry)
-    return {'message': "Order placed."}
-
-
-def gift():
-    missing = missing_attributes(['session_id', 'amount', 'object_name', 'username'])
-    if missing:
-        return bad_request(missing)
-    if not model.ownable_name_exists(request.json['object_name']):
-        return bad_request('This kind of object can not be given away.')
-    if request.json['username'] == 'bank' or not model.user_exists(request.json['username']):
-        return bad_request('There is no user with this name.')
-    try:
-        amount = float(request.json['amount'])
-    except ValueError:
-        return bad_request('Invalid amount.')
-    ownable_id = model.ownable_id_by_name(request.json['object_name'])
-    sender_id = model.get_user_id_by_session_id(request.json['session_id'])
-
-    if model.available_amount(sender_id, ownable_id) == 0:
-        return bad_request('You do not own any of these.')
-    if not model.user_has_at_least_available(amount, sender_id, ownable_id):
-        # for example if you have a 1.23532143213 Kollar and want to give them all away
-        amount = model.available_amount(sender_id, ownable_id)
-
-    recipient_id = model.get_user_id_by_name(request.json['username'])
-
-    model.send_ownable(sender_id,
-                       recipient_id,
-                       ownable_id,
-                       amount)
-
-    return {'message': "Gift sent."}
-
-
-def orders():
-    missing = missing_attributes(['session_id'])
-    if missing:
-        return bad_request(missing)
-    data = model.get_user_orders(model.get_user_id_by_session_id(request.json['session_id']))
-    return {'data': data}
-
-
-def orders_on():
-    missing = missing_attributes(['session_id', 'ownable'])
-    if missing:
-        return bad_request(missing)
-    if not model.ownable_name_exists(request.json['ownable']):
-        return bad_request('This kind of object can not be ordered.')
-    user_id = model.get_user_id_by_session_id(request.json['session_id'])
-    ownable_id = model.ownable_id_by_name(request.json['ownable'])
-    data = model.get_ownable_orders(user_id, ownable_id)
-    return {'data': data}
-
-
-def old_orders():
-    missing = missing_attributes(['session_id', 'include_canceled', 'include_executed', 'limit'])
-    if missing:
-        return bad_request(missing)
-    include_executed = request.json['include_executed']
-    include_canceled = request.json['include_canceled']
-    user_id = model.get_user_id_by_session_id(request.json['session_id'])
-    limit = request.json['limit']
-    data = model.get_old_orders(user_id, include_executed, include_canceled, limit)
-    return {'data': data}
-
-
-def cancel_order():
-    missing = missing_attributes(['session_id', 'order_id'])
-    if missing:
-        return bad_request(missing)
-    if not model.user_has_order_with_id(request.json['session_id'], request.json['order_id']):
-        return bad_request('You do not have an order with that number.')
-    model.delete_order(request.json['order_id'], 'Canceled')
-    return {'message': "Successfully deleted order"}
-
-
-def change_password():
-    missing = missing_attributes(['session_id', 'password'])
-    if missing:
-        return bad_request(missing)
-    hashed_password = sha256_crypt.encrypt(request.json['password'] + salt)
-    model.change_password(request.json['session_id'], hashed_password)
-    model.sign_out_user(request.json['session_id'])
-    return {'message': "Successfully changed password"}
-
-
-def news():
-    return {'data': model.news()}
-
-
-def tradables():
-    return {'data': model.ownables()}
-
-
-def trades():
-    missing = missing_attributes(['session_id', 'limit'])
-    if missing:
-        return bad_request(missing)
-    return {'data': model.trades(model.get_user_id_by_session_id(request.json['session_id']), request.json['limit'])}
-
-
-def trades_on():
-    missing = missing_attributes(['session_id', 'ownable', 'limit'])
-    if missing:
-        return bad_request(missing)
-    if not model.ownable_name_exists(request.json['ownable']):
-        return bad_request('This kind of object can not have transactions.')
-    return {'data': model.trades_on(model.ownable_id_by_name(request.json['ownable']), request.json['limit'])}
-
-
-def leaderboard():
-    return {'data': model.leaderboard()}
-
-
-def not_found(msg=''):
-    response.status = 404
-    if debug:
-        msg = str(response.status) + ': ' + msg
-    response.content_type = 'application/json'
-    return json.dumps({"error_message": msg})
-
-
-def forbidden(msg=''):
-    response.status = 403
-    if debug:
-        msg = str(response.status) + ': ' + msg
-    response.content_type = 'application/json'
-    return json.dumps({"error_message": msg})
-
-
-def bad_request(msg=''):
-    response.status = 400
-    if debug:
-        msg = str(response.status) + ': ' + msg
-    response.content_type = 'application/json'
-    return json.dumps({"error_message": msg})
-
-
-def internal_server_error(msg=''):
-    response.status = 500
-    if debug:
-        msg = str(response.status) + ': ' + msg
-    response.content_type = 'application/json'
-    return json.dumps({"error_message": msg})
+import uuid
+from datetime import timedelta, datetime
+from math import ceil, floor
+
+from bottle import request
+from passlib.hash import sha256_crypt
+
+import model
+from connection import check_missing_attributes, BadRequest, Forbidden
+
+
+def missing_attributes(attributes):
+    for attr in attributes:
+        if attr not in request.json or request.json[attr] == '' or request.json[attr] is None:
+            if str(attr) == 'session_id':
+                return 'You are not signed in.'
+            return 'Missing value for attribute ' + str(attr)
+        if str(attr) == 'session_id':
+            if not model.valid_session_id(request.json['session_id']):
+                return 'You are not signed in.'
+    return False
+
+
+def login(json_request):
+
+    check_missing_attributes(json_request, ['username', 'password'])
+    username = request.json['username']
+    password = request.json['password']
+    session_id = model.login(username, password)
+    if session_id:
+        return {'session_id': session_id}
+    else:
+        return Forbidden('Invalid login data')
+
+
+def depot(json_request):
+    check_missing_attributes(json_request, ['session_id'])
+    user_id = model.get_user_id_by_session_id(request.json['session_id'])
+    return {'data': model.get_user_ownership(user_id),
+            'own_wealth': model.user_wealth(user_id)}
+
+
+def register(json_request):
+    check_missing_attributes(json_request, ['username', 'password'])
+    username = request.json['username'].strip()
+    if username == '':
+        return BadRequest('Username can not be empty.')
+    if model.user_exists(username):
+        return BadRequest('User already exists.')
+    game_key = ''
+    if 'game_key' in request.json:
+        game_key = request.json['game_key'].strip().upper()
+        if game_key != '' and not model.valid_key(game_key):
+            return BadRequest('Game key is not valid.')
+    if model.register(username, request.json['password'], game_key):
+        return {'message': "successfully registered user"}
+    else:
+        return BadRequest('Registration not successful')
+
+
+def activate_key(json_request):
+    check_missing_attributes(json_request, ['key', 'session_id'])
+    if model.valid_key(request.json['key']):
+        user_id = model.get_user_id_by_session_id(request.json['session_id'])
+        model.activate_key(request.json['key'], user_id)
+        return {'message': "successfully activated key"}
+    else:
+        return BadRequest('Invalid key.')
+
+
+def order(json_request):
+    check_missing_attributes(json_request, ['buy', 'session_id', 'amount', 'ownable', 'time_until_expiration'])
+    if not model.ownable_name_exists(request.json['ownable']):
+        return BadRequest('This kind of object can not be ordered.')
+
+    buy = request.json['buy']
+    sell = not buy
+    if not isinstance(buy, bool):
+        return BadRequest('`buy` must be a boolean')
+
+    session_id = request.json['session_id']
+    amount = request.json['amount']
+    try:
+        amount = int(amount)
+    except ValueError:
+        return BadRequest('Invalid amount.')
+    if amount < 0:
+        return BadRequest('You can not order a negative amount.')
+    if amount < 1:
+        return BadRequest('The minimum order size is 1.')
+    ownable_name = request.json['ownable']
+    time_until_expiration = float(request.json['time_until_expiration'])
+    if time_until_expiration < 0:
+        return BadRequest('Invalid expiration time.')
+    ownable_id = model.ownable_id_by_name(ownable_name)
+    user_id = model.get_user_id_by_session_id(session_id)
+    model.own(user_id, ownable_name)
+    ownership_id = model.get_ownership_id(ownable_id, user_id)
+
+    try:
+        if request.json['limit'] == '':
+            limit = None
+        elif request.json['limit'] is None:
+            limit = None
+        else:
+            if buy:
+                limit = floor(float(request.json['limit']) * 10000) / 10000
+            else:
+                limit = ceil(float(request.json['limit']) * 10000) / 10000
+    except ValueError:  # for example when float fails
+        return BadRequest('Invalid limit.')
+    except KeyError:  # for example when limit was not specified
+        limit = None
+
+    if limit < 0:
+        return BadRequest('Limit must not be negative.')
+
+    try:
+        if request.json['stop_loss'] == '':
+            stop_loss = None
+        elif request.json['stop_loss'] is None:
+            stop_loss = None
+        else:
+            stop_loss = 'stop_loss' in request.json and request.json['stop_loss']
+        if stop_loss is not None and limit is None:
+            return BadRequest('Can only set stop-loss for limit orders')
+    except KeyError:  # for example when stop_loss was not specified
+        stop_loss = None
+
+    if sell:
+        if not model.user_has_at_least_available(amount, user_id, ownable_id):
+            return BadRequest('You can not sell more than you own.')
+    try:
+        expiry = datetime.strptime(model.current_db_time(), '%Y-%m-%d %H:%M:%S') + \
+                 timedelta(minutes=time_until_expiration)
+    except OverflowError:
+        return BadRequest('The expiration time is too far in the future.')
+    model.place_order(buy, ownership_id, limit, stop_loss, amount, expiry)
+    return {'message': "Order placed."}
+
+
+def gift(json_request):
+    check_missing_attributes(json_request, ['session_id', 'amount', 'object_name', 'username'])
+    if not model.ownable_name_exists(request.json['object_name']):
+        return BadRequest('This kind of object can not be given away.')
+    if request.json['username'] == 'bank' or not model.user_exists(request.json['username']):
+        return BadRequest('There is no user with this name.')
+    try:
+        amount = float(request.json['amount'])
+    except ValueError:
+        return BadRequest('Invalid amount.')
+    ownable_id = model.ownable_id_by_name(request.json['object_name'])
+    sender_id = model.get_user_id_by_session_id(request.json['session_id'])
+
+    if model.available_amount(sender_id, ownable_id) == 0:
+        return BadRequest('You do not own any of these.')
+    if not model.user_has_at_least_available(amount, sender_id, ownable_id):
+        # for example if you have a 1.23532143213 Kollar and want to give them all away
+        amount = model.available_amount(sender_id, ownable_id)
+
+    recipient_id = model.get_user_id_by_name(request.json['username'])
+
+    model.send_ownable(sender_id,
+                       recipient_id,
+                       ownable_id,
+                       amount)
+
+    return {'message': "Gift sent."}
+
+
+def orders(json_request):
+    check_missing_attributes(json_request, ['session_id'])
+    data = model.get_user_orders(model.get_user_id_by_session_id(request.json['session_id']))
+    return {'data': data}
+
+
+def orders_on(json_request):
+    check_missing_attributes(json_request, ['session_id', 'ownable'])
+    if not model.ownable_name_exists(request.json['ownable']):
+        return BadRequest('This kind of object can not be ordered.')
+    user_id = model.get_user_id_by_session_id(request.json['session_id'])
+    ownable_id = model.ownable_id_by_name(request.json['ownable'])
+    data = model.get_ownable_orders(user_id, ownable_id)
+    return {'data': data}
+
+
+def old_orders(json_request):
+    check_missing_attributes(json_request, ['session_id', 'include_canceled', 'include_executed', 'limit'])
+    include_executed = request.json['include_executed']
+    include_canceled = request.json['include_canceled']
+    user_id = model.get_user_id_by_session_id(request.json['session_id'])
+    limit = request.json['limit']
+    data = model.get_old_orders(user_id, include_executed, include_canceled, limit)
+    return {'data': data}
+
+
+def cancel_order(json_request):
+    check_missing_attributes(json_request, ['session_id', 'order_id'])
+    if not model.user_has_order_with_id(request.json['session_id'], request.json['order_id']):
+        return BadRequest('You do not have an order with that number.')
+    model.delete_order(request.json['order_id'], 'Canceled')
+    return {'message': "Successfully deleted order"}
+
+
+def change_password(json_request):
+    check_missing_attributes(json_request, ['session_id', 'password'])
+    salt = str(uuid.uuid4())
+    hashed_password = sha256_crypt.encrypt(request.json['password'] + salt)
+    model.change_password(request.json['session_id'], hashed_password, salt)
+    model.sign_out_user(request.json['session_id'])
+    return {'message': "Successfully changed password"}
+
+
+def news(_json_request):
+    return {'data': model.news()}
+
+
+def tradables(_json_request):
+    return {'data': model.ownables()}
+
+
+def trades(json_request):
+    check_missing_attributes(json_request, ['session_id', 'limit'])
+    return {'data': model.trades(model.get_user_id_by_session_id(request.json['session_id']), request.json['limit'])}
+
+
+def trades_on(json_request):
+    check_missing_attributes(json_request, ['session_id', 'ownable', 'limit'])
+    if not model.ownable_name_exists(request.json['ownable']):
+        return BadRequest('This kind of object can not have transactions.')
+    return {'data': model.trades_on(model.ownable_id_by_name(request.json['ownable']), request.json['limit'])}
+
+
+def leaderboard(_json_request):
+    return {'data': model.leaderboard()}

test/__init__.py

@@ -0,0 +1,8 @@
+from random import randint
+
+
+def random_db_name():
+    return 'test' + str(randint(0, 1000000))
+
+
+failed_requests = []

test/do_some_requests/__init__.py

@@ -0,0 +1,218 @@
+import json
+import random
+import re
+import sys
+from datetime import datetime
+from time import perf_counter
+from typing import Dict, Callable
+from uuid import uuid4
+
+import requests
+
+import connection
+import test.do_some_requests.current_websocket
+from test import failed_requests
+from util import round_to_n
+
+DEFAULT_PW = 'pw'
+
+PORT = connection.PORT
+
+HOST = 'http://127.0.0.1' + ':' + str(PORT)
+# HOST = 'http://koljastrohm-games.com' + ':' + str(PORT)
+
+JSON_HEADERS = {'Content-type': 'application/json'}
+
+EXIT_ON_FAILED_REQUEST = True
+
+response_collection: Dict[str, Dict] = {}
+default_request_method: Callable[[str, Dict], Dict]
+
+
+def receive_answer(token):
+    """Waits until the server sends an answer that contains the desired request_token.
+    All intermediate requests are also collected for later use, or, if they contain no token, they are just printed out.
+    """
+    if token in response_collection:
+        json_content = response_collection[token]
+        del response_collection[token]
+        return json_content
+
+    json_content = {}
+    while 'request_token' not in json_content or json_content['request_token'] != token:
+        if 'request_token' in json_content:
+            response_collection[json_content['request_token']] = json_content
+        received = test.do_some_requests.current_websocket.current_websocket.recv_data_frame()[1].data
+        content = received.decode('utf-8')
+        formatted_content = re.sub(r'{([^}]*?):(.*?)}', r'\n{\g<1>:\g<2>}', content)
+        print('Received through websocket: ' + formatted_content)
+        json_content = json.loads(content)
+
+    return json_content
+
+
+def websocket_request(route: str, data: Dict) -> Dict:
+    original_data = data
+    if not test.do_some_requests.current_websocket.current_websocket.connected:
+        ws_host = HOST.replace('http://', 'ws://')
+        test.do_some_requests.current_websocket.current_websocket.connect(ws_host + '/websocket')
+    token = str(uuid4())
+    data = json.dumps({'route': route, 'body': data, 'request_token': token})
+    print('Sending to websocket:', str(data).replace('{', '\n{')[1:])
+    test.do_some_requests.current_websocket.current_websocket.send(data, opcode=2)
+    json_content = receive_answer(token)
+    print()
+
+    status_code = json_content['http_status_code']
+    if status_code == 200:
+        pass
+    elif status_code == 451:  # Copyright problems, likely a bug in the upload filter
+        # Try again
+        return websocket_request(route, original_data)
+    else:
+        if EXIT_ON_FAILED_REQUEST:
+            if not test.do_some_requests.current_websocket.current_websocket.connected:
+                test.do_some_requests.current_websocket.current_websocket.close()
+            sys.exit(status_code)
+        failed_requests.append((route, status_code))
+
+    return json_content['body']
+
+
+def http_request(route: str, data: Dict) -> Dict:
+    original_data = data
+    data = json.dumps(data)
+    print('Sending to /' + route + ':', str(data).replace('{', '\n{')[1:])
+    r = requests.post(HOST + '/json/' + route, data=data,
+                      headers=JSON_HEADERS)
+    content = r.content.decode()
+    print('Request returned: ' + content.replace('{', '\n{'))
+    print()
+    if r.status_code == 200:
+        pass
+    elif r.status_code == 451:
+        return http_request(route, original_data)
+    else:
+        if EXIT_ON_FAILED_REQUEST:
+            sys.exit(r.status_code)
+        failed_requests.append((route, r.status_code))
+    return json.loads(content)
+
+
+default_request_method: Callable[[str, Dict], Dict] = http_request
+
+
+# default_request_method = websocket_request
+
+
+def random_time():
+    start = random.randrange(140)
+    start = 1561960800 + start * 21600  # somewhere in july or at the beginning of august 2019
+    return {
+        'dt_start': start,
+        'dt_end': start + random.choice([1800, 3600, 7200, 86400]),
+    }
+
+
+def run_tests():
+    print('You are currently in debug mode.')
+    print('Host:', str(HOST))
+    usernames = [f'user{datetime.now().timestamp()}',
+                 f'user{datetime.now().timestamp()}+1']
+
+    session_ids = {}
+
+    message = {}
+    route = 'news'
+    default_request_method(route, message)
+
+    message = {}
+    route = 'leaderboard'
+    default_request_method(route, message)
+
+    message = {}
+    route = 'tradables'
+    default_request_method(route, message)
+
+    for username in usernames:
+        message = {'username': username, 'password': DEFAULT_PW}
+        route = 'register'
+        default_request_method(route, message)
+
+        message = {'username': username, 'password': DEFAULT_PW}
+        route = 'login'
+        session_ids[username] = default_request_method(route, message)['session_id']
+
+        message = {'session_id': session_ids[username]}
+        route = 'logout'
+        default_request_method(route, message)
+
+        message = {'username': username, 'password': DEFAULT_PW}
+        route = 'login'
+        session_ids[username] = default_request_method(route, message)['session_id']
+
+        message = {'session_id': session_ids[username]}
+        route = 'depot'
+        default_request_method(route, message)
+
+        message = {'session_id': session_ids[username]}
+        route = 'orders'
+        default_request_method(route, message)
+
+        message = {'session_id': session_ids[username], "ownable": "\u20adollar"}
+        route = 'orders_on'
+        default_request_method(route, message)
+
+        for password in ['pw2', DEFAULT_PW]:
+            message = {'session_id': session_ids[username], 'password': password}
+            route = 'change_password'
+            default_request_method(route, message)
+
+            message = {'session_id': session_ids[username]}
+            route = 'logout'
+            default_request_method(route, message)
+
+            message = {'username': username, 'password': DEFAULT_PW}
+            route = 'login'
+            session_ids[username] = default_request_method(route, message)['session_id']
+
+        for limit in [0, 5, 10, 20, 50]:
+            message = {'session_id': session_ids[username], 'limit': limit}
+            route = 'trades'
+            data = default_request_method(route, message)['data']
+            assert len(data) <= limit
+
+    for session_id in session_ids:
+        message = {'session_id': session_id}
+        route = 'logout'
+        default_request_method(route, message)
+
+
+def main():
+    global default_request_method
+    for m in [
+        test.do_some_requests.websocket_request,
+        # test.do_some_requests.http_request
+    ]:
+        # print('Removing existing database (if exists)...', end='')
+        # try:
+        #     os.remove(DB_NAME + '.db')
+        # except PermissionError:
+        #     print('Could not recreate database')
+        #     sys.exit(-1)
+        # except FileNotFoundError:
+        #     pass
+        # print('done')
+        default_request_method = m
+        start = perf_counter()
+        run_tests()
+        print()
+        print('Failed requests:', failed_requests)
+        print('Total time:' + str(round_to_n(perf_counter() - start, 4)) + 's,')
+    if test.do_some_requests.current_websocket.current_websocket.connected:
+        test.do_some_requests.current_websocket.current_websocket.close()
+    sys.exit(len(failed_requests))
+
+
+if __name__ == '__main__':
+    main()

test/do_some_requests/current_websocket.py

@@ -0,0 +1,3 @@
+import websocket
+
+current_websocket: websocket.WebSocket = websocket.WebSocket()

util.py

@@ -1,63 +1,287 @@
-from random import random
-
-import tabulate
-
-# noinspection SpellCheckingInspection
-salt = 'orderer_is_a_cool_application_]{][{²$%WT§$%GV§$%SF$%&S$%FGGFHBDHJZIF254325'
-
-chars = [str(d) for d in range(1, 10)]
-
-ps = [1. for _ in chars]
-# the first number is absolute and the second relative
-letter_dist = [("E", 21912, 12.02), ("T", 16587, 9.1), ("A", 14810, 8.12), ("O", 14003, 7.68), ("I", 13318, 7.31),
-               ("N", 12666, 6.95), ("S", 11450, 6.28), ("R", 10977, 6.02), ("H", 10795, 5.92), ("D", 7874, 4.32),
-               ("L", 7253, 3.98), ("U", 5246, 2.88), ("C", 4943, 2.71), ("M", 4761, 2.61), ("F", 4200, 2.3),
-               ("Y", 3853, 2.11), ("W", 3819, 2.09), ("G", 3693, 2.03), ("P", 3316, 1.82), ("B", 2715, 1.49),
-               ("V", 2019, 1.11), ("K", 1257, 0.69), ("X", 315, 0.17), ("Q", 205, 0.11), ("J", 188, 0.1),
-               ("Z", 128, 0.07), ]
-sp = sum(ps)
-for row in letter_dist:
-    chars.append(row[0])
-    ps.append(float(row[2]))
-ps = [p / sum(ps) for p in ps]
-
-
-def choice(sequence, probabilities):
-    # if sum(probabilities) != 1:
-    #     raise AssertionError('Probabilities must sum up to 1')
-    r = random()
-    for idx, c in enumerate(sequence):
-        r -= probabilities[idx]
-        if r < 0:
-            return c
-    raise AssertionError('Probabilities must sum up to 1')
-
-
-def random_chars(count):
-    return ''.join(choice(chars, probabilities=ps) for _ in range(count))
-
-
-def str2bool(v):
-    v = str(v).strip().lower()
-    if v in ["yes", 'y' "true", "t", "1"]:
-        return True
-    if v in ["no", 'n' "false", "f", "0", '', 'null', 'none']:
-        return False
-    raise ValueError('Can not convert `' + v + '` to bool')
-
-
-def my_tabulate(data, **params):
-    if data == [] and 'headers' in params:
-        data = [(None for _ in params['headers'])]
-    tabulate.MIN_PADDING = 0
-    return tabulate.tabulate(data, **params)
-
-
-def yn_dialog(msg):
-    while True:
-        result = input(msg + ' [y/n]: ')
-        if result == 'y':
-            return True
-        if result == 'n':
-            return False
-        print('Type in \'y\' or \'n\'!')
+import faulthandler
+import functools
+import inspect
+import json
+import os
+import random
+import time
+from bisect import bisect_left
+from datetime import datetime, timedelta
+from math import floor, log10, sqrt, nan, inf
+
+import scipy.stats
+import tabulate
+from numpy.random.mtrand import binomial
+
+from lib import stack_tracer
+from lib.print_exc_plus import print_exc_plus
+
+chars = [str(d) for d in range(1, 10)]
+digits = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ps = [1. for _ in chars]
+# the first number is absolute and the second relative
+letter_dist = [("E", 21912, 12.02), ("T", 16587, 9.1), ("A", 14810, 8.12), ("O", 14003, 7.68), ("I", 13318, 7.31),
+               ("N", 12666, 6.95), ("S", 11450, 6.28), ("R", 10977, 6.02), ("H", 10795, 5.92), ("D", 7874, 4.32),
+               ("L", 7253, 3.98), ("U", 5246, 2.88), ("C", 4943, 2.71), ("M", 4761, 2.61), ("F", 4200, 2.3),
+               ("Y", 3853, 2.11), ("W", 3819, 2.09), ("G", 3693, 2.03), ("P", 3316, 1.82), ("B", 2715, 1.49),
+               ("V", 2019, 1.11), ("K", 1257, 0.69), ("X", 315, 0.17), ("Q", 205, 0.11), ("J", 188, 0.1),
+               ("Z", 128, 0.07), ]
+sp = sum(ps)
+for row in letter_dist:
+    chars.append(row[0])
+    ps.append(float(row[2]))
+ps = [p / sum(ps) for p in ps]
+
+
+def choice(sequence, probabilities):
+    # if sum(probabilities) != 1:
+    #     raise AssertionError('Probabilities must sum to 1')
+    r = random.random()
+    for idx, c in enumerate(sequence):
+        r -= probabilities[idx]
+        if r < 0:
+            return c
+    raise AssertionError('Probabilities must sum to 1')
+
+
+def multiple_choice(sequence, count):
+    results = []
+    num_remaining = len(sequence)
+    for _ in range(count):
+        idx = random.randrange(num_remaining)
+        results.append(sequence[idx])
+        del sequence[idx]
+        num_remaining -= 1
+    return results
+
+
+try:
+    import winsound as win_sound
+
+
+    def beep(*args, **kwargs):
+        win_sound.Beep(*args, **kwargs)
+except ImportError:
+    win_sound = None
+
+
+    def beep(*_args, **_kwargs):
+        pass
+
+
+def main_wrapper(f):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        # import lib.stack_tracer
+        import __main__
+        # does not help much
+        # monitoring_thread = hanging_threads.start_monitoring(seconds_frozen=180, test_interval=1000)
+        os.makedirs('logs', exist_ok=True)
+        stack_tracer.trace_start('logs/' + os.path.split(__main__.__file__)[-1] + '.html', interval=5)
+        faulthandler.enable()
+        profile_wall_time_instead_if_profiling()
+
+        # noinspection PyBroadException
+        try:
+            f(*args, **kwargs)
+        except Exception:
+            print_exc_plus()
+            exit(-1)
+        finally:
+            total_time = time.perf_counter() - start
+            frequency = 2000
+            duration = 500
+            beep(frequency, duration)
+            print('Total time', total_time)
+
+    return wrapper
+
+
+def random_chars(count):
+    return ''.join(choice(chars, probabilities=ps) for _ in range(count))
+
+
+def str2bool(v):
+    v = str(v).strip().lower()
+    if v in ["yes", 'y' "true", "t", "1"]:
+        return True
+    if v in ["no", 'n' "false", "f", "0", '', 'null', 'none']:
+        return False
+    raise ValueError('Can not convert `' + v + '` to bool')
+
+
+def my_tabulate(data, **params):
+    if data == [] and 'headers' in params:
+        data = [(None for _ in params['headers'])]
+    tabulate.MIN_PADDING = 0
+    return tabulate.tabulate(data, **params)
+
+
+def yn_dialog(msg):
+    while True:
+        result = input(msg + ' [y/n]: ')
+        if result == 'y':
+            return True
+        if result == 'n':
+            return False
+        print('Type in \'y\' or \'n\'!')
+
+
+def round_to_closest_value(x, values):
+    values = sorted(values)
+    next_largest = bisect_left(values, x)  # binary search
+    if next_largest == 0:
+        return values[0]
+    if next_largest == len(values):
+        return values[-1]
+    next_smallest = next_largest - 1
+    smaller = values[next_smallest]
+    larger = values[next_largest]
+    if abs(smaller - x) < abs(larger - x):
+        return smaller
+    else:
+        return larger
+
+
+def binary_search(a, x, lo=0, hi=None):
+    hi = hi if hi is not None else len(a)  # hi defaults to len(a)
+
+    pos = bisect_left(a, x, lo, hi)  # find insertion position
+
+    return pos if pos != hi and a[pos] == x else -1  # don't walk off the end
+
+
+def ceil_to_closest_value(x, values):
+    values = sorted(values)
+    next_largest = bisect_left(values, x)  # binary search
+    if next_largest < len(values):
+        return values[next_largest]
+    else:
+        return values[-1]  # if there is no larger value use the largest one
+
+
+def upset_binomial(mu, p, factor):
+    if factor > 1:
+        raise NotImplementedError()
+    return (binomial(mu / p, p) - mu) * factor + mu
+
+
+def multinomial(n, bins):
+    if bins == 0:
+        if n > 0:
+            raise ValueError('Cannot distribute to 0 bins.')
+        return []
+    remaining = n
+    results = []
+    for i in range(bins - 1):
+        x = binomial(remaining, 1 / (bins - i))
+        results.append(x)
+        remaining -= x
+
+    results.append(remaining)
+    return results
+
+
+def round_to_n(x, n):
+    return round(x, -int(floor(log10(x))) + (n - 1))
+
+
+def get_all_subclasses(klass):
+    all_subclasses = []
+
+    for subclass in klass.__subclasses__():
+        all_subclasses.append(subclass)
+        all_subclasses.extend(get_all_subclasses(subclass))
+
+    return all_subclasses
+
+
+def latin1_json(data):
+    return json.dumps(data, ensure_ascii=False).encode('latin-1')
+
+
+def l2_norm(v1, v2):
+    if len(v1) != len(v2):
+        raise ValueError('Both vectors must be of the same size')
+    return sqrt(sum([(x1 - x2) * (x1 - x2) for x1, x2 in zip(v1, v2)]))
+
+
+def allow_additional_unused_keyword_arguments(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        allowed_kwargs = [param.name for param in inspect.signature(func).parameters.values()]
+        allowed_kwargs = {a: kwargs[a] for a in kwargs if a in allowed_kwargs}
+        return func(*args, **allowed_kwargs)
+
+    return wrapper
+
+
+def rename(new_name):
+    def decorator(f):
+        f.__name__ = new_name
+        return f
+
+    return decorator
+
+
+def mean_confidence_interval_size(data, confidence=0.95):
+    if len(data) == 0:
+        return nan
+    if len(data) == 1:
+        return inf
+    if scipy.stats.sem(data) == 0:
+        return 0
+    return len(data) / sum(data) - scipy.stats.t.interval(confidence, len(data) - 1,
+                                                          loc=len(data) / sum(data),
+                                                          scale=scipy.stats.sem(data))[0]
+
+
+class LogicError(Exception):
+    pass
+
+
+def round_time(dt=None, precision=60):
+    """Round a datetime object to any time lapse in seconds
+    dt : datetime.datetime object, default now.
+    roundTo : Closest number of seconds to round to, default 1 minute.
+    Author: Thierry Husson 2012 - Use it as you want but don't blame me.
+    """
+    if dt is None:
+        dt = datetime.now()
+    if isinstance(precision, timedelta):
+        precision = precision.total_seconds()
+    seconds = (dt.replace(tzinfo=None) - dt.min).seconds
+    rounding = (seconds + precision / 2) // precision * precision
+    return dt + timedelta(seconds=rounding - seconds,
+                          microseconds=dt.microsecond)
+
+
+def profile_wall_time_instead_if_profiling():
+    try:
+        import yappi
+    except ModuleNotFoundError:
+        return
+    currently_profiling = len(yappi.get_func_stats())
+    if currently_profiling and yappi.get_clock_type() != 'wall':
+        print('Changing yappi clock type to wall and restarting yappi.')
+        yappi.stop()
+        yappi.clear_stats()
+        yappi.set_clock_type("wall")
+        yappi.start()
+
+
+def dummy_computation(_data):
+    return
+
+
+def current_year_begin():
+    return datetime(datetime.today().year, 1, 1).timestamp()
+
+
+def current_day_begin():
+    return datetime.today().timestamp() // (3600 * 24) * (3600 * 24)
+
+
+def current_second_begin():
+    return floor(datetime.today().timestamp())