initial commit of non-prototype

23 files changed, 1675 insertions, 424 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f3b938a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+/*.db
diff --git a/plan.org b/plan.org
new file mode 100644
index 0000000..4bb52f1
--- /dev/null
+++ b/plan.org
@@ -0,0 +1,40 @@
+* running info
+** port 7066
+** http via flask
+** endpoint specified by http point instead of a method param
+* required endpoints
+** old scheme
+{
+    "check_auth":      ["user", "auth_hash"],
+    "is_registered":   ["target_user"],
+    "is_admin":        ["target_user"],
+    "thread_index":    [],
+    "thread_load":     ["thread_id"],
+    "thread_create":   ["title", "body"],
+    "thread_reply":    ["thread_id", "body"],
+    "edit_post":       ["thread_id", "post_id", "body"],
+    "edit_query":      ["thread_id", "post_id"],
+    "can_edit":        ["thread_id", "post_id"],
+    "user_register":   ["user", "auth_hash", "quip", "bio"],
+    "user_get":        ["target_user"],
+    "user_name_to_id": ["target_user"]
+}
+** checking auth for user/hash pair (bool)
+** checking if a user is registered (bool)
+** checking is a user is an admin (bool)
+** index of all threads sans replies
+** load a thread by id
+** create a thread
+** reply to a thread
+** edit a post within 24hrs (unless admin)
+** query a postid with the current auth info to see if elegible (bool)
+** register a new user into the system
+** retrieve a full user object by name or ID
+** retrieve a username for ID and vice versa
+* authorization
+** SHA256 passhash and username in headers
+* what to do differently
+** concurrency, sql oriented from the beginning
+** dont tell anyone about it until its done
+** http instead of socket ports
+** start with formatting in mind, store it in plain and render accordingly on the fly
diff --git a/clients/elisp/bbj.el b/prototype/clients/elisp/bbj.el
index 86c1fb5..8e71b25 100644
--- a/clients/elisp/bbj.el
+++ b/prototype/clients/elisp/bbj.el
@@ -62,6 +62,7 @@
   (local-set-key (kbd "c")       'bbj-compose)
 
   (local-set-key (kbd "C-h SPC") 'bbj-pop-help)
+  (local-set-key (kbd "?")       'bbj-pop-help)
   (local-set-key (kbd "e")       'bbj-edit-post)
   (local-set-key (kbd "C-c C-c") 'bbj-aux)
   (local-set-key (kbd "r")       'bbj-quote-current-post))
diff --git a/prototype/clients/network_client.py b/prototype/clients/network_client.py
new file mode 100644
index 0000000..64e5e8d
--- /dev/null
+++ b/prototype/clients/network_client.py
@@ -0,0 +1,64 @@
+from hashlib import sha256
+import socket
+import json
+
+
+class BBJ:
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+        self.username  = None
+        self.auth_hash = None
+
+
+    def __call__(self, method, **params):
+        return self.request(method, **params)
+
+
+    def setuser(self, username, unhashed_password):
+        self.auth_hash = sha256(bytes(unhashed_password, "utf8")).hexdigest()
+        self.username = username
+        return self.auth_hash
+
+
+    def request(self, method, **params):
+        params["method"] = method
+
+        if not params.get("user") and self.username:
+            params["user"] = self.username
+
+        if not params.get("auth_hash") and self.auth_hash:
+            params["auth_hash"] = self.auth_hash
+
+
+        connection = socket.create_connection((self.host, self.port))
+        connection.sendall(bytes(json.dumps(params), "utf8"))
+        connection.shutdown(socket.SHUT_WR)
+
+        try:
+            buff, length = bytes(), 1
+            while length != 0:
+                recv = connection.recv(2048)
+                length = len(recv)
+                buff += recv
+
+        finally:
+            connection.close()
+
+        response = json.loads(str(buff, "utf8"))
+        if not isinstance(response, dict):
+            return response
+
+        error = response.get("error")
+        if not error:
+            return response
+
+        code, desc = error["code"], error["description"]
+
+        # tfw no qt3.14 python case switches
+        if error in (0, 1):
+            raise ChildProcessError("internal server error: " + desc)
+        elif error in (2, 3):
+            raise ChildProcessError(desc)
+
+        return response
diff --git a/prototype/clients/urwid/main.py b/prototype/clients/urwid/main.py
new file mode 100644
index 0000000..d29bcac
--- /dev/null
+++ b/prototype/clients/urwid/main.py
@@ -0,0 +1,88 @@
+from src import network
+
+
+bbj = network.BBJ("192.168.1.137", 7066)
+
+
+def geterr(obj):
+    """
+    Returns false if there are no errors in a network response,
+    else a tuple of (code integer, description string)
+    """
+    error = obj.get("error")
+    if not error:
+        return False
+    return (error["code"], error["description"])
+
+
+def register_prompt(user, initial=True):
+    if initial:
+        print("Register for BBJ as {}?".format(user))
+        reply = input("(y[es], d[ifferent name], q[uit])> ").lower()
+
+        if reply.startswith("d"):
+            register_prompt(input("(Username)> "))
+        elif reply.startswith("q"):
+            exit("bye!")
+
+        def getpass(ok):
+            p1 = input(
+                "(Choose a password)> " if ok else \
+                "(Those didn't match. Try again)> ")
+            p2 = input("(Now type it one more time)> ")
+            return p1 if p1 == p2 else getpass(False)
+
+        # this method will sha256 it for us
+        bbj.setuser(user, getpass(True))
+
+    response = bbj("user_register", quip="", bio="")
+    error = geterr(response)
+    if error:
+        exit("Registration error: " + error[1])
+    return response
+
+
+def login(user, ok=True):
+    if not bbj("is_registered", target_user=user):
+        register_prompt(user)
+    else:
+        bbj.setuser(user, input(
+            "(Password)> " if ok else \
+            "(Invalid password, try again)> "))
+
+    if not bbj("check_auth"):
+        login(user, ok=False)
+
+    return bbj("user_get", target_user=user)
+
+
+
+
+
+
+
+# user = input("(BBJ Username)> ")
+# if not bbj("is_registered", target_user=user):
+
+
+login(input("(Username)> "))
+
+import urwid
+
+f = urwid.Frame(
+    urwid.ListBox(
+        urwid.SimpleFocusListWalker(
+            [urwid.Text(i["body"]) for i in bbj("thread_index")["threads"]]
+        )
+    )
+)
+
+t = urwid.Overlay(
+    f, urwid.SolidFill('!'),
+    align='center',
+    width=('relative', 80),
+    height=('relative', 80),
+    valign='middle'
+)
+
+loop = urwid.MainLoop(t)
diff --git a/prototype/clients/urwid/src/network.py b/prototype/clients/urwid/src/network.py
new file mode 120000
index 0000000..62f1ad5
--- /dev/null
+++ b/prototype/clients/urwid/src/network.py
@@ -0,0 +1 @@
+../../network_client.py
\ No newline at end of file
diff --git a/prototype/clients/urwid/src/widgets.py b/prototype/clients/urwid/src/widgets.py
new file mode 100644
index 0000000..aa35748
--- /dev/null
+++ b/prototype/clients/urwid/src/widgets.py
@@ -0,0 +1,4 @@
+import urwid
+
+class PostBox(urwid.ListBox):
+    pass
diff --git a/docs/protocol.org b/prototype/docs/protocol.org
index 1a4a319..1a4a319 100644
--- a/docs/protocol.org
+++ b/prototype/docs/protocol.org
diff --git a/main.py b/prototype/main.py
index 78a55a6..78a55a6 100644
--- a/main.py
+++ b/prototype/main.py
diff --git a/prototype/src/db.py b/prototype/src/db.py
new file mode 100644
index 0000000..0cbaf5a
--- /dev/null
+++ b/prototype/src/db.py
@@ -0,0 +1,240 @@
+from src import formatting
+from uuid import uuid1
+from src import schema
+from time import time
+from os import path
+import json
+
+PATH = "/home/desvox/bbj/"
+
+if not path.isdir(PATH):
+    path.os.mkdir(PATH, mode=0o744)
+
+if not path.isdir(path.join(PATH, "threads")):
+    path.os.mkdir(path.join(PATH, "threads"), mode=0o744)
+
+try:
+    with open(path.join(PATH, "userdb"), "r") as f:
+        USERDB = json.loads(f.read())
+
+except FileNotFoundError:
+    USERDB = dict(namemap=dict())
+    with open(path.join(PATH, "userdb"), "w") as f:
+        f.write(json.dumps(USERDB))
+    path.os.chmod(path.join(PATH, "userdb"), 0o600)
+
+
+### THREAD MANAGEMENT ###
+
+def thread_index(key="lastmod", markup=True):
+    result = list()
+    for ID in path.os.listdir(path.join(PATH, "threads")):
+        thread = thread_load(ID, markup)
+        thread.pop("replies")
+        result.append(thread)
+    return sorted(result, key=lambda i: i[key], reverse=True)
+
+
+def thread_create(author, body, title, tags):
+    ID = uuid1().hex
+    if tags:
+        tags = [tag.strip() for tag in tags.split(",")]
+    else: # make sure None, False, and empty arrays are always repped consistently
+        tags = []
+    scheme = schema.thread(ID, author, body, title, tags)
+    thread_dump(ID, scheme)
+    return scheme
+
+
+def thread_load(ID, markup=True):
+    try:
+        with open(path.join(PATH, "threads", ID), "r") as f:
+            return json.loads(f.read())
+    except FileNotFoundError:
+        return False
+
+
+def thread_dump(ID, obj):
+    with open(path.join(PATH, "threads", ID), "w") as f:
+        f.write(json.dumps(obj))
+
+
+def thread_reply(ID, author, body):
+    thread = thread_load(ID)
+    if not thread:
+        return schema.error(7, "Requested thread does not exist.")
+
+    thread["reply_count"] += 1
+    thread["lastmod"] = time()
+
+    if thread["replies"]:
+        lastpost = thread["replies"][-1]["post_id"]
+    else:
+        lastpost = 1
+
+    reply = schema.reply(lastpost + 1, author, body)
+    thread["replies"].append(reply)
+    thread_dump(ID, thread)
+    return reply
+
+
+def index_reply(reply_list, post_id):
+    for index, reply in enumerate(reply_list):
+        if reply["post_id"] == post_id:
+            return index
+    else:
+        raise IndexError
+
+
+def edit_handler(json, thread=None):
+    try:
+        target_id = json["post_id"]
+        if not thread:
+            thread = thread_load(json["thread_id"])
+            if not thread:
+                return False, schema.error(7, "Requested thread does not exist.")
+
+
+        if target_id == 1:
+            target = thread
+        else:
+            target = thread["replies"][
+                index_reply(thread["replies"], target_id)]
+
+        if not user_is_admin(json["user"]):
+            if json["user"] != target["author"]:
+                return False, schema.error(10,
+                    "non-admin attempt to edit another user's message")
+
+            elif (time() - target["created"]) > 86400:
+                return False, schema.error(9,
+                    "message is too old to edit (24hr limit)")
+
+        return True, target
+
+    except IndexError:
+        return False, schema.error(3, "post_id out of bounds for requested thread")
+
+
+### USER MANAGEMENT ###
+
+def user_dbdump(dictionary):
+    with open(path.join(PATH, "userdb"), "w") as f:
+        f.write(json.dumps(dictionary))
+
+
+def user_resolve(name_or_id):
+    check = USERDB.get(name_or_id)
+    try:
+        if check:
+            return name_or_id
+        else:
+            return USERDB["namemap"][name_or_id]
+    except KeyError:
+        return False
+
+
+def user_register(auth_hash, name, quip, bio):
+    if USERDB["namemap"].get(name):
+        return schema.error(4, "Username taken.")
+
+    for ok, error in [
+        user_namecheck(name),
+        user_authcheck(auth_hash),
+        user_quipcheck(quip),
+        user_biocheck(bio)]:
+
+        if not ok:
+            return error
+
+    ID = uuid1().hex
+    scheme = schema.user_internal(ID, auth_hash, name, quip, bio, False)
+    USERDB.update({ID: scheme})
+    USERDB["namemap"].update({name: ID})
+    user_dbdump(USERDB)
+    return scheme
+
+
+def user_get(ID):
+    user = USERDB[ID]
+    return schema.user_external(
+        ID, user["name"], user["quip"],
+        user["bio"], user["admin"])
+
+
+def user_auth(ID, auth_hash):
+    return auth_hash == USERDB[ID]["auth_hash"]
+
+
+def user_is_admin(ID):
+    return USERDB[ID]["admin"]
+
+
+def user_update(ID, **params):
+    USERDB[ID].update(params)
+    return USERDB[ID]
+
+
+### SANITY CHECKS ###
+
+def contains_nonspaces(string):
+    return any([char in string for char in "\t\n\r\x0b\x0c"])
+
+
+def user_namecheck(name):
+    if not name:
+        return False, schema.error(4,
+            "Username may not be empty.")
+
+    elif contains_nonspaces(name):
+        return False, schema.error(4,
+            "Username cannot contain whitespace chars besides spaces.")
+
+    elif not name.strip():
+        return False, schema.error(4,
+            "Username must contain at least one non-space character")
+
+
+    elif len(name) > 24:
+        return False, schema.error(4,
+            "Username is too long (max 24 chars)")
+
+    return True, True
+
+
+def user_authcheck(auth_hash):
+    if not auth_hash:
+        return False, schema.error(3,
+            "auth_hash may not be empty")
+
+    elif len(auth_hash) != 64:
+        return False, schema.error(4,
+            "Client error: invalid SHA-256 hash.")
+
+    return True, True
+
+
+def user_quipcheck(quip):
+    if not quip:
+        return True, True
+
+    elif contains_nonspaces(quip):
+        return False, schema.error(4,
+            "Quip cannot contain whitespace chars besides spaces.")
+
+    elif len(quip) > 120:
+        return False, schema.error(4,
+            "Quip is too long (max 120 chars)")
+
+    return True, True
+
+
+def user_biocheck(bio):
+    if not bio:
+        return True, True
+
+    elif len(bio) > 4096:
+        return False, schema.error(4,
+            "Bio is too long (max 4096 chars)")
+
+    return True, True
diff --git a/prototype/src/endpoints.py b/prototype/src/endpoints.py
new file mode 100644
index 0000000..973a528
--- /dev/null
+++ b/prototype/src/endpoints.py
@@ -0,0 +1,174 @@
+from src import formatting
+from src import schema
+from time import time
+from src import db
+
+
+endpoints = {
+    "check_auth":      ["user", "auth_hash"],
+    "is_registered":   ["target_user"],
+    "is_admin":        ["target_user"],
+    "thread_index":    [],
+    "thread_load":     ["thread_id"],
+    "thread_create":   ["title", "body", "tags"],
+    "thread_reply":    ["thread_id", "body"],
+    "edit_post":       ["thread_id", "post_id", "body"],
+    "edit_query":      ["thread_id", "post_id"],
+    "can_edit":        ["thread_id", "post_id"],
+    "user_register":   ["user", "auth_hash", "quip", "bio"],
+    "user_get":        ["target_user"],
+    "user_name_to_id": ["target_user"]
+}
+
+
+authless = [
+    "is_registered",
+    "user_register"
+]
+
+
+# this is not actually an endpoint, but produces a required
+# element of thread responses.
+def create_usermap(thread, index=False):
+    if index:
+        return {user: db.user_get(user) for user in
+                  {i["author"] for i in thread}}
+
+    result = {reply["author"] for reply in thread["replies"]}
+    result.add(thread["author"])
+    return {x: db.user_get(x) for x in result}
+
+
+def user_name_to_id(json):
+    """
+    Returns a string of the target_user's ID when it is
+    part of the database: a non-existent user will return
+    a boolean false.
+    """
+    return db.user_resolve(json["target_user"])
+
+
+def is_registered(json):
+    """
+    Returns true or false whether target_user is registered
+    in the system. This function only takes usernames: not
+    user IDs.
+    """
+    return bool(db.USERDB["namemap"].get(json["target_user"]))
+
+
+def check_auth(json):
+    "Returns true or false whether auth_hashes matches user."
+    return bool(db.user_auth(json["user"], json["auth_hash"]))
+
+
+def is_admin(json):
+    """
+    Returns true or false whether target_user is a system
+    administrator. Takes a username or user ID. Nonexistent
+    users return false.
+    """
+    user = db.user_resolve(json["target_user"])
+    if user:
+        return db.user_is_admin(user)
+    return False
+
+
+def user_register(json):
+    """
+    Registers a new user into the system. Returns the new internal user
+    object on success, or an error response.
+
+    auth_hash should be a hexadecimal SHA-256 string, produced from a
+      UTF-8 password string.
+
+    user should be a string containing no newlines and
+      under 24 characters in length.
+
+    quip is a string, up to 120 characters, provided by the user
+      the acts as small bio, suitable for display next to posts
+      if the client wants to. Whitespace characters besides space
+      are not allowed. The string may be empty.
+
+    bio is a string, up to 4096 chars, provided by the user that
+      can be shown on profiles. There are no character type limits
+      for this entry. The string may be empty.
+
+    All errors for this endpoint with code 4 should show the
+    description direcrtly to the user.
+
+    """
+
+    return schema.response(
+        db.user_register(
+            json["auth_hash"],
+            json["user"],
+            json["quip"],
+            json["bio"]))
+
+
+def user_get(json):
+    """
+    On success, returns an external user object for target_user (ID or name).
+    If the user isn't in the system, returns false.
+    """
+    user = db.user_resolve(json["target_user"])
+    if not user:
+        return False
+    return db.user_get(user)
+
+
+def thread_index(json):
+    index = db.thread_index(markup=not json.get("nomarkup"))
+    return schema.response({"threads": index}, create_usermap(index, True))
+
+
+def thread_load(json):
+    thread = db.thread_load(json["thread_id"], not json.get("nomarkup"))
+    if not thread:
+        return schema.error(7, "Requested thread does not exist")
+    return schema.response(thread, create_usermap(thread))
+
+
+def thread_create(json):
+    thread = db.thread_create(
+        json["user"],
+        json["body"],
+        json["title"],
+        json["tags"])
+    return schema.response(thread)
+
+
+def thread_reply(json):
+    reply = db.thread_reply(
+        json["thread_id"],
+        json["user"],
+        json["body"])
+    return schema.response(reply)
+
+
+def edit_query(json):
+    return db.edit_handler(json)[1]
+
+
+def can_edit(json):
+    return db.edit_handler(json)[0]
+
+
+def edit_post(json):
+    thread = db.thread_load(json["thread_id"])
+    admin = db.user_is_admin(json["user"])
+    target_id = json["post_id"]
+    ok, obj = db.edit_handler(json, thread)
+
+    if ok:
+
+        if json.get("reformat"):
+            json["body"] = formatting.parse(json["body"])
+
+        obj["body"] = json["body"]
+        obj["lastmod"] = time()
+        obj["edited"] = True
+        db.thread_dump(json["thread_id"], thread)
+
+    return obj
diff --git a/prototype/src/formatting.py b/prototype/src/formatting.py
new file mode 100644
index 0000000..56614c3
--- /dev/null
+++ b/prototype/src/formatting.py
@@ -0,0 +1,28 @@
+from markdown import markdown
+from html import escape
+import re
+
+
+COLORS = ["red", "green", "yellow", "blue", "magenta", "cyan"]
+MARKUP = ["bold", "italic", "underline", "strike"]
+TOKENS = re.compile(r"\[({}): (.+?)]".format("|".join(COLORS + MARKUP)), flags=re.DOTALL)
+QUOTES = re.compile(">>([0-9]+)")
+LINEQUOTES = re.compile("^(>.+)$", flags=re.MULTILINE)
+
+
+def map_html(match):
+    directive, body = match.group(1).lower(), match.group(2)
+    if directive in COLORS:
+        return '<span color="{0}" style="color: {0};">{1}</span>'.format(directive, body)
+    elif directive in MARKUP:
+        return '<{0}>{1}</{0}>'.format(directive[0], body)
+    return body
+
+
+def parse(text, doquotes=True):
+    text = TOKENS.sub(map_html, escape(text))
+    if doquotes:
+        text = QUOTES.sub(r'<span post="\1" class="quote">\g<0></span>', text)
+    return markdown(
+        LINEQUOTES.sub(r'<span class="linequote">\1</span><br>', text)
+    )
diff --git a/prototype/src/schema.py b/prototype/src/schema.py
new file mode 100644
index 0000000..8e3ca35
--- /dev/null
+++ b/prototype/src/schema.py
@@ -0,0 +1,97 @@
+from src import formatting
+from time import time
+
+
+def base():
+    return {
+        "error": False
+    }
+
+
+def response(dictionary, usermap=None):
+    result = base()
+    result.update(dictionary)
+    if usermap:
+        result["usermap"] = usermap
+    return result
+
+
+def error(code, description):
+    result = base()
+    result.update({
+        "error": {
+            "description": description, # string
+            "code": code                # integer
+        }
+    })
+    return result
+
+
+def user_internal(ID, auth_hash, name, quip, bio, admin):
+    if not quip:
+        quip = ""
+
+    if not bio:
+        bio = ""
+
+    return {
+        "user_id":   ID,       # string
+        "quip":      quip,     # (possibly empty) string
+        "bio":       bio,      # (possibly empty) string
+        "name":      name,     # string
+        "admin":     admin,    # boolean
+        "auth_hash": auth_hash # SHA256 string
+    }
+
+
+def user_external(ID, name, quip, bio, admin):
+    if not quip:
+        quip = ""
+
+    if not bio:
+        bio = ""
+
+    return {
+        "user_id":   ID,   # string
+        "quip":      quip, # (possibly empty) string
+        "name":      name, # string
+        "bio":       bio,  # string
+        "admin":     admin # boolean
+    }
+
+
+def thread(ID, author, body, title, tags):
+    if not tags:
+        tags = list()
+
+    body = formatting.parse(body, doquotes=False)
+    now = time()
+
+    return {
+        "thread_id":   ID,     # string
+        "post_id":     1,      # integer
+        "author":      author, # string
+        "body":        body,   # string
+        "title":       title,  # string
+        "tags":        tags,   # (possibly empty) list of strings
+        "replies":     list(), # (possibly empty) list of reply objects
+        "reply_count": 0,      # integer
+        "edited":      False,  # boolean
+        "lastmod":     now,    # floating point unix timestamp
+        "created":     now     # floating point unix timestamp
+    }
+
+
+def reply(ID, author, body):
+
+    body = formatting.parse(body)
+    now = time()
+
+    return {
+        "post_id":  ID,     # integer
+        "author":   author, # string
+        "body":     body,   # string
+        "edited":   False,  # boolean
+        "lastmod":  now,    # floating point unix timestamp
+        "created":  now     # floating point unix timestamp
+    }
diff --git a/src/server.py b/prototype/src/server.py
index 345cc68..345cc68 100644
--- a/src/server.py
+++ b/prototype/src/server.py
diff --git a/schema.sql b/schema.sql
new file mode 100644
index 0000000..8afa657
--- /dev/null
+++ b/schema.sql
@@ -0,0 +1,35 @@
+drop table if exists users;
+drop table if exists threads;
+drop table if exists messages;
+
+
+create table users (
+  user_id text,     -- string (uuid1)
+  user_name text,   -- string
+  auth_hash text,   -- string (sha256 hash)
+  quip text,        -- string (possibly empty)
+  bio text,         -- string (possibly empty)
+  color int,        -- int (from 0 to 8)
+  is_admin int,     -- bool
+  created real      -- floating point unix timestamp (when this user registered)
+);
+
+
+create table threads (
+  thread_id text,   -- uuid string
+  author text,      -- string (uuid1, user.user_id)
+  title text,       -- string
+  last_mod real,    -- floating point unix timestamp (of last post or post edit)
+  created real,     -- floating point unix timestamp (when thread was made)
+  reply_count int   -- integer (incremental, starting with 0)
+);
+
+
+create table messages (
+  thread_id text,   -- string (uuid1 of parent thread)
+  post_id int,      -- integer (incrementing from 1)
+  author text,      -- string (uuid1, user.user_id)
+  created real,     -- floating point unix timestamp (when reply was posted)
+  edited int,       -- bool
+  body text         -- string
+);
diff --git a/server.py b/server.py
new file mode 100644
index 0000000..957d23b
--- /dev/null
+++ b/server.py
@@ -0,0 +1,220 @@
+from src.exceptions import BBJException, BBJParameterError
+from src import db, schema, endpoints
+from functools import wraps
+import cherrypy
+import sqlite3
+import json
+
+dbname = "data.sqlite"
+
+with sqlite3.connect(dbname) as _c:
+    if not db.user_resolve(_c, "anonymous"):
+        db.user_register(_c, *db.anonymous)
+
+
+# creates a database connection for each thread
+def connect(_):
+    cherrypy.thread_data.db = sqlite3.connect(dbname)
+cherrypy.engine.subscribe('start_thread', connect)
+
+
+def bbjapi(function):
+    """
+    A wrapper that handles encoding of objects and errors to a
+    standard format for the API, resolves and authorizes users
+    from header data, and prepares thread data to handle the
+    request.
+
+    In addition, all BBJException's will return their attached
+    schema, and unhandled exceptions return a code 1 error schema.
+    """
+    @wraps(function)
+    def wrapper(*args, **kwargs):
+        headers = cherrypy.request.headers
+        username = headers.get("User")
+        auth = headers.get("Auth")
+        anon = False
+
+        if not username and not auth:
+            user = db.user_resolve(cherrypy.thread_data.db, "anonymous")
+            anon = True
+        elif not username or not auth:
+            return json.dumps(schema.error(5,
+                "User or Auth was given without the other."))
+
+        if not anon:
+            user = db.user_resolve(cherrypy.thread_data.db, username)
+            if not user:
+                return json.dumps(schema.error(4,
+                    "Username is not registered."))
+
+            elif auth != user["auth_hash"]:
+                return json.dumps(schema.error(5,
+                    "Invalid authorization key for user."))
+
+        cherrypy.thread_data.user = user
+        cherrypy.thread_data.anon = anon
+
+        try:
+            value = function(*args, **kwargs)
+
+        except BBJException as e:
+            value = e.schema
+
+        except Exception as e:
+            value = schema.error(1, str(e))
+
+        return json.dumps(value)
+    return wrapper
+
+
+def create_usermap(connection, obj):
+    """
+    Creates a mapping of all the user_ids that occur in OBJ to
+    their full user objects (names, profile info, etc). Can
+    be a thread_index or a messages object from one.
+    """
+
+    if isinstance(obj, dict):
+        # this is a message object for a thread, unravel it
+        obj = [value for key, value in obj.items()]
+
+    return {
+        user_id: db.user_resolve(
+            connection,
+            user_id,
+            externalize=True,
+            return_false=False)
+        for user_id in {
+                item["author"] for item in obj
+        }
+    }
+
+
+
+def validate(json, args):
+    """
+    Ensure the json object contains all the keys needed to satisfy
+    its endpoint.
+    """
+    for arg in args:
+        if arg not in json.keys():
+            raise BBJParameterError(
+                "Required parameter %s is "
+                "absent from the request." % arg)
+
+
+APICONFIG = {
+    "/": {
+        "tools.response_headers.on": True,
+        "tools.response_headers.headers": [
+            ("Content-Type", "application/json")
+        ],
+    }
+}
+
+class API(object):
+    @bbjapi
+    @cherrypy.expose
+    def thread_index(self):
+        threads = db.thread_index(cherrypy.thread_data.db)
+        usermap = create_usermap(cherrypy.thread_data.db, threads)
+        return schema.response({
+            "data": threads,
+            "usermap": usermap
+        })
+
+
+    @bbjapi
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    def thread_create(self):
+        args = cherrypy.request.json
+        validate(args, ["body", "title"])
+
+        thread = db.thread_create(
+            cherrypy.thread_data.db,
+            cherrypy.thread_data.user["user_id"],
+            args["body"], args["title"])
+
+        usermap = {
+            cherrypy.thread_data.user["user_id"]:
+              cherrypy.thread_data.user
+        }
+
+        return schema.response({
+            "data": thread,
+            "usermap": usermap
+        })
+
+
+    @bbjapi
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    def thread_reply(self):
+        args = cherrypy.request.json
+        validate(args, ["thread_id", "body"])
+        return schema.response({
+            "data": db.thread_reply(
+                cherrypy.thread_data.db,
+                cherrypy.thread_data.user["user_id"],
+                args["thread_id"], args["body"])
+        })
+
+
+    @bbjapi
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    def thread_load(self):
+        args = cherrypy.request.json
+        validate(args, ["thread_id"])
+
+        thread = db.thread_get(
+            cherrypy.thread_data.db,
+            args["thread_id"])
+
+        usermap = create_usermap(
+            cherrypy.thread_data.db,
+            thread["messages"])
+
+        return schema.response({
+            "data": thread,
+            "usermap": usermap
+        })
+
+
+    @bbjapi
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    def user_register(self):
+        args = cherrypy.request.json
+        validate(args, ["user_name", "auth_hash"])
+        return schema.response({
+            "data": db.user_register(
+                cherrypy.thread_data.db,
+                args["user_name"], args["auth_hash"])
+        })
+
+
+    @bbjapi
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    def edit_query(self):
+        args = cherrypy.request.json
+        validate(args, ["thread_id", "post_id"])
+        return schema.response({
+            "data": message_edit_query(
+                cherrypy.thread_data.db,
+                cherrypy.thread_data.user["user_id"],
+                args["thread_id"],
+                args["post_id"])
+        })
+
+
+
+def run():
+    cherrypy.quickstart(API(), "/api")
+
+
+if __name__ == "__main__":
+    print("wew")
diff --git a/setup.sh b/setup.sh
new file mode 100755
index 0000000..9d508bc
--- /dev/null
+++ b/setup.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+if [[ $1 == --init ]]; then
+	sqlite3 bbj.db < schema.sql
+	echo cleared
+	exit
+fi
+
+DEPS=(
+    cherrypy
+    markdown
+)
+
+if [[ -z $1 ]]; then
+    cat << EOF
+Pass the python interpreter to use for pip installation
+(either a venv or a system interpreter)
+EOF
+    exit
+fi
+
+$1 -m pip install ${DEPS[*]}
+
+echo "Enter [i] to initialize a new database"
+read CLEAR
+[[ $CLEAR == "i" ]] && sqlite3 bbj.db < schema.sql
diff --git a/src/db.py b/src/db.py
index 0cbaf5a..93e37e6 100644
--- a/src/db.py
+++ b/src/db.py
@@ -1,178 +1,274 @@
-from src import formatting
-from uuid import uuid1
+"""
+This module contains all of the interaction with the SQLite database. It
+doesnt hold a connection itself, rather, a connection is passed in as
+an argument to all the functions and is maintained by CherryPy's threading
+system. This is clunky but fuck it, it works.
+
+All post and thread data are stored in the database without formatting.
+This is questionable, as it causes formatting to be reapplied with each
+pull for the database. Im debating whether posts should be stored in all
+4 formats, or if maybe a caching system should be used.
+
+The database, nor ANY part of the server, DOES NOT HANDLE PASSWORD HASHING!
+Clients are responsible for creation of hashes and passwords should never
+be sent unhashed. User registration and update endpoints will not accept
+hashes that != 64 characters in length, as a basic measure to enforce the
+use of sha256.
+"""
+
+from src.exceptions import BBJParameterError, BBJUserError
+from src.utils import ordered_keys, schema_values
 from src import schema
+from uuid import uuid1
 from time import time
-from os import path
+import pickle
 import json
+import os
+
+anonymous = \
+    ("anonymous",
+     "5430eeed859cad61d925097ec4f53246"
+     "1ccf1ab6b9802b09a313be1478a4d614")
+     # this is the hash for "anon"
+
+# if os.path.exists("cache"):
+#     os.rmdir("cache")
+# os.mkdir("cache")
+
+### THREADS ###
+
+def thread_get(connection, thread_id, messages=True):
+    """
+    Fetch the thread_id from the database, and assign and format
+    all of its messages as requested.
+
+    MESSAGES, if False, will omit the inclusion of a thread's messages
+    and only get its metadata, such as title, author, etc.
+
+    FORMATTER should be a callable object who takes a body string
+    as it's only argument and returns an object to be sent in the
+    response. It isn't strictly necessary that it returns a string,
+    for example the entity parser will return an array with the
+    body string and another array with indices and types of objects
+    contained in it.
+    """
+    c = connection.cursor()
+    c.execute("SELECT * FROM threads WHERE thread_id = ?", (thread_id,))
+    thread = c.fetchone()
 
-PATH = "/home/desvox/bbj/"
-
-if not path.isdir(PATH):
-    path.os.mkdir(PATH, mode=0o744)
-
-if not path.isdir(path.join(PATH, "threads")):
-    path.os.mkdir(path.join(PATH, "threads"), mode=0o744)
+    if not thread:
+        raise BBJParameterError("Thread does not exist.")
+    thread = schema.thread(*thread)
+
+    if messages:
+        c.execute("SELECT * FROM messages WHERE thread_id = ?", (thread_id,))
+        # create a dictionary where each message is accessible by its
+        # integer post_id as a key
+        thread["messages"] = \
+            {message["post_id"]: message for message in
+              [schema.message(*values) for values in c.fetchall()]}
+
+    return thread
+
+
+def thread_index(connection):
+    c = connection.cursor()
+    c.execute("""
+    SELECT thread_id FROM threads
+    ORDER BY last_mod DESC""")
+    threads = [
+        thread_get(connection, obj[0], messages=False)
+            for obj in c.fetchall()
+    ]
+    return threads
+
+
+def thread_create(connection, author_id, body, title):
+    validate([
+        ("body",  body),
+        ("title", title)
+    ])
+
+    now = time()
+    thread_id = uuid1().hex
+    scheme = schema.thread(
+        thread_id, author_id, title,
+        now, now, -1) # see below for why i set -1 instead of 0
+
+    connection.cursor().execute("""
+        INSERT INTO threads
+        VALUES (?,?,?,?,?,?)
+    """, schema_values("thread", scheme))
+    connection.commit()
+
+    scheme["messages"] = {
+        0: thread_reply(connection, author_id, thread_id, body, time_override=now)
+    }
+    scheme["reply_count"] = 0
+    # note that thread_reply returns a schema object
+    # after committing the new message to the database.
+    # here i mimic a real thread_get by including a mock
+    # message dictionary, and then setting the reply_count
+    # to reflect its new database value, so the response
+    # can be loaded as a normal thread object
+    return scheme
 
-try:
-    with open(path.join(PATH, "userdb"), "r") as f:
-        USERDB = json.loads(f.read())
 
-except FileNotFoundError:
-    USERDB = dict(namemap=dict())
-    with open(path.join(PATH, "userdb"), "w") as f:
-        f.write(json.dumps(USERDB))
-    path.os.chmod(path.join(PATH, "userdb"), 0o600)
+def thread_reply(connection, author_id, thread_id, body, time_override=None):
+    validate([("body", body)])
 
+    now = time_override or time()
+    thread = thread_get(connection, thread_id, messages=False)
+    count = thread["reply_count"] + 1
+    scheme = schema.message(
+        thread_id, count, author_id,
+        now, False, body)
 
-### THREAD MANAGEMENT ###
+    c = connection.cursor()
 
-def thread_index(key="lastmod", markup=True):
-    result = list()
-    for ID in path.os.listdir(path.join(PATH, "threads")):
-        thread = thread_load(ID, markup)
-        thread.pop("replies")
-        result.append(thread)
-    return sorted(result, key=lambda i: i[key], reverse=True)
+    c.execute("""
+        INSERT INTO messages
+        VALUES (?,?,?,?,?,?)
+    """, schema_values("message", scheme))
 
+    c.execute("""
+        UPDATE threads SET
+        reply_count = ?,
+        last_mod = ?
+        WHERE thread_id = ?
+    """, (count, now, thread_id))
 
-def thread_create(author, body, title, tags):
-    ID = uuid1().hex
-    if tags:
-        tags = [tag.strip() for tag in tags.split(",")]
-    else: # make sure None, False, and empty arrays are always repped consistently
-        tags = []
-    scheme = schema.thread(ID, author, body, title, tags)
-    thread_dump(ID, scheme)
+    connection.commit()
     return scheme
 
 
-def thread_load(ID, markup=True):
-    try:
-        with open(path.join(PATH, "threads", ID), "r") as f:
-            return json.loads(f.read())
-    except FileNotFoundError:
-        return False
-
-
-def thread_dump(ID, obj):
-    with open(path.join(PATH, "threads", ID), "w") as f:
-        f.write(json.dumps(obj))
+def message_edit_query(connection, author, thread_id, post_id):
+    user = user_resolve(connection, author)
+    thread = thread_get(connection, thread_id)
 
+    try: message = thread["messages"][post_id]
+    except KeyError:
+        raise BBJParameterError("post_id out of bounds for requested thread")
 
-def thread_reply(ID, author, body):
-    thread = thread_load(ID)
-    if not thread:
-        return schema.error(7, "Requested thread does not exist.")
+    if not user["admin"]:
+        if not user["user_id"] == message["author"]:
+            raise BBJUserError(
+                "non-admin attempt to edit another user's message")
 
-    thread["reply_count"] += 1
-    thread["lastmod"] = time()
+        elif (time() - message["created"]) > 86400:
+            raise BBJUserError(
+                "message is too old to edit (24hr limit)")
 
-    if thread["replies"]:
-        lastpost = thread["replies"][-1]["post_id"]
-    else:
-        lastpost = 1
+    return message
 
-    reply = schema.reply(lastpost + 1, author, body)
-    thread["replies"].append(reply)
-    thread_dump(ID, thread)
-    return reply
 
+def message_edit_commit(connection, author_id, thread_id, post_id, new_body):
+    validate([("body", new_body)])
+    message = message_edit_query(author_id, thread_id, post_id)
+    message["body"] = new_body
+    message["edited"] = True
 
-def index_reply(reply_list, post_id):
-    for index, reply in enumerate(reply_list):
-        if reply["post_id"] == post_id:
-            return index
-    else:
-        raise IndexError
+    connection.cursor().excute("""
+        UPDATE messages SET
+        body = ? edited = ?
+        WHERE
+          thread_id = ? AND post_id = ?
+    """, (new_body, True, thread_id, post_id))
 
+    connection.commit()
+    return message
 
-def edit_handler(json, thread=None):
-    try:
-        target_id = json["post_id"]
-        if not thread:
-            thread = thread_load(json["thread_id"])
-            if not thread:
-                return False, schema.error(7, "Requested thread does not exist.")
 
+### USERS ####
 
-        if target_id == 1:
-            target = thread
-        else:
-            target = thread["replies"][
-                index_reply(thread["replies"], target_id)]
 
-        if not user_is_admin(json["user"]):
-            if json["user"] != target["author"]:
-                return False, schema.error(10,
-                    "non-admin attempt to edit another user's message")
+def user_register(connection, user_name, auth_hash):
+    """
+    Registers a new user into the system. Ensures the user
+    is not already registered, and that the hash and name
+    meet the requirements of their respective sanity checks
+    """
+    validate([
+        ("user_name", user_name),
+        ("auth_hash", auth_hash)
+    ])
 
-            elif (time() - target["created"]) > 86400:
-                return False, schema.error(9,
-                    "message is too old to edit (24hr limit)")
+    if user_resolve(connection, user_name):
+        raise BBJUserError("Username already registered")
 
-        return True, target
+    scheme = schema.user_internal(
+        uuid1().hex, user_name, auth_hash,
+        "", "", 0, False, time())
 
-    except IndexError:
-        return False, schema.error(3, "post_id out of bounds for requested thread")
+    connection.cursor().execute("""
+         INSERT INTO users
+         VALUES (?,?,?,?,?,?,?,?)
+    """, schema_values("user", scheme))
 
+    connection.commit()
+    return scheme
 
-### USER MANAGEMENT ###
 
-def user_dbdump(dictionary):
-    with open(path.join(PATH, "userdb"), "w") as f:
-        f.write(json.dumps(dictionary))
+def user_resolve(connection, name_or_id, externalize=False, return_false=True):
+    c = connection.cursor()
+    c.execute("""
+         SELECT * FROM users
+         WHERE user_name = ?
+            OR user_id = ?
+    """, (name_or_id, name_or_id))
 
+    user = c.fetchone()
+    if user:
+        user = schema.user_internal(*user)
+        if externalize:
+            return user_externalize(user)
+        return user
 
-def user_resolve(name_or_id):
-    check = USERDB.get(name_or_id)
-    try:
-        if check:
-            return name_or_id
-        else:
-            return USERDB["namemap"][name_or_id]
-    except KeyError:
+    if return_false:
         return False
+    raise BBJParameterError(
+        "Requested user element ({})"
+        " does not exist".format(name_or_id))
 
 
-def user_register(auth_hash, name, quip, bio):
-    if USERDB["namemap"].get(name):
-        return schema.error(4, "Username taken.")
-
-    for ok, error in [
-        user_namecheck(name),
-        user_authcheck(auth_hash),
-        user_quipcheck(quip),
-        user_biocheck(bio)]:
-
-        if not ok:
-            return error
-
-    ID = uuid1().hex
-    scheme = schema.user_internal(ID, auth_hash, name, quip, bio, False)
-    USERDB.update({ID: scheme})
-    USERDB["namemap"].update({name: ID})
-    user_dbdump(USERDB)
-    return scheme
-
+def user_update(connection, user_object, parameters):
+    user_id = user_object["user_id"]
+    for key in ("user_name", "auth_hash", "quip", "bio", "color"):
+        value = parameters.get(key)
+        if value:
+            validate([(key, value)])
+            user_object[key] = value
 
-def user_get(ID):
-    user = USERDB[ID]
-    return schema.user_external(
-        ID, user["name"], user["quip"],
-        user["bio"], user["admin"])
+    values = ordered_keys(user_object,
+        "user_name", "quip", "auth_hash",
+        "bio", "color", "user_id")
 
+    connection.cursor().execute("""
+        UPDATE users SET
+        user_name = ?, quip = ?,
+        auth_hash = ?, bio = ?,
+        color = ? WHERE user_id = ?
+        """, values)
 
-def user_auth(ID, auth_hash):
-    return auth_hash == USERDB[ID]["auth_hash"]
+    connection.commit()
+    return user_resolve(connection, user_id)
 
 
-def user_is_admin(ID):
-    return USERDB[ID]["admin"]
+def user_externalize(user_object):
+    """
+    Cleanse private/internal data from a user object
+    and make it suitable to serve.
+    """
+    # only secret value right now is the auth_hash,
+    # but this may change in the future
+    for key in ("auth_hash",):
+        user_object.pop(key)
+    return user_object
 
 
-def user_update(ID, **params):
-    USERDB[ID].update(params)
-    return USERDB[ID]
+def user_auth(auth_hash, user_object):
+    # nominating this for most useless function in the program
+    return auth_hash == user_object["auth_hash"]
 
 
 ### SANITY CHECKS ###
@@ -181,60 +277,225 @@ def contains_nonspaces(string):
     return any([char in string for char in "\t\n\r\x0b\x0c"])
 
 
-def user_namecheck(name):
-    if not name:
-        return False, schema.error(4,
-            "Username may not be empty.")
-
-    elif contains_nonspaces(name):
-        return False, schema.error(4,
-            "Username cannot contain whitespace chars besides spaces.")
-
-    elif not name.strip():
-        return False, schema.error(4,
-            "Username must contain at least one non-space character")
-
-
-    elif len(name) > 24:
-        return False, schema.error(4,
-            "Username is too long (max 24 chars)")
-
-    return True, True
-
-
-def user_authcheck(auth_hash):
-    if not auth_hash:
-        return False, schema.error(3,
-            "auth_hash may not be empty")
-
-    elif len(auth_hash) != 64:
-        return False, schema.error(4,
-            "Client error: invalid SHA-256 hash.")
-
-    return True, True
-
-
-def user_quipcheck(quip):
-    if not quip:
-        return True, True
-
-    elif contains_nonspaces(quip):
-        return False, schema.error(4,
-            "Quip cannot contain whitespace chars besides spaces.")
-
-    elif len(quip) > 120:
-        return False, schema.error(4,
-            "Quip is too long (max 120 chars)")
-
-    return True, True
-
-
-def user_biocheck(bio):
-    if not bio:
-        return True, True
-
-    elif len(bio) > 4096:
-        return False, schema.error(4,
-            "Bio is too long (max 4096 chars)")
-
-    return True, True
+def validate(keys_and_values):
+    """
+    The line of defense against garbage user input.
+
+    Recieves an iterable containing iterables, where [0]
+    is a string representing the value type, and [1]
+    is the value to compare against a set of rules for
+    it's type. The function returns the boolean value
+    True when everything is okay, or raises a BBJException
+    to be handled by higher levels of the program if something
+    is wrong (immediately stopping execution at the db level)
+    """
+    for key, value in keys_and_values:
+
+        if key == "user_name":
+            if not value:
+                raise BBJUserError(
+                    "Username may not be empty.")
+
+            elif contains_nonspaces(value):
+                raise BBJUserError(
+                    "Username cannot contain whitespace chars besides spaces.")
+
+            elif not value.strip():
+                raise BBJUserError(
+                    "Username must contain at least one non-space character")
+
+            elif len(value) > 24:
+                raise BBJUserError(
+                    "Username is too long (max 24 chars)")
+
+        elif key == "auth_hash":
+            if not value:
+                raise BBJParameterError(
+                    "auth_hash may not be empty")
+
+            elif len(value) != 64:
+                raise BBJParameterError(
+                    "Client error: invalid SHA-256 hash.")
+
+        elif key == "quip":
+            if contains_nonspaces(value):
+                raise BBJUserError(
+                    "Quip cannot contain whitespace chars besides spaces.")
+
+            elif len(value) > 120:
+                raise BBJUserError(
+                    "Quip is too long (max 120 chars)")
+
+        elif key == "bio":
+            if len(value) > 4096:
+                raise BBJUserError(
+                    "Bio is too long (max 4096 chars)")
+
+        elif key == "title":
+            if not value:
+                raise BBJUserError(
+                    "Title cannot be empty")
+
+            elif contains_nonspaces(value):
+                raise BBJUserError(
+                    "Titles cannot contain whitespace chars besides spaces.")
+
+            elif len(value) > 120:
+                raise BBJUserError(
+                    "Title is too long (max 120 chars)")
+
+        elif key == "body":
+            if not value:
+                raise BBJUserError(
+                    "Post body cannot be empty")
+
+
+        elif key == "color":
+            if color in range(0, 9):
+                continue
+            raise BBJParameterError(
+                "Color specification out of range (int 0-8)")
+
+    return True
+
+
+### OLD SHIT ###
+
+# def thread_index(key="lastmod", markup=True):
+#     result = list()
+#     for ID in path.os.listdir(path.join(PATH, "threads")):
+#         thread = thread_load(ID, markup)
+#         thread.pop("replies")
+#         result.append(thread)
+#     return sorted(result, key=lambda i: i[key], reverse=True)
+#
+#
+#
+#
+# def thread_load(ID, markup=True):
+#     try:
+#         with open(path.join(PATH, "threads", ID), "r") as f:
+#             return json.loads(f.read())
+#     except FileNotFoundError:
+#         return False
+#
+#
+# def thread_dump(ID, obj):
+#     with open(path.join(PATH, "threads", ID), "w") as f:
+#         f.write(json.dumps(obj))
+#
+#
+# def thread_reply(ID, author, body):
+#     thread = thread_load(ID)
+#     if not thread:
+#         return schema.error(7, "Requested thread does not exist.")
+#
+#     thread["reply_count"] += 1
+#     thread["lastmod"] = time()
+#
+#     if thread["replies"]:
+#         lastpost = thread["replies"][-1]["post_id"]
+#     else:
+#         lastpost = 1
+#
+#     reply = schema.reply(lastpost + 1, author, body)
+#     thread["replies"].append(reply)
+#     thread_dump(ID, thread)
+#     return reply
+#
+#
+# def index_reply(reply_list, post_id):
+#     for index, reply in enumerate(reply_list):
+#         if reply["post_id"] == post_id:
+#             return index
+#     else:
+#         raise IndexError
+#
+#
+# def edit_handler(json, thread=None):
+#     try:
+#         target_id = json["post_id"]
+#         if not thread:
+#             thread = thread_load(json["thread_id"])
+#             if not thread:
+#                 return False, schema.error(7, "Requested thread does not exist.")
+#
+#
+#         if target_id == 1:
+#             target = thread
+#         else:
+#             target = thread["replies"][
+#                 index_reply(thread["replies"], target_id)]
+#
+#         if not user_is_admin(json["user"]):
+#             if json["user"] != target["author"]:
+#                 return False, schema.error(10,
+#                     "non-admin attempt to edit another user's message")
+#
+#             elif (time() - target["created"]) > 86400:
+#                 return False, schema.error(9,
+#                     "message is too old to edit (24hr limit)")
+#
+#         return True, target
+#
+#     except IndexError:
+#         return False, schema.error(3, "post_id out of bounds for requested thread")
+#
+#
+# ### USER MANAGEMENT ###
+#
+# def user_dbdump(dictionary):
+#     with open(path.join(PATH, "userdb"), "w") as f:
+#         f.write(json.dumps(dictionary))
+#
+#
+# def user_resolve(name_or_id):
+#     check = USERDB.get(name_or_id)
+#     try:
+#         if check:
+#             return name_or_id
+#         else:
+#             return USERDB["namemap"][name_or_id]
+#     except KeyError:
+#         return False
+#
+#
+# def user_register(auth_hash, name, quip, bio):
+#     if USERDB["namemap"].get(name):
+#         return schema.error(4, "Username taken.")
+#
+#     for ok, error in [
+#         user_namecheck(name),
+#         user_authcheck(auth_hash),
+#         user_quipcheck(quip),
+#         user_biocheck(bio)]:
+#
+#         if not ok:
+#             return error
+#
+#     ID = uuid1().hex
+#     scheme = schema.user_internal(ID, auth_hash, name, quip, bio, False)
+#     USERDB.update({ID: scheme})
+#     USERDB["namemap"].update({name: ID})
+#     user_dbdump(USERDB)
+#     return scheme
+#
+#
+# def user_get(ID):
+#     user = USERDB[ID]
+#     return schema.user_external(
+#         ID, user["name"], user["quip"],
+#         user["bio"], user["admin"])
+#
+#
+# def user_auth(ID, auth_hash):
+#     return auth_hash == USERDB[ID]["auth_hash"]
+#
+#
+#
+#
+# def user_update(ID, **params):
+#     USERDB[ID].update(params)
+#     return USERDB[ID]
+#
+#
diff --git a/src/endpoints.py b/src/endpoints.py
index 973a528..f1b5913 100644
--- a/src/endpoints.py
+++ b/src/endpoints.py
@@ -1,174 +1,4 @@
-from src import formatting
-from src import schema
-from time import time
-from src import db
+from src import db, schema
 
-
-endpoints = {
-    "check_auth":      ["user", "auth_hash"],
-    "is_registered":   ["target_user"],
-    "is_admin":        ["target_user"],
-    "thread_index":    [],
-    "thread_load":     ["thread_id"],
-    "thread_create":   ["title", "body", "tags"],
-    "thread_reply":    ["thread_id", "body"],
-    "edit_post":       ["thread_id", "post_id", "body"],
-    "edit_query":      ["thread_id", "post_id"],
-    "can_edit":        ["thread_id", "post_id"],
-    "user_register":   ["user", "auth_hash", "quip", "bio"],
-    "user_get":        ["target_user"],
-    "user_name_to_id": ["target_user"]
-}
-
-
-authless = [
-    "is_registered",
-    "user_register"
-]
-
-
-# this is not actually an endpoint, but produces a required
-# element of thread responses.
-def create_usermap(thread, index=False):
-    if index:
-        return {user: db.user_get(user) for user in
-                  {i["author"] for i in thread}}
-
-    result = {reply["author"] for reply in thread["replies"]}
-    result.add(thread["author"])
-    return {x: db.user_get(x) for x in result}
-
-
-def user_name_to_id(json):
-    """
-    Returns a string of the target_user's ID when it is
-    part of the database: a non-existent user will return
-    a boolean false.
-    """
-    return db.user_resolve(json["target_user"])
-
-
-def is_registered(json):
-    """
-    Returns true or false whether target_user is registered
-    in the system. This function only takes usernames: not
-    user IDs.
-    """
-    return bool(db.USERDB["namemap"].get(json["target_user"]))
-
-
-def check_auth(json):
-    "Returns true or false whether auth_hashes matches user."
-    return bool(db.user_auth(json["user"], json["auth_hash"]))
-
-
-def is_admin(json):
-    """
-    Returns true or false whether target_user is a system
-    administrator. Takes a username or user ID. Nonexistent
-    users return false.
-    """
-    user = db.user_resolve(json["target_user"])
-    if user:
-        return db.user_is_admin(user)
-    return False
-
-
-def user_register(json):
-    """
-    Registers a new user into the system. Returns the new internal user
-    object on success, or an error response.
-
-    auth_hash should be a hexadecimal SHA-256 string, produced from a
-      UTF-8 password string.
-
-    user should be a string containing no newlines and
-      under 24 characters in length.
-
-    quip is a string, up to 120 characters, provided by the user
-      the acts as small bio, suitable for display next to posts
-      if the client wants to. Whitespace characters besides space
-      are not allowed. The string may be empty.
-
-    bio is a string, up to 4096 chars, provided by the user that
-      can be shown on profiles. There are no character type limits
-      for this entry. The string may be empty.
-
-    All errors for this endpoint with code 4 should show the
-    description direcrtly to the user.
-
-    """
-
-    return schema.response(
-        db.user_register(
-            json["auth_hash"],
-            json["user"],
-            json["quip"],
-            json["bio"]))
-
-
-def user_get(json):
-    """
-    On success, returns an external user object for target_user (ID or name).
-    If the user isn't in the system, returns false.
-    """
-    user = db.user_resolve(json["target_user"])
-    if not user:
-        return False
-    return db.user_get(user)
-
-
-def thread_index(json):
-    index = db.thread_index(markup=not json.get("nomarkup"))
-    return schema.response({"threads": index}, create_usermap(index, True))
-
-
-def thread_load(json):
-    thread = db.thread_load(json["thread_id"], not json.get("nomarkup"))
-    if not thread:
-        return schema.error(7, "Requested thread does not exist")
-    return schema.response(thread, create_usermap(thread))
-
-
-def thread_create(json):
-    thread = db.thread_create(
-        json["user"],
-        json["body"],
-        json["title"],
-        json["tags"])
-    return schema.response(thread)
-
-
-def thread_reply(json):
-    reply = db.thread_reply(
-        json["thread_id"],
-        json["user"],
-        json["body"])
-    return schema.response(reply)
-
-
-def edit_query(json):
-    return db.edit_handler(json)[1]
-
-
-def can_edit(json):
-    return db.edit_handler(json)[0]
-
-
-def edit_post(json):
-    thread = db.thread_load(json["thread_id"])
-    admin = db.user_is_admin(json["user"])
-    target_id = json["post_id"]
-    ok, obj = db.edit_handler(json, thread)
-
-    if ok:
-
-        if json.get("reformat"):
-            json["body"] = formatting.parse(json["body"])
-
-        obj["body"] = json["body"]
-        obj["lastmod"] = time()
-        obj["edited"] = True
-        db.thread_dump(json["thread_id"], thread)
-
-    return obj
+def user_register(user_name, auth_hash):
+    return db.user_register(user_name, auth_hash)
diff --git a/src/exceptions.py b/src/exceptions.py
new file mode 100644
index 0000000..a34e02c
--- /dev/null
+++ b/src/exceptions.py
@@ -0,0 +1,54 @@
+from src.schema import error
+
+
+class BBJException(Exception):
+    """
+    Base class for all exceptions specific to BBJ. These also
+    hold schema error objects, reducing the amount of code
+    required to produce useful errors.
+    """
+    def __init__(self, code, description):
+        self.schema = error(code, description)
+        self.description = description
+        self.code = code
+
+    def __str__(self):
+        return self.description
+
+
+class BBJParameterError(BBJException):
+    """
+    This class of error holds code 3. This is a general
+    classification used to report errors on behalf of
+    the client. It covers malformed or missing parameter
+    values for endpoints, type errors, index errors, etc.
+    A complete client should not encounter these and the
+    descriptions are geared towards client developers
+    rather than users.
+    """
+    def __init__(self, description):
+        super().__init__(3, description)
+
+
+class BBJUserError(BBJException):
+    """
+    This class of error holds code 4. Its description should
+    be shown verbatim in clients, as it deals with invalid user
+    actions rather than client or server errors. It is especially
+    useful during registration, and reporting lack of admin privs
+    when editing messages.
+    """
+    def __init__(self, description):
+        super().__init__(4, description)
+
+
+class BBJAuthError(BBJException):
+    """
+    This class of error holds code 5. Similar to code 4,
+    these should be shown to users verbatim. Provided when:
+
+      * a client tries to post without user/auth_hash pair
+      * the auth_hash does not match the given user
+    """
+    def __init__(self, description):
+        super().__init__(5, description)
diff --git a/src/formatting.py b/src/formatting.py
index 56614c3..d327a24 100644
--- a/src/formatting.py
+++ b/src/formatting.py
@@ -2,27 +2,63 @@ from markdown import markdown
 from html import escape
 import re
 
+colors = [
+    "red", "green", "yellow", "blue", "magenta", "cyan"
+]
 
-COLORS = ["red", "green", "yellow", "blue", "magenta", "cyan"]
-MARKUP = ["bold", "italic", "underline", "strike"]
-TOKENS = re.compile(r"\[({}): (.+?)]".format("|".join(COLORS + MARKUP)), flags=re.DOTALL)
-QUOTES = re.compile("&gt;&gt;([0-9]+)")
-LINEQUOTES = re.compile("^(&gt;.+)$", flags=re.MULTILINE)
+markup = [
+    "bold", "italic", "underline", "strike"
+]
 
+tokens = re.compile(
+    r"\[({}): (.+?)]".format(
+        "|".join(colors + markup)),
+    flags=re.DOTALL)
 
+quotes = re.compile("&gt;&gt;([0-9]+)")
+linequotes = re.compile("^(&gt;.+)$",
+    flags=re.MULTILINE)
+
+
+def apply_formatting(msg_obj, formatter):
+    """
+    Receives a messages object from a thread and returns it with
+    all the message bodies passed through FORMATTER.
+    """
+    for x in msg_obj["messages"].keys():
+        msg_obj["messages"][x]["body"] = \
+            formatter(msg_obj["messages"][x]["body"])
+    return msg_obj
+
+
+def raw(text):
+    """
+    Just return the message in the same state that it was submitted.
+    """
+    return text
+
+
+def html(text):
+    """
+    Returns messages in html format, after being sent through markdown.
+    Color directives are given as:
+      <span color="{COLOR}" style="color: {COLOR};">content</span>
+
+    Directives may be nested. If you don't have access to a fully featured
+    and compliant html renderer in your client, you should use one of the
+    simpler directives like strip, indice, or raw.
+    """
+
+    text = TOKENS.sub(map_html, escape(text))
+    text = QUOTES.sub(r'<span post="\1" class="quote">\g<0></span>', text)
+    return markdown(
+        LINEQUOTES.sub(r'<span class="linequote">\1</span><br>', text))
+
+# and this is the callback used by the sub statement
 def map_html(match):
     directive, body = match.group(1).lower(), match.group(2)
-    if directive in COLORS:
+    if directive in colors:
         return '<span color="{0}" style="color: {0};">{1}</span>'.format(directive, body)
-    elif directive in MARKUP:
+    elif directive in markup:
         return '<{0}>{1}</{0}>'.format(directive[0], body)
     return body
-
-
-def parse(text, doquotes=True):
-    text = TOKENS.sub(map_html, escape(text))
-    if doquotes:
-        text = QUOTES.sub(r'<span post="\1" class="quote">\g<0></span>', text)
-    return markdown(
-        LINEQUOTES.sub(r'<span class="linequote">\1</span><br>', text)
-    )
diff --git a/src/schema.py b/src/schema.py
index 8e3ca35..b376d53 100644
--- a/src/schema.py
+++ b/src/schema.py
@@ -1,18 +1,12 @@
-from src import formatting
-from time import time
-
-
 def base():
     return {
         "error": False
     }
 
 
-def response(dictionary, usermap=None):
+def response(dictionary):
     result = base()
     result.update(dictionary)
-    if usermap:
-        result["usermap"] = usermap
     return result
 
 
@@ -27,71 +21,97 @@ def error(code, description):
     return result
 
 
-def user_internal(ID, auth_hash, name, quip, bio, admin):
+def user_internal(
+        user_id,   # string (uuid1)
+        user_name, # string
+        auth_hash, # string (sha256 hash)
+        quip,      # string (possibly empty)
+        bio,       # string (possibly empty)
+        color,     # int from 0 to 8
+        is_admin,  # bool (supply as either False/True or 0/1)
+        created):  # floating point unix timestamp (when user registered)
+
     if not quip:
         quip = ""
 
     if not bio:
         bio = ""
 
+    if not color:
+        color = 0
+
     return {
-        "user_id":   ID,       # string
-        "quip":      quip,     # (possibly empty) string
-        "bio":       bio,      # (possibly empty) string
-        "name":      name,     # string
-        "admin":     admin,    # boolean
-        "auth_hash": auth_hash # SHA256 string
+        "user_id":   user_id,
+        "user_name": user_name,
+        "auth_hash": auth_hash,
+        "quip":      quip,
+        "bio":       bio,
+        "color":     color,
+        "is_admin":  bool(is_admin),
+        "created":   created
     }
 
 
-def user_external(ID, name, quip, bio, admin):
+def user_external(
+        user_id,   # string (uuid1)
+        user_name, # string
+        quip,      # string (possibly empty)
+        bio,       # string (possibly empty)
+        color,     # int from 0 to 8
+        admin,     # bool (can be supplied as False/True or 0/1)
+        created):  # floating point unix timestamp (when user registered)
+
     if not quip:
         quip = ""
 
     if not bio:
         bio = ""
 
+    if not color:
+        color = 0
+
     return {
-        "user_id":   ID,   # string
-        "quip":      quip, # (possibly empty) string
-        "name":      name, # string
-        "bio":       bio,  # string
-        "admin":     admin # boolean
+        "user_id":   user_id,
+        "user_name": user_name,
+        "quip":      quip,
+        "bio":       bio,
+        "color":     color,
+        "is_admin":  admin,
+        "created":   created
     }
 
 
-def thread(ID, author, body, title, tags):
-    if not tags:
-        tags = list()
-
-    body = formatting.parse(body, doquotes=False)
-    now = time()
+def thread(
+        thread_id,    # uuid string
+        author,       # string (uuid1, user.user_id)
+        title,        # string
+        last_mod,     # floating point unix timestamp (of last post or post edit)
+        created,      # floating point unix timestamp (when thread was made)
+        reply_count): # integer (incremental, starting with 0)
 
     return {
-        "thread_id":   ID,     # string
-        "post_id":     1,      # integer
-        "author":      author, # string
-        "body":        body,   # string
-        "title":       title,  # string
-        "tags":        tags,   # (possibly empty) list of strings
-        "replies":     list(), # (possibly empty) list of reply objects
-        "reply_count": 0,      # integer
-        "edited":      False,  # boolean
-        "lastmod":     now,    # floating point unix timestamp
-        "created":     now     # floating point unix timestamp
+        "thread_id":   thread_id,
+        "author":      author,
+        "title":       title,
+        "last_mod":    last_mod,
+        "created":     created,
+        "reply_count": reply_count,
     }
 
 
-def reply(ID, author, body):
-
-    body = formatting.parse(body)
-    now = time()
+def message(
+        thread_id, # string (uuid1 of parent thread)
+        post_id,   # integer (incrementing from 1)
+        author,    # string (uuid1, user.user_id)
+        created,   # floating point unix timestamp (when reply was posted)
+        edited,    # bool
+        body):     # string
 
     return {
-        "post_id":  ID,     # integer
-        "author":   author, # string
-        "body":     body,   # string
-        "edited":   False,  # boolean
-        "lastmod":  now,    # floating point unix timestamp
-        "created":  now     # floating point unix timestamp
+        "thread_id": thread_id,
+        "post_id":   post_id,
+        "author":    author,
+        "created":   created,
+        "edited":    bool(edited),
+        "body":      body
     }
diff --git a/src/utils.py b/src/utils.py
new file mode 100644
index 0000000..4f4ad0b
--- /dev/null
+++ b/src/utils.py
@@ -0,0 +1,30 @@
+from src import schema
+
+def ordered_keys(subscriptable_object, *keys):
+    """
+    returns a list with the values for KEYS in the order KEYS are provided,
+    from SUBSCRIPTABLE_OBJECT. Useful for working with dictionaries when
+    parameter ordering is important. Used for sql transactions
+    """
+    return tuple([subscriptable_object[key] for key in keys])
+
+
+def schema_values(scheme, obj):
+    """
+    Returns the values in the database order for a given
+    schema. Used for sql transactions
+    """
+    if scheme == "user":
+        return ordered_keys(obj,
+            "user_id", "user_name", "auth_hash", "quip",
+            "bio", "color", "is_admin", "created")
+
+    elif scheme == "thread":
+        return ordered_keys(obj,
+            "thread_id", "author", "title",
+            "last_mod", "created", "reply_count")
+
+    elif scheme == "message":
+        return ordered_keys(obj,
+            "thread_id", "post_id", "author",
+            "created", "edited", "body")