major: rewrite database connections

.gitignore

@@ -133,4 +133,5 @@ dmypy.json
 config.yaml
 scans.json
 test.*
-app.db
+*.db
+.vscode

app/database.py

@@ -1,123 +0,0 @@
-from app import LOGGER
-from psycopg2 import connect as psyconn, ProgrammingError, errors
-from yaml import safe_load
-
-
-class Database:
-
-    def __init__(self, **kwargs):
-        pass
-
-    def connect(self, **kwargs):
-        with open('configs/config.yaml', 'r') as file:
-            data = safe_load(file)['database']
-        LOGGER.debug('Merging passed arguments with default arguments.')
-        for key, value in data.items():
-            if key not in kwargs or not kwargs[key]:
-                kwargs[key] = value
-        LOGGER.info('Connecting to Database.')
-        self.conn = psyconn(host=kwargs["host"], port=kwargs["port"], dbname=kwargs["database"],
-                            user=kwargs["user"], password=kwargs["password"])
-        self.conn.autocommit = True
-
-    def test_connection(self):
-        if not hasattr(self, "conn"):
-            LOGGER.info("Connection was not set, setting...")
-            self.connect()
-        else:
-            try:
-                with self.conn.cursor() as cursor:
-                    try:
-                        cursor.execute("SELECT 1;")
-                        cursor.fetchall()
-                    except:
-                        LOGGER.warn(
-                            'Connection seem to timed out, reconnecting...')
-                        self.connect()
-            except:
-                LOGGER.warn('Connection seem to timed out, reconnecting...')
-                self.connect()
-
-    def connectionpersistence(func):
-        def wrapper(*args, **kwargs):
-            self = args[0]
-            self.test_connection()
-            return func(*args, **kwargs)
-        return wrapper
-
-    @connectionpersistence
-    def get_user(self, **kwargs):
-        result = ()
-        if 'login' in kwargs:
-            query = "SELECT login FROM users WHERE login = %(login)s;"
-            with self.conn.cursor() as cursor:
-                cursor.execute(query, kwargs)
-                try:
-                    result = cursor.fetchall()
-                except ProgrammingError as e:
-                    LOGGER.exception("")
-                except Exception as e:
-                    LOGGER.exception("")
-        return result
-
-    @connectionpersistence
-    def get_report(self, **kwargs) -> list:
-        query = "SELECT u.name, bp.date, i.name, bp.amount, bp.price FROM bought_with_prices bp INNER JOIN items i ON bp.item = i.id INNER JOIN users u ON bp.user = u.login"
-        if kwargs:
-            query += " WHERE "
-            tempquery = []
-            if "user" in kwargs and kwargs['user']:
-                tempquery.append(f"bp.user = '{kwargs['user']}'")
-            if "year" in kwargs and kwargs['year']:
-                tempstring = "bp.date BETWEEN "
-                if "month" in kwargs and kwargs['month']:
-                    tempstring += f"'{kwargs['year']}-{kwargs['month']}-01' AND "
-                    tempstring += f"'{kwargs['year']+1}-01-01'" if kwargs['month'] == 12 else f"'{kwargs['year']}-{kwargs['month']+1}-01'"
-                else:
-                    tempstring += f"'{kwargs['year']}-01-01' AND '{kwargs['year']+1}-01-01'"
-                tempstring += "::date - INTERVAL '1' DAY"
-                tempquery.append(tempstring)
-            query += " AND ".join(tempquery)
-        query += " ORDER BY u.name, bp.date, i.name ASC;"
-        LOGGER.debug(f"Executing query: {query}")
-        result = []
-        with self.conn.cursor() as cursor:
-            cursor.execute(query)
-            try:
-                result = cursor.fetchall()
-            except ProgrammingError as e:
-                LOGGER.exception("")
-            except Exception as e:
-                LOGGER.exception("")
-        return result
-
-    @connectionpersistence
-    def insert_bought_items(self, user: str, items: dict, date: str = None):
-        temp = ['"user", item, amount', "%(user)s, %(item)s, %(amount)s",
-                "bought.user = %(user)s AND bought.item = %(item)s AND bought.date = " + ("%(date)s" if date else "NOW()::date")]
-        if date:
-            temp[0] += ", date"
-            temp[1] += ", %(date)s"
-            values = [{'user': user, 'item': int(key), 'amount': value, 'date': date} for key, value in items.items()]
-        else:
-            values = [{'user': user, 'item': int(key), 'amount': value} for key, value in items.items()]
-        query = f"INSERT INTO bought({temp[0]}) VALUES({temp[1]}) ON CONFLICT ON CONSTRAINT bought_user_item_date DO UPDATE SET amount = bought.amount + %(amount)s WHERE {temp[2]};"
-        with self.conn.cursor() as cursor:
-            failed = {}
-            for value in values:
-                try:
-                    cursor.execute(query, value)
-                except errors.ForeignKeyViolation as e:
-                    if failed:
-                        failed['items'][value['item']] = value['amount']
-                    else:
-                        failed = {'user': user, 'items': {value['item']: value['amount']}}
-                        if date:
-                            failed['date'] = date
-                    LOGGER.exception("")
-                except Exception as e:
-                    LOGGER.exception("")
-        return failed
-
-    def __delete__(self):
-        self.conn.close()

app/forms.py

@@ -15,7 +15,7 @@ class NewItemForm(FlaskForm):
     description = StringField("Description", validators=[DataRequired()])
     date = DateField("Insert Date", validators=[DataRequired()])
     price_change = FloatField("Price", validators=[DataRequired()])
-    amount_change = IntegerField("Amount", validators=[DataRequired()])
+    amount_change = IntegerField("Amount")
     category = SelectMultipleField("Categories", choices=[(c.id, c.name) for c in Category.query.order_by("name").all()], validators=[DataRequired()])
     brand = SelectField("Brand", choices=[(b.id, b.name) for b in Brand.query.order_by("name").all()], validators=[DataRequired()])
     submit = SubmitField("Submit")

app/models.py

@@ -1,5 +1,7 @@
 from app import db, login
 from flask_login import UserMixin
+from sqlalchemy.sql import text
+from sqlalchemy_utils.view import create_view
 from werkzeug.security import generate_password_hash, check_password_hash
 
 item_category = db.Table("item_category",
@@ -9,8 +11,8 @@ item_category = db.Table("item_category",
 
 class User(UserMixin, db.Model):
     id = db.Column(db.String(10), primary_key=True)
-    name = db.Column(db.String(64))
+    name = db.Column(db.String(64), nullable=False)
-    password_hash = db.Column(db.String(128))
+    password_hash = db.Column(db.String(128), nullable=False)
 
     Bought = db.relationship("Bought", backref='User', lazy='dynamic')
 
@@ -23,16 +25,20 @@ class User(UserMixin, db.Model):
     def __repr__(self) -> str:
         return f"<User {self.id} ({self.name})>"
 
+class LoginToken(db.Model):
+    user = db.Column(db.ForeignKey('user.id'), primary_key=True)
+    token = db.Column(db.String(32), nullable=False)
+
 class Brand(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    name = db.Column(db.String(32))
+    name = db.Column(db.String(32), nullable=False)
 
     def __repr__(self) -> str:
         return f"<Brand {self.id} ({self.name})>"
 
 class Category(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    name = db.Column(db.String(32))
+    name = db.Column(db.String(32), nullable=False)
 
     Item = db.relationship("Item", secondary=item_category, lazy="dynamic", back_populates="Category")
     
@@ -41,13 +47,14 @@ class Category(db.Model):
 
 class Item(db.Model):
     id = db.Column(db.BigInteger, primary_key=True)
-    name = db.Column(db.String(64))
+    name = db.Column(db.String(64), nullable=False)
-    brand = db.Column(db.ForeignKey('brand.id'))
+    brand = db.Column(db.ForeignKey('brand.id'), nullable=False)
-    description = db.Column(db.Text)
+    description = db.Column(db.Text, nullable=False)
 
     Category = db.relationship("Category", secondary=item_category, lazy="dynamic", back_populates="Item")
-
+    Bought = db.relationship("Bought", backref='Item', lazy='dynamic')
-    Item = db.relationship("Bought", backref='Item', lazy='dynamic')
+    PriceChange = db.relationship("PriceChange", backref='Item', lazy='dynamic')
+    AmountChange = db.relationship("AmountChange", backref='Item', lazy='dynamic')
     
     def __repr__(self) -> str:
         return f"<Item {self.id} ({self.name})>"
@@ -56,17 +63,17 @@ class Bought(db.Model):
     user = db.Column(db.ForeignKey('user.id'), primary_key=True)
     item = db.Column(db.ForeignKey('item.id'), primary_key=True)
     date = db.Column(db.Date, primary_key=True)
-    amount = db.Column(db.SmallInteger)
+    amount = db.Column(db.SmallInteger, nullable=False)
+    # registered = db.Column(db.Boolean, nullable=False, default=False)
+    # paid = db.Column(db.SmallInteger, nullable=False, default=0)
 
-
-    
     def __repr__(self) -> str:
         return f"<Bought Object>"
 
 class PriceChange(db.Model):
     item = db.Column(db.ForeignKey('item.id'), primary_key=True)
     date = db.Column(db.Date, primary_key=True)
-    price = db.Column(db.SmallInteger)
+    price = db.Column(db.SmallInteger, nullable=False)
     
     def __repr__(self) -> str:
         return f"<Price_Change {self.item} ({self.date})>"
@@ -74,11 +81,53 @@ class PriceChange(db.Model):
 class AmountChange(db.Model):
     item = db.Column(db.ForeignKey('item.id'), primary_key=True)
     date = db.Column(db.Date, primary_key=True)
-    amount = db.Column(db.SmallInteger)
+    amount = db.Column(db.SmallInteger, nullable=False, default=1)
     
     def __repr__(self) -> str:
         return f"<Amount_Change {self.item} ({self.date})>"
 
+class Receipt(db.Model):
+    id = db.Column(db.Numeric(precision=22, scale=0), primary_key=True)
+    date = db.Column(db.Date, nullable=False)
+    registered = db.Column(db.Boolean, nullable=False, default=False)
+    paid = db.Column(db.SmallInteger, nullable=False, default=0)
+    
+    def __repr__(self) -> str:
+        return f"<Receipt {self.id}>"
+
+class ItemReceipt(db.Model):
+    receipt = db.Column(db.ForeignKey("receipt.id"), primary_key=True)
+    item = db.Column(db.ForeignKey("item.id"), primary_key=True)
+    amount = db.Column(db.SmallInteger, nullable=False)
+    
+    def __repr__(self) -> str:
+        return f"<ItemReceipt {self.receipt}: {self.item}>"
+
+def query_price_per_amount_view():
+    p = db.aliased(PriceChange, name="p")
+    a = db.aliased(AmountChange, name="a")
+    date = db.func.greatest(p.date, a.date).label("date")
+    price = (db.func.ceil(p.price.cast(db.Float)/db.func.coalesce(a.amount, 1))/100).label("price")
+    select = db.select(p.item.label("item"), date, price)
+    select = select.distinct(p.item, date)
+    select = select.join(a, p.item==a.item, isouter=True)
+    select = select.order_by(p.item, db.desc(db.func.greatest(p.date, a.date)))
+    return select
+
+price_per_amount = create_view("price_per_amount", query_price_per_amount_view(), db.metadata)
+
+def query_bought_with_prices_view():
+    b = db.aliased(Bought, name="b")
+    ppa = price_per_amount.alias("ppa")
+    select = db.select(b.user.label("user"), b.date.label("date"), b.item.label("item"), b.amount.label("amount"), ppa.c.price.label("price"))
+    select = select.distinct(b.user, b.date, b.item)
+    select = select.join(ppa, b.item==ppa.c.item)
+    select = select.where(ppa.c.date<=b.date)
+    select = select.order_by(db.desc(b.user), db.desc(b.date), db.desc(b.item), db.desc(ppa.c.date))
+    return select
+
+bought_with_prices = create_view("bought_with_prices", query_bought_with_prices_view(), db.metadata)
+
 @login.user_loader
 def load_user(id):
     return User.query.get(int(id))

app/utils/database_utils.py

@@ -0,0 +1,39 @@
+from calendar import month
+from app import db, LOGGER
+from app.models import Bought, bought_with_prices
+from copy import deepcopy
+from datetime import date as dtdate, timedelta
+from psycopg2 import errors
+from sqlalchemy import text
+from sqlalchemy.dialects.postgresql import insert
+
+def insert_bought_items(user: str, items: dict, date: str = None):
+    if not date:
+        date = dtdate.today()
+    for item, amount in deepcopy(items).items():
+        query_insert = insert(Bought).values(user=user, item=int(item), date=date, amount=int(amount))
+        query_insert = query_insert.on_conflict_do_update("bought_pkey", set_=dict(amount=text(f'bought.amount + {amount}')))
+        try:
+            db.session.execute(query_insert)
+            db.session.commit()
+        except errors.ForeignKeyViolation as e:
+            db.session.rollback()
+        except Exception as e:
+            db.session.rollback()
+            LOGGER.exception()
+        else:
+            del(items[item])
+    return {'user':user, 'date': date, 'items': items} if items else {}
+
+def get_report(**kwargs):
+    query_select = bought_with_prices.select()
+    if "user" in kwargs:
+        query_select = query_select.where(bought_with_prices.c.user == kwargs['user'])
+    match kwargs:
+        case {"month": month}:
+            year = kwargs["year"] if "year" in kwargs else dtdate.today().year
+            query_select = query_select.where(bought_with_prices.c.date.between(dtdate(int(year), int(month), 1), dtdate(int(year), int(month)+1, 1)-timedelta(days=1)))
+        case {"year": year}:
+            query_select = query_select.where(bought_with_prices.c.date.between(dtdate(int(year), 1, 1), dtdate(int(year), 12, 31)))
+    results = db.session.execute(query_select)
+    return tuple(results)

app/utils/view_utils.py

@@ -0,0 +1,18 @@
+from app import LOGGER
+
+def group_results(results: tuple) -> dict:
+    result_dict = {}
+    LOGGER.debug("Grouping...")
+    for result in results:
+        if result[0] not in result_dict:
+            result_dict[result[0]] = {"sum": 0}
+        if str(result[1]) not in result_dict[result[0]]:
+            result_dict[result[0]][str(result[1])] = {}
+        result_dict[result[0]][str(result[1])][result[2]] = (
+            result[3], result[4])
+        price = int(result[3]) * int(float(result[4].split(" ")[0].replace(",", "."))*100)
+        result_dict[result[0]]["sum"] += price
+    for key in result_dict.keys():
+        result_dict[key]["sum"] /= 100
+    LOGGER.debug("Grouped.")
+    return result_dict

app/views.py

@@ -1,10 +1,9 @@
 from app import app, LOGGER
-from app.database import Database
+#from app.forms import NewItemForm
-from app.forms import NewItemForm
+from app.models import User
 from flask import abort, request, render_template
 from flask.json import jsonify
-
+from app.utils import view_utils, database_utils
-DATABASE = Database()
 
 APPNAME = "scan2kasse"
 
@@ -17,15 +16,15 @@ def index():
 def test():
     if request.args:
         LOGGER.debug(request.args['testing'])
-    form = NewItemForm()
+    #form = NewItemForm()
-    return render_template("test.html", form=form)
+    #return render_template("test.html", form=form)
 
 
 @app.route(f'/{APPNAME}/login')
 def login():
     if not request.json or 'login' not in request.json:
         abort(400)
-    if not DATABASE.get_user(login = request.json['login']):
+    if not User.query.get(request.json['login']):
         abort(403)
     return jsonify({}), 200
 
@@ -34,55 +33,34 @@ def login():
 def insert():
     match request.json:
         case {'user': user, 'items': items, 'date': date}:
-            failed = DATABASE.insert_bought_items(user, items, date)
+            failed = database_utils.insert_bought_items(user, items, date)
-            if failed:
-                return jsonify(failed), 400
-            return jsonify({'inserted': True}), 201
         case {'user': user, 'items': items}:
-            failed = DATABASE.insert_bought_items(user, items)
+            failed = database_utils.insert_bought_items(user, items)
-            if failed:
-                return jsonify(failed), 400
-            return jsonify({'inserted': True}), 201
         case _:
             abort(400)
+    if failed:
+        return jsonify(failed), 400
+    return jsonify({'inserted': True}), 201
 
 @app.route(f'/{APPNAME}/overview', methods=['GET'])
 def get_report_from_user():
-    user, month, year = []*3
+    user, month, year = [None]*3
     if request.args:
         args = request.args
         if 'month' in args:
-            month = args['month']
+            month = int(args['month'])
         if 'year' in args:
-            year = args['year']
+            year = int(args['year'])
     if month and (month > 12 or month < 1):
         abort(400)
     LOGGER.info("Getting results.")
-    results = DATABASE.get_report(user=user, year=year, month=month)
+    results = database_utils.get_report(user=user, year=year, month=month)
     LOGGER.debug(f"Results received: {results}")
     if results:
-        result_dict = group_results(results)
+        result_dict = view_utils.group_results(results)
     else:
         result_dict = {}
     if request.content_type == "application/json":
         return jsonify(result_dict)
     else:
-        return render_template("overview.html", results=result_dict)
+        return render_template("overview.html", results=result_dict)
-
-
-def group_results(results: tuple) -> dict:
-    result_dict = {}
-    LOGGER.debug("Grouping...")
-    for result in results:
-        if result[0] not in result_dict:
-            result_dict[result[0]] = {"sum": 0}
-        if str(result[1]) not in result_dict[result[0]]:
-            result_dict[result[0]][str(result[1])] = {}
-        result_dict[result[0]][str(result[1])][result[2]] = (
-            result[3], result[4])
-        price = int(result[3]) * int(float(result[4].split(" ")[0].replace(",", "."))*100)
-        result_dict[result[0]]["sum"] += price
-    for key in result_dict.keys():
-        result_dict[key]["sum"] /= 100
-    LOGGER.debug("Grouped.")
-    return result_dict

migrations/versions/f79ee0125ba6_add_views.py

@@ -0,0 +1,44 @@
+"""add views
+
+Revision ID: f79ee0125ba6
+Revises: 60e8f49dee49
+Create Date: 2022-02-07 13:29:26.663482
+
+"""
+import sqlalchemy as sa
+from app import db
+from app.models import query_price_per_amount_view, query_bought_with_prices_view
+from alembic import op
+from sqlalchemy_utils import create_view
+
+
+# revision identifiers, used by Alembic.
+revision = 'f79ee0125ba6'
+down_revision = '60e8f49dee49'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    metadata = sa.MetaData()
+    create_view('price_per_amount',
+    query_price_per_amount_view(),
+    metadata
+    )
+    create_view('bought_with_prices',
+    query_bought_with_prices_view(),
+    metadata
+    )
+    metadata.create_all(db.engine)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with db.engine.connect() as con:
+        con.execute("DROP VIEW bought_with_prices;")
+        con.execute("DROP VIEW price_per_amount;")
+    #op.drop_table('bought_with_prices')
+    #op.drop_table('price_per_amount')
+    # ### end Alembic commands ###