kpohl/CostHive
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects 1 Releases 2 Wiki Activity

major: rewrite database connections

Browse Source
This commit is contained in:
Lunaresk 2022-02-10 16:15:02 +01:00
parent afc5c2854f
commit 1a96e54934
9 changed files with 184 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -133,4 +133,5 @@ dmypy.json
config.yaml
scans.json
test.*
app.db
*.db
.vscode

123
app/database.py
View File

@ -1,123 +0,0 @@
from app import LOGGER
from psycopg2 import connect as psyconn, ProgrammingError, errors
from yaml import safe_load
class Database:
def __init__(self, **kwargs):
pass
def connect(self, **kwargs):
with open('configs/config.yaml', 'r') as file:
data = safe_load(file)['database']
LOGGER.debug('Merging passed arguments with default arguments.')
for key, value in data.items():
if key not in kwargs or not kwargs[key]:
kwargs[key] = value
LOGGER.info('Connecting to Database.')
self.conn = psyconn(host=kwargs["host"], port=kwargs["port"], dbname=kwargs["database"],
user=kwargs["user"], password=kwargs["password"])
self.conn.autocommit = True
def test_connection(self):
if not hasattr(self, "conn"):
LOGGER.info("Connection was not set, setting...")
self.connect()
else:
try:
with self.conn.cursor() as cursor:
try:
cursor.execute("SELECT 1;")
cursor.fetchall()
except:
LOGGER.warn(
'Connection seem to timed out, reconnecting...')
self.connect()
except:
LOGGER.warn('Connection seem to timed out, reconnecting...')
self.connect()
def connectionpersistence(func):
def wrapper(*args, **kwargs):
self = args[0]
self.test_connection()
return func(*args, **kwargs)
return wrapper
@connectionpersistence
def get_user(self, **kwargs):
result = ()
if 'login' in kwargs:
query = "SELECT login FROM users WHERE login = %(login)s;"
with self.conn.cursor() as cursor:
cursor.execute(query, kwargs)
try:
result = cursor.fetchall()
except ProgrammingError as e:
LOGGER.exception("")
except Exception as e:
LOGGER.exception("")
return result
@connectionpersistence
def get_report(self, **kwargs) -> list:
query = "SELECT u.name, bp.date, i.name, bp.amount, bp.price FROM bought_with_prices bp INNER JOIN items i ON bp.item = i.id INNER JOIN users u ON bp.user = u.login"
if kwargs:
query += " WHERE "
tempquery = []
if "user" in kwargs and kwargs['user']:
tempquery.append(f"bp.user = '{kwargs['user']}'")
if "year" in kwargs and kwargs['year']:
tempstring = "bp.date BETWEEN "
if "month" in kwargs and kwargs['month']:
tempstring += f"'{kwargs['year']}-{kwargs['month']}-01' AND "
tempstring += f"'{kwargs['year']+1}-01-01'" if kwargs['month'] == 12 else f"'{kwargs['year']}-{kwargs['month']+1}-01'"
else:
tempstring += f"'{kwargs['year']}-01-01' AND '{kwargs['year']+1}-01-01'"
tempstring += "::date - INTERVAL '1' DAY"
tempquery.append(tempstring)
query += " AND ".join(tempquery)
query += " ORDER BY u.name, bp.date, i.name ASC;"
LOGGER.debug(f"Executing query: {query}")
result = []
with self.conn.cursor() as cursor:
cursor.execute(query)
try:
result = cursor.fetchall()
except ProgrammingError as e:
LOGGER.exception("")
except Exception as e:
LOGGER.exception("")
return result
@connectionpersistence
def insert_bought_items(self, user: str, items: dict, date: str = None):
temp = ['"user", item, amount', "%(user)s, %(item)s, %(amount)s",
"bought.user = %(user)s AND bought.item = %(item)s AND bought.date = " + ("%(date)s" if date else "NOW()::date")]
if date:
temp[0] += ", date"
temp[1] += ", %(date)s"
values = [{'user': user, 'item': int(key), 'amount': value, 'date': date} for key, value in items.items()]
else:
values = [{'user': user, 'item': int(key), 'amount': value} for key, value in items.items()]
query = f"INSERT INTO bought({temp[0]}) VALUES({temp[1]}) ON CONFLICT ON CONSTRAINT bought_user_item_date DO UPDATE SET amount = bought.amount + %(amount)s WHERE {temp[2]};"
with self.conn.cursor() as cursor:
failed = {}
for value in values:
try:
cursor.execute(query, value)
except errors.ForeignKeyViolation as e:
if failed:
failed['items'][value['item']] = value['amount']
else:
failed = {'user': user, 'items': {value['item']: value['amount']}}
if date:
failed['date'] = date
LOGGER.exception("")
except Exception as e:
LOGGER.exception("")
return failed
def __delete__(self):
self.conn.close()

2
app/forms.py
View File

@ -15,7 +15,7 @@ class NewItemForm(FlaskForm):
description = StringField("Description", validators=[DataRequired()])
date = DateField("Insert Date", validators=[DataRequired()])
price_change = FloatField("Price", validators=[DataRequired()])
amount_change = IntegerField("Amount", validators=[DataRequired()])
amount_change = IntegerField("Amount")
category = SelectMultipleField("Categories", choices=[(c.id, c.name) for c in Category.query.order_by("name").all()], validators=[DataRequired()])
brand = SelectField("Brand", choices=[(b.id, b.name) for b in Brand.query.order_by("name").all()], validators=[DataRequired()])
submit = SubmitField("Submit")

77
app/models.py
View File

@ -1,5 +1,7 @@
from app import db, login
from flask_login import UserMixin
from sqlalchemy.sql import text
from sqlalchemy_utils.view import create_view
from werkzeug.security import generate_password_hash, check_password_hash
item_category = db.Table("item_category",
@ -9,8 +11,8 @@ item_category = db.Table("item_category",
class User(UserMixin, db.Model):
id = db.Column(db.String(10), primary_key=True)
name = db.Column(db.String(64))
password_hash = db.Column(db.String(128))
name = db.Column(db.String(64), nullable=False)
password_hash = db.Column(db.String(128), nullable=False)
Bought = db.relationship("Bought", backref='User', lazy='dynamic')
@ -23,16 +25,20 @@ class User(UserMixin, db.Model):
def __repr__(self) -> str:
return f"<User {self.id} ({self.name})>"
class LoginToken(db.Model):
user = db.Column(db.ForeignKey('user.id'), primary_key=True)
token = db.Column(db.String(32), nullable=False)
class Brand(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(32))
name = db.Column(db.String(32), nullable=False)
def __repr__(self) -> str:
return f"<Brand {self.id} ({self.name})>"
class Category(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(32))
name = db.Column(db.String(32), nullable=False)
Item = db.relationship("Item", secondary=item_category, lazy="dynamic", back_populates="Category")
@ -41,13 +47,14 @@ class Category(db.Model):
class Item(db.Model):
id = db.Column(db.BigInteger, primary_key=True)
name = db.Column(db.String(64))
brand = db.Column(db.ForeignKey('brand.id'))
description = db.Column(db.Text)
name = db.Column(db.String(64), nullable=False)
brand = db.Column(db.ForeignKey('brand.id'), nullable=False)
description = db.Column(db.Text, nullable=False)
Category = db.relationship("Category", secondary=item_category, lazy="dynamic", back_populates="Item")
Item = db.relationship("Bought", backref='Item', lazy='dynamic')
Bought = db.relationship("Bought", backref='Item', lazy='dynamic')
PriceChange = db.relationship("PriceChange", backref='Item', lazy='dynamic')
AmountChange = db.relationship("AmountChange", backref='Item', lazy='dynamic')
def __repr__(self) -> str:
return f"<Item {self.id} ({self.name})>"
@ -56,17 +63,17 @@ class Bought(db.Model):
user = db.Column(db.ForeignKey('user.id'), primary_key=True)
item = db.Column(db.ForeignKey('item.id'), primary_key=True)
date = db.Column(db.Date, primary_key=True)
amount = db.Column(db.SmallInteger)
amount = db.Column(db.SmallInteger, nullable=False)
# registered = db.Column(db.Boolean, nullable=False, default=False)
# paid = db.Column(db.SmallInteger, nullable=False, default=0)
def __repr__(self) -> str:
return f"<Bought Object>"
class PriceChange(db.Model):
item = db.Column(db.ForeignKey('item.id'), primary_key=True)
date = db.Column(db.Date, primary_key=True)
price = db.Column(db.SmallInteger)
price = db.Column(db.SmallInteger, nullable=False)
def __repr__(self) -> str:
return f"<Price_Change {self.item} ({self.date})>"
@ -74,11 +81,53 @@ class PriceChange(db.Model):
class AmountChange(db.Model):
item = db.Column(db.ForeignKey('item.id'), primary_key=True)
date = db.Column(db.Date, primary_key=True)
amount = db.Column(db.SmallInteger)
amount = db.Column(db.SmallInteger, nullable=False, default=1)
def __repr__(self) -> str:
return f"<Amount_Change {self.item} ({self.date})>"
class Receipt(db.Model):
id = db.Column(db.Numeric(precision=22, scale=0), primary_key=True)
date = db.Column(db.Date, nullable=False)
registered = db.Column(db.Boolean, nullable=False, default=False)
paid = db.Column(db.SmallInteger, nullable=False, default=0)
def __repr__(self) -> str:
return f"<Receipt {self.id}>"
class ItemReceipt(db.Model):
receipt = db.Column(db.ForeignKey("receipt.id"), primary_key=True)
item = db.Column(db.ForeignKey("item.id"), primary_key=True)
amount = db.Column(db.SmallInteger, nullable=False)
def __repr__(self) -> str:
return f"<ItemReceipt {self.receipt}: {self.item}>"
def query_price_per_amount_view():
p = db.aliased(PriceChange, name="p")
a = db.aliased(AmountChange, name="a")
date = db.func.greatest(p.date, a.date).label("date")
price = (db.func.ceil(p.price.cast(db.Float)/db.func.coalesce(a.amount, 1))/100).label("price")
select = db.select(p.item.label("item"), date, price)
select = select.distinct(p.item, date)
select = select.join(a, p.item==a.item, isouter=True)
select = select.order_by(p.item, db.desc(db.func.greatest(p.date, a.date)))
return select
price_per_amount = create_view("price_per_amount", query_price_per_amount_view(), db.metadata)
def query_bought_with_prices_view():
b = db.aliased(Bought, name="b")
ppa = price_per_amount.alias("ppa")
select = db.select(b.user.label("user"), b.date.label("date"), b.item.label("item"), b.amount.label("amount"), ppa.c.price.label("price"))
select = select.distinct(b.user, b.date, b.item)
select = select.join(ppa, b.item==ppa.c.item)
select = select.where(ppa.c.date<=b.date)
select = select.order_by(db.desc(b.user), db.desc(b.date), db.desc(b.item), db.desc(ppa.c.date))
return select
bought_with_prices = create_view("bought_with_prices", query_bought_with_prices_view(), db.metadata)
@login.user_loader
def load_user(id):
return User.query.get(int(id))

39
app/utils/database_utils.py Normal file
View File

@ -0,0 +1,39 @@
from calendar import month
from app import db, LOGGER
from app.models import Bought, bought_with_prices
from copy import deepcopy
from datetime import date as dtdate, timedelta
from psycopg2 import errors
from sqlalchemy import text
from sqlalchemy.dialects.postgresql import insert
def insert_bought_items(user: str, items: dict, date: str = None):
if not date:
date = dtdate.today()
for item, amount in deepcopy(items).items():
query_insert = insert(Bought).values(user=user, item=int(item), date=date, amount=int(amount))
query_insert = query_insert.on_conflict_do_update("bought_pkey", set_=dict(amount=text(f'bought.amount + {amount}')))
try:
db.session.execute(query_insert)
db.session.commit()
except errors.ForeignKeyViolation as e:
db.session.rollback()
except Exception as e:
db.session.rollback()
LOGGER.exception()
else:
del(items[item])
return {'user':user, 'date': date, 'items': items} if items else {}
def get_report(**kwargs):
query_select = bought_with_prices.select()
if "user" in kwargs:
query_select = query_select.where(bought_with_prices.c.user == kwargs['user'])
match kwargs:
case {"month": month}:
year = kwargs["year"] if "year" in kwargs else dtdate.today().year
query_select = query_select.where(bought_with_prices.c.date.between(dtdate(int(year), int(month), 1), dtdate(int(year), int(month)+1, 1)-timedelta(days=1)))
case {"year": year}:
query_select = query_select.where(bought_with_prices.c.date.between(dtdate(int(year), 1, 1), dtdate(int(year), 12, 31)))
results = db.session.execute(query_select)
return tuple(results)

18
app/utils/view_utils.py Normal file
View File

@ -0,0 +1,18 @@
from app import LOGGER
def group_results(results: tuple) -> dict:
result_dict = {}
LOGGER.debug("Grouping...")
for result in results:
if result[0] not in result_dict:
result_dict[result[0]] = {"sum": 0}
if str(result[1]) not in result_dict[result[0]]:
result_dict[result[0]][str(result[1])] = {}
result_dict[result[0]][str(result[1])][result[2]] = (
result[3], result[4])
price = int(result[3]) * int(float(result[4].split(" ")[0].replace(",", "."))*100)
result_dict[result[0]]["sum"] += price
for key in result_dict.keys():
result_dict[key]["sum"] /= 100
LOGGER.debug("Grouped.")
return result_dict

56
app/views.py
View File

@ -1,10 +1,9 @@
from app import app, LOGGER
from app.database import Database
from app.forms import NewItemForm
#from app.forms import NewItemForm
from app.models import User
from flask import abort, request, render_template
from flask.json import jsonify
DATABASE = Database()
from app.utils import view_utils, database_utils
APPNAME = "scan2kasse"
@ -17,15 +16,15 @@ def index():
def test():
if request.args:
LOGGER.debug(request.args['testing'])
form = NewItemForm()
return render_template("test.html", form=form)
#form = NewItemForm()
#return render_template("test.html", form=form)
@app.route(f'/{APPNAME}/login')
def login():
if not request.json or 'login' not in request.json:
abort(400)
if not DATABASE.get_user(login = request.json['login']):
if not User.query.get(request.json['login']):
abort(403)
return jsonify({}), 200
@ -34,55 +33,34 @@ def login():
def insert():
match request.json:
case {'user': user, 'items': items, 'date': date}:
failed = DATABASE.insert_bought_items(user, items, date)
if failed:
return jsonify(failed), 400
return jsonify({'inserted': True}), 201
failed = database_utils.insert_bought_items(user, items, date)
case {'user': user, 'items': items}:
failed = DATABASE.insert_bought_items(user, items)
if failed:
return jsonify(failed), 400
return jsonify({'inserted': True}), 201
failed = database_utils.insert_bought_items(user, items)
case _:
abort(400)
if failed:
return jsonify(failed), 400
return jsonify({'inserted': True}), 201
@app.route(f'/{APPNAME}/overview', methods=['GET'])
def get_report_from_user():
user, month, year = []*3
user, month, year = [None]*3
if request.args:
args = request.args
if 'month' in args:
month = args['month']
month = int(args['month'])
if 'year' in args:
year = args['year']
year = int(args['year'])
if month and (month > 12 or month < 1):
abort(400)
LOGGER.info("Getting results.")
results = DATABASE.get_report(user=user, year=year, month=month)
results = database_utils.get_report(user=user, year=year, month=month)
LOGGER.debug(f"Results received: {results}")
if results:
result_dict = group_results(results)
result_dict = view_utils.group_results(results)
else:
result_dict = {}
if request.content_type == "application/json":
return jsonify(result_dict)
else:
return render_template("overview.html", results=result_dict)
def group_results(results: tuple) -> dict:
result_dict = {}
LOGGER.debug("Grouping...")
for result in results:
if result[0] not in result_dict:
result_dict[result[0]] = {"sum": 0}
if str(result[1]) not in result_dict[result[0]]:
result_dict[result[0]][str(result[1])] = {}
result_dict[result[0]][str(result[1])][result[2]] = (
result[3], result[4])
price = int(result[3]) * int(float(result[4].split(" ")[0].replace(",", "."))*100)
result_dict[result[0]]["sum"] += price
for key in result_dict.keys():
result_dict[key]["sum"] /= 100
LOGGER.debug("Grouped.")
return result_dict
return render_template("overview.html", results=result_dict)

44
migrations/versions/f79ee0125ba6_add_views.py Normal file
View File

@ -0,0 +1,44 @@
"""add views
Revision ID: f79ee0125ba6
Revises: 60e8f49dee49
Create Date: 2022-02-07 13:29:26.663482
"""
import sqlalchemy as sa
from app import db
from app.models import query_price_per_amount_view, query_bought_with_prices_view
from alembic import op
from sqlalchemy_utils import create_view
# revision identifiers, used by Alembic.
revision = 'f79ee0125ba6'
down_revision = '60e8f49dee49'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
metadata = sa.MetaData()
create_view('price_per_amount',
query_price_per_amount_view(),
metadata
)
create_view('bought_with_prices',
query_bought_with_prices_view(),
metadata
)
metadata.create_all(db.engine)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with db.engine.connect() as con:
con.execute("DROP VIEW bought_with_prices;")
con.execute("DROP VIEW price_per_amount;")
#op.drop_table('bought_with_prices')
#op.drop_table('price_per_amount')
# ### end Alembic commands ###

BIN
requirements.txt
View File

Binary file not shown.