make ND db match DLU

fixes an issue with models on newer versions of python

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -34,7 +34,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           # generate Docker tags based on the following events/attributes
@@ -46,7 +46,7 @@ jobs:
             type=semver,pattern={{major}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}

.gitignore

@@ -19,3 +19,6 @@ app/settings.py
 *.exe
 *.csv
 *.sql
+bin
+lib
+include

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM python:3.8-slim-buster
+FROM python:3.11-slim-bookworm
 
 RUN apt update
 RUN apt -y install zip

README.md

@@ -98,7 +98,7 @@ docker run -d \
 
 ### Environmental Variables
 
-Please Reference `app/settings_exmaple.py` to see all the variables
+Please Reference `app/settings_example.py` to see all the variables
 
   * Required:
     * APP_SECRET_KEY (Must be provided)

app/__init__.py

@@ -19,7 +19,8 @@ from app.commands import (
     gen_image_cache,
     gen_model_cache,
     fix_clone_ids,
-    remove_buffs
+    remove_buffs,
+    find_missing_commendation_items
 )
 from app.models import Account, AccountInvitation, AuditLog
 
@@ -96,6 +97,7 @@ def create_app():
     app.cli.add_command(gen_model_cache)
     app.cli.add_command(fix_clone_ids)
     app.cli.add_command(remove_buffs)
+    app.cli.add_command(find_missing_commendation_items)
 
     register_logging(app)
     register_settings(app)

app/accounts.py

@@ -252,6 +252,9 @@ def get():
         # Delete
         # </a>
 
+        if not current_app.config["USER_ENABLE_EMAIL"]:
+            account["2"] = '''N/A'''
+
         if account["4"]:
             account["4"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
         else:
@@ -267,20 +270,11 @@ def get():
         else:
             account["6"] = '''<h2 class="far fa-check-square text-success"></h2>'''
 
-        if current_app.config["USER_ENABLE_EMAIL"]:
+        if not current_app.config["USER_ENABLE_EMAIL"]:
-            if account["8"]:
+            account["8"] = '''<h2 class="far fa-times-circle text-muted"></h2>'''
-                account["8"] = '''<h2 class="far fa-check-square text-success"></h2>'''
+        elif account["8"]:
-            else:
+            account["8"] = '''<h2 class="far fa-check-square text-success"></h2>'''
-                account["8"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
         else:
-            # shift columns to fill in gap of 2
+            account["8"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
-            account["2"] = account["3"]
-            account["3"] = account["4"]
-            account["4"] = account["5"]
-            account["5"] = account["6"]
-            account["6"] = account["7"]
-            # remove last two columns
-            del account["7"]
-            del account["8"]
 
     return data

app/commands.py

@@ -4,7 +4,7 @@ import random
 import string
 import datetime
 from flask_user import current_app
-from app import db
+from app import db, luclient
 from app.models import Account, PlayKey, CharacterInfo, Property, PropertyContent, UGC, Mail, CharacterXML
 import pathlib
 import zlib
@@ -281,3 +281,41 @@ def find_or_create_account(name, email, password, gm_level=9):
         db.session.add(play_key)
         db.session.commit()
     return  # account
+
+@click.command("find_missing_commendation_items")
+@with_appcontext
+def find_missing_commendation_items():
+    data = dict()
+    lots = set()
+    reward_items = luclient.query_cdclient("Select reward_item1 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item2 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item3 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item4 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    lots.remove(0)
+    lots.remove(-1)
+
+    for lot in lots:
+        itemcompid = luclient.query_cdclient(
+            "Select component_id from ComponentsRegistry where component_type = 11 and id = ?;",
+            [lot],
+            one=True
+        )[0]
+        
+        itemcomp = luclient.query_cdclient(
+            "Select commendationLOT, commendationCost from ItemComponent where id = ?;",
+            [itemcompid],
+            one=True
+        )
+        if itemcomp[0] is None or itemcomp[1] is None:
+            data[lot] = {"name": luclient.get_lot_name(lot)}
+    print(data)
+
+

app/forms.py

@@ -4,7 +4,8 @@ from flask import current_app
 from flask_user.forms import (
     unique_email_validator,
     LoginForm,
-    RegisterForm
+    RegisterForm,
+    ChangePasswordForm
 )
 from flask_user import UserManager
 from wtforms.widgets import TextArea, NumberInput
@@ -14,25 +15,47 @@ from wtforms import (
     SubmitField,
     validators,
     IntegerField,
-    SelectField
+    SelectField,
+    PasswordField
 )
 
 from wtforms.validators import DataRequired, Optional
 from app.models import PlayKey
 
+def password_check(form, field):
+    """
+    Validates that the password does not contain a colon, is between 6 and 40 characters long and has an uppercase letter, lowercase letter and a number
+    """
+    error_msg = "Password must be between 6 and 40 characters long, contain a lowercase letter, an uppercase letter, a number, and cannot contain a colon"
+    password = field.data
+    pass_len = len(password)
+    if pass_len < 6:
+        raise validators.ValidationError(error_msg)
+    if ':' in password:
+        raise validators.ValidationError(error_msg)
+    if not any(c.islower() for c in password):
+        raise validators.ValidationError(error_msg)
+    if not any(c.isupper() for c in password):
+        raise validators.ValidationError(error_msg)
+    if not any(c.isdigit() for c in password):
+        raise validators.ValidationError(error_msg)
+    if pass_len > 40:
+        raise validators.ValidationError(error_msg)
+    return True
+    
 
 def validate_play_key(form, field):
-    """Validates a field for a valid phone number
+    """Validates a field for a valid play kyey
     Args:
         form: REQUIRED, the field's parent form
         field: REQUIRED, the field with data
     Returns:
         None, raises ValidationError if failed
     """
-    # jank to get the fireign key that we need back into the field
+    # jank to get the foreign key that we need back into the field
     if current_app.config["REQUIRE_PLAY_KEY"]:
         field.data = PlayKey.key_is_valid(key_string=field.data)
-    return
+    return True
 
 class CustomRecaptcha(Recaptcha):
     def __call__(self, form, field):
@@ -45,18 +68,20 @@ class CustomUserManager(UserManager):
     def customize(self, app):
         self.RegisterFormClass = CustomRegisterForm
         self.LoginFormClass = CustomLoginForm
+        self.ChangePasswordFormClass = ColonlessChangePasswordForm
 
 class CustomRegisterForm(RegisterForm):
     play_key_id = StringField(
         'Play Key',
-        validators=[
+        validators=[validate_play_key]
-            Optional(),
-            validate_play_key,
-        ]
     )
     recaptcha = RecaptchaField(
         validators=[CustomRecaptcha()]
     )
+    password=PasswordField(
+		'Password',
+		validators=[DataRequired(), password_check]
+	)
 
 class CustomLoginForm(LoginForm):
     recaptcha = RecaptchaField(
@@ -196,3 +221,9 @@ class CharXMLUploadForm(FlaskForm):
     )
 
     submit = SubmitField('Submit')
+
+class ColonlessChangePasswordForm(ChangePasswordForm):
+    new_password = PasswordField(
+        'New Password',
+        validators=[validators.DataRequired(), password_check]
+    )

app/models.py

@@ -7,7 +7,6 @@ import logging
 from flask_sqlalchemy.query import Query
 from sqlalchemy.dialects import mysql
 from sqlalchemy.exc import OperationalError, StatementError
-from sqlalchemy.types import JSON
 from time import sleep
 import random
 import string
@@ -140,6 +139,7 @@ class Account(db.Model, UserMixin):
         unique=True
     )
 
+    # ND Exclusive
     email = db.Column(
         db.Unicode(255),
         nullable=True,
@@ -147,6 +147,7 @@ class Account(db.Model, UserMixin):
         unique=False
     )
 
+    # ND Exclusive
     email_confirmed_at = db.Column(db.DateTime())
 
     password = db.Column(
@@ -167,6 +168,7 @@ class Account(db.Model, UserMixin):
         server_default='0'
     )
 
+    # ND Exclusive
     active = db.Column(
         mysql.BOOLEAN,
         nullable=False,
@@ -216,7 +218,7 @@ class Account(db.Model, UserMixin):
         db.session.delete(self)
         db.session.commit()
 
-
+# ND Exclusive
 class AccountInvitation(db.Model):
     __tablename__ = 'account_invites'
     id = db.Column(db.Integer, primary_key=True)
@@ -457,17 +459,34 @@ class Leaderboard(db.Model):
         passive_deletes=True
     )
 
-    time = db.Column(
+    primaryScore = db.Column(
         mysql.BIGINT(unsigned=True),
         nullable=False,
         server_default='0'
     )
 
-    score = db.Column(
+    secondaryScore = db.Column(
         mysql.BIGINT(unsigned=True),
         nullable=False,
         server_default='0'
     )
+    tertiaryScore = db.Column(
+        mysql.BIGINT(unsigned=True),
+        nullable=False,
+        server_default='0'
+    )
+
+    numWins = db.Column(
+        mysql.INTEGER(unsigned=True),
+        nullable=False,
+        server_default='0'
+    )   
+    
+    timesPlayed = db.Column(
+        mysql.INTEGER(unsigned=True),
+        nullable=False,
+        server_default='0'
+    )
 
     def save(self):
         db.session.add(self)
@@ -487,7 +506,7 @@ class Mail(db.Model):
     )
 
     sender_id = db.Column(
-        mysql.INTEGER,
+        mysql.BIGINT,
         nullable=False
     )
 
@@ -585,12 +604,15 @@ class PetNames(db.Model):
         mysql.TEXT,
         nullable=False
     )
+
+    # ND Exclusive
     approved = db.Column(
         mysql.INTEGER(unsigned=True),
         nullable=False,
         server_default='0'
     )
 
+    # ND Exclusive
     owner_id = db.Column(
         mysql.BIGINT,
         nullable=True
@@ -722,7 +744,7 @@ class Property(db.Model):
 class UGC(db.Model):
     __tablename__ = 'ugc'
     id = db.Column(
-        mysql.INTEGER,
+        mysql.BIGINT,
         primary_key=True
     )
     account_id = db.Column(
@@ -802,7 +824,7 @@ class PropertyContent(db.Model):
     )
 
     ugc_id = db.Column(
-        db.INT,
+        db.BIGINT,
         db.ForeignKey(UGC.id, ondelete='CASCADE'),
         nullable=True
     )
@@ -853,6 +875,39 @@ class PropertyContent(db.Model):
         nullable=False,
     )
 
+    model_name = db.Column(
+        mysql.TEXT,
+        nullable=False,
+        server_default=''
+    )
+
+    model_description = db.Column(
+        mysql.TEXT,
+        nullable=False,
+        server_default=''
+    )
+
+    behavior_1 = db.Column(
+        mysql.BIGINT(),
+        server_default='0'
+    )
+    behavior_2 = db.Column(
+        mysql.BIGINT(),
+        server_default='0'
+    )
+    behavior_3 = db.Column(
+        mysql.BIGINT(),
+        server_default='0'
+    )
+    behavior_4 = db.Column(
+        mysql.BIGINT(),
+        server_default='0'
+    )
+    behavior_5 = db.Column(
+        mysql.BIGINT(),
+        server_default='0'
+    )
+
     def save(self):
         db.session.add(self)
         db.session.commit()
@@ -957,7 +1012,7 @@ class BugReport(db.Model):
     )
 
     reporter_id = db.Column(
-        mysql.INTEGER(),
+        mysql.BIGINT,
         nullable=False,
         server_default='0'
     )
@@ -1018,7 +1073,7 @@ class Reports(db.Model):
     __tablename__ = 'reports'
 
     data = db.Column(
-        JSON(),
+        mysql.MEDIUMBLOB(),
         nullable=False
     )
 

app/reports.py

@@ -4,8 +4,9 @@ from app.models import CharacterInfo, Account, CharacterXML, Reports
 from app.luclient import get_lot_name
 from app import gm_level, scheduler
 from sqlalchemy.orm import load_only
-import datetime
+import xmltodict, gzip, json, datetime
-import xmltodict
+from collections import OrderedDict
+import xml.etree.ElementTree as ET
 
 reports_blueprint = Blueprint('reports', __name__)
 
@@ -44,6 +45,8 @@ def index():
 @gm_level(3)
 def items_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "items").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/items/by_date.html.j2', data=data, date=date)
 
 
@@ -62,6 +65,8 @@ def items_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
         for key in entry.data:
             items[key] = get_lot_name(key)
@@ -104,6 +109,8 @@ def items_graph(start, end):
 @gm_level(3)
 def currency_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "currency").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/currency/by_date.html.j2', data=data, date=date)
 
 
@@ -121,6 +128,8 @@ def currency_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
     for character in characters:
         data = []
@@ -155,6 +164,8 @@ def currency_graph(start, end):
 @gm_level(3)
 def uscore_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "uscore").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/uscore/by_date.html.j2', data=data, date=date)
 
 
@@ -172,6 +183,8 @@ def uscore_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
     for character in characters:
         data = []
@@ -228,39 +241,27 @@ def gen_item_report():
             for char_xml in char_xmls:
                 name = CharacterInfo.query.filter(CharacterInfo.id == char_xml.id).first().name
                 try:
-                    character_json = xmltodict.parse(
+                    xml_data = ET.fromstring(char_xml.xml_data)
-                        char_xml.xml_data,
+                    inventories = xml_data.findall(".//inv/items/in")
-                        attr_prefix="attr_"
+                    for inv in inventories:
-                    )
+                        if inv.attrib["t"] in ["0", "1"]:
-                    for inv in character_json["obj"]["inv"]["items"]["in"]:
+                            for item in inv.findall("i"):
-                        if "i" in inv.keys() and type(inv["i"]) == list and (int(inv["attr_t"]) == 0 or int(inv["attr_t"]) == 1):
+                                item_attr_l = item.attrib["l"]
-                            for item in inv["i"]:
+                                item_attr_c = int(item.attrib["c"]) if "c" in item.attrib else 1
-                                if item["attr_l"] in report_data:
+                                if item_attr_l in report_data:
-                                    if ("attr_c" in item):
+                                    report_data[item_attr_l]["item_count"] = report_data[item_attr_l]["item_count"] + item_attr_c
-                                        report_data[item["attr_l"]]["item_count"] = report_data[item["attr_l"]]["item_count"] + int(item["attr_c"])
-                                    else:
-                                        report_data[item["attr_l"]]["item_count"] = report_data[item["attr_l"]]["item_count"] + 1
                                 else:
-                                    if ("attr_c" in item):
+                                    report_data[item_attr_l] = {"item_count": item_attr_c, "chars": {}}
-                                        report_data[item["attr_l"]] = {"item_count": int(item["attr_c"]), "chars": {}}
+                                if name in report_data[item_attr_l]["chars"]:
-                                    else:
+                                    report_data[item_attr_l]["chars"][name] = report_data[item_attr_l]["chars"][name] + item_attr_c
-                                        report_data[item["attr_l"]] = {"item_count": 1, "chars": {}}
-                                if name in report_data[item["attr_l"]]["chars"]:
-                                    if ("attr_c" in item):
-                                        report_data[item["attr_l"]]["chars"][name] = report_data[item["attr_l"]]["chars"][name] + int(item["attr_c"])
-                                    else:
-                                        report_data[item["attr_l"]]["chars"][name] = report_data[item["attr_l"]]["chars"][name] + 1
                                 else:
-                                    if ("attr_c" in item):
+                                    report_data[item_attr_l]["chars"][name] = item_attr_c
-                                        report_data[item["attr_l"]]["chars"][name] = int(item["attr_c"])
-                                    else:
-                                        report_data[item["attr_l"]]["chars"][name] = 1
                 except Exception as e:
                     current_app.logger.error(f"REPORT::ITEMS - ERROR PARSING CHARACTER {char_xml.id}")
                     current_app.logger.error(f"REPORT::ITEMS - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="items",
                 date=date
             )
@@ -298,17 +299,15 @@ def gen_currency_report():
 
             for character in characters:
                 try:
-                    character_json = xmltodict.parse(
+                    xml_data = ET.fromstring(character.xml_data)
-                        character.xml_data,
+                    char = xml_data.find(".//char")
-                        attr_prefix="attr_"
+                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(char.attrib.get("cc"))
-                    )
-                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(character_json["obj"]["char"]["attr_cc"])
                 except Exception as e:
                     current_app.logger.error(f"REPORT::CURRENCY - ERROR PARSING CHARACTER {character.id}")
                     current_app.logger.error(f"REPORT::CURRENCY - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="currency",
                 date=date
             )
@@ -346,17 +345,15 @@ def gen_uscore_report():
 
             for character in characters:
                 try:
-                    character_json = xmltodict.parse(
+                    xml_data = ET.fromstring(character.xml_data)
-                        character.xml_data,
+                    char = xml_data.find(".//char")
-                        attr_prefix="attr_"
+                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(char.attrib.get("ls"))
-                    )
-                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(character_json["obj"]["char"]["attr_ls"])
                 except Exception as e:
                     current_app.logger.error(f"REPORT::U-SCORE - ERROR PARSING CHARACTER {character.id}")
                     current_app.logger.error(f"REPORT::U-SCORE - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="uscore",
                 date=date
             )

app/templates/accounts/index.html.j2

@@ -14,17 +14,13 @@
         <tr>
             <th>Actions</th>
             <th>Name</th>
-            {% if config.USER_ENABLE_EMAIL %}
+            <th>Email</th>
-              <th>Email</th>
-            {% endif %}
             <th>GM Level</th>
             <th>Locked</th>
             <th>Banned</th>
             <th>Muted</th>
             <th>Registered</th>
-            {% if config.USER_ENABLE_EMAIL %}
+            <th>Email Confirmed</th>
-              <th>Email Confirmed</th>
-            {% endif %}
         </tr>
     </thead>
     <tbody></tbody>

app/templates/partials/_account.html.j2

@@ -116,7 +116,7 @@
       </div>
     {% endif %}
 
-    {% if account_data.play_key and current_user.gm_level > 3 and config.REQUIRE_PLAY_KEY %}
+    {% if account_data.play_key and config.REQUIRE_PLAY_KEY %}
       <hr class="bg-primary"/>
       <div class="row">
         <div class="col text-center">
@@ -133,26 +133,28 @@
           {{ account_data.play_key.key_string }}
         </div>
       </div>
-      <div class="row">
+      {% if current_user.gm_level > 3 %}
-        <div class="col text-right">
+        <div class="row">
-          Uses Left:
+          <div class="col text-right">
+            Uses Left:
+          </div>
+          <div class="col">
+            {{ account_data.play_key.key_uses }}
+          </div>
         </div>
-        <div class="col">
+        <div class="row">
-          {{ account_data.play_key.key_uses }}
+          <div class="col text-right">
+            Active:
+          </div>
+          <div class="col">
+            {% if account_data.active %}
+              <h5 class="far fa-check-square text-success"></h5>
+            {% else %}
+              <h5 class="far fa-times-circle text-danger"></h5>
+            {% endif %}
+          </div>
         </div>
-      </div>
+      {% endif %}
-      <div class="row">
-        <div class="col text-right">
-          Active:
-        </div>
-        <div class="col">
-          {% if account_data.active %}
-            <h5 class="far fa-check-square text-success"></h5>
-          {% else %}
-            <h5 class="far fa-times-circle text-danger"></h5>
-          {% endif %}
-        </div>
-      </div>
     {% endif %}
 
     {% if current_user.id != account_data.id and current_user.gm_level > 3 %}

migrations/env.py

@@ -22,8 +22,8 @@ logger = logging.getLogger('alembic.env')
 # target_metadata = mymodel.Base.metadata
 config.set_main_option(
     'sqlalchemy.url',
-    str(current_app.extensions['migrate'].db.get_engine().url).replace(
+    current_app.config.get('SQLALCHEMY_DATABASE_URI')
-        '%', '%%'))
+)
 target_metadata = current_app.extensions['migrate'].db.metadata
 
 # other values from the config, defined by the needs of env.py,

migrations/versions/1164e037907f_compressss_reports.py

@@ -0,0 +1,164 @@
+"""compressss reports
+
+Revision ID: 1164e037907f
+Revises: a6e42ef03da7
+Create Date: 2023-11-18 01:38:00.127472
+
+"""
+from alembic import op
+from sqlalchemy.orm.session import Session
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+from sqlalchemy.types import JSON
+from sqlalchemy.ext.declarative import declarative_base
+import gzip
+import json
+
+
+Base = declarative_base()
+
+
+# revision identifiers, used by Alembic.
+revision = '1164e037907f'
+down_revision = 'a6e42ef03da7'
+branch_labels = None
+depends_on = None
+
+class ReportsUpgradeNew(Base):
+    __tablename__ = 'reports'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        mysql.MEDIUMBLOB(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+    def save(self):
+        sa.session.add(self)
+        sa.session.commit()
+        sa.session.refresh(self)
+
+
+class ReportsUpgradeOld(Base):
+    __tablename__ = 'reports_old'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        JSON(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+class ReportsDowngradeOld(Base):
+    __tablename__ = 'reports'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        mysql.MEDIUMBLOB(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+    def save(self):
+        sa.session.add(self)
+        sa.session.commit()
+        sa.session.refresh(self)
+
+
+class ReportsDowngradeNew(Base):
+    __tablename__ = 'reports_old'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        JSON(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.rename_table('reports', 'reports_old')
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    reports = session.query(ReportsUpgradeOld)
+    op.create_table('reports',
+        sa.Column('data', mysql.MEDIUMBLOB(), nullable=False),
+        sa.Column('report_type', sa.VARCHAR(length=35), autoincrement=False, nullable=False),
+        sa.Column('date', sa.Date(), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('report_type', 'date')
+    )
+    # insert records
+    new_reports = []
+    # insert records
+    for report in reports:
+        new_reports.append({
+            "data":gzip.compress(json.dumps(report.data).encode('utf-8')), 
+            "report_type":report.report_type, 
+            "date":report.date
+        })
+    op.bulk_insert(ReportsUpgradeNew.__table__, new_reports)
+    op.drop_table('reports_old')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('reports')
+    op.create_table('reports',
+        sa.Column('data', JSON(), nullable=False),
+        sa.Column('report_type', sa.VARCHAR(length=35), autoincrement=False, nullable=False),
+        sa.Column('date', sa.Date(), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('report_type', 'date')
+    )
+    # ### end Alembic commands ###

requirements.txt

@@ -1,68 +1,21 @@
 alembic==1.7.5
-APScheduler==3.8.1
-astroid==2.9.1
 autopep8==1.6.0
-backports.zoneinfo==0.2.1
+bcrypt==4.0.1
-bcrypt==3.2.0
-blinker==1.7.0
-cffi==1.14.6
-click==8.1.7
-colorama==0.4.4
-cryptography==36.0.0
-dnspython==2.1.0
-dominate==2.6.0
 email-validator==1.1.3
 Flask==3.0.0
 Flask-APScheduler==1.12.3
 Flask-Assets==2.1.0
-Flask-Login==0.6.3
-Flask-Mail==0.9.1
-flask-marshmallow==0.15.0
 Flask-Migrate==3.1.0
 Flask-SQLAlchemy==3.1.1
 Flask-User==1.0.2.2
 Flask-WTF==1.2.1
-greenlet==1.1.0
 gunicorn==21.2.0
-idna==3.3
-importlib-metadata==6.8.0
-importlib-resources==6.1.1
-isort==5.10.1
-itsdangerous==2.1.2
-Jinja2==3.1.2
-lazy-object-proxy==1.7.1
 libsass==0.21.0
-Mako==1.2.2
-MarkupSafe==2.1.3
-marshmallow==3.14.1
-marshmallow-sqlalchemy==0.26.1
-mccabe==0.6.1
-packaging==23.2
-passlib==1.7.4
-platformdirs==2.4.1
-pycodestyle==2.8.0
-pycparser==2.20
-pydocstyle==6.1.1
-pyflakes==2.4.0
-pylama==8.3.3
-pylint==2.12.2
 PyMySQL==1.0.2
-python-dateutil==2.8.2
+SQLAlchemy==2.0.40
-pytz==2021.3
-pytz-deprecation-shim==0.1.0.post0
-six==1.16.0
-snowballstemmer==2.2.0
-SQLAlchemy==2.0.23
 sqlalchemy-datatables==2.0.1
-toml==0.10.2
-typing_extensions==4.8.0
-tzdata==2021.5
-tzlocal==4.1
-visitor==0.1.3
 Wand==0.6.7
 webassets==2.0
 Werkzeug==3.0.1
-wrapt==1.13.3
 WTForms==3.0.0
 xmltodict==0.12.0
-zipp==3.17.0