Update SQLAlchemy version to 2.0.40

fixes an issue with models on newer versions of python

.github/workflows/ci.yml

@@ -0,0 +1,54 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*.*.*"
+  pull_request:
+    branches:
+      - "main"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=ref,event=pr
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -19,3 +19,6 @@ app/settings.py
 *.exe
 *.csv
 *.sql
+bin
+lib
+include

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM python:3.8-slim-buster
+FROM python:3.11-slim-bookworm
 
 RUN apt update
 RUN apt -y install zip

README.md

@@ -83,11 +83,12 @@ docker run -d \
     -e APP_SECRET_KEY='<secret_key>' \
     -e APP_DATABASE_URI='mysql+pymysql://<username>:<password>@<host>:<port>/<database>' \
     # you can include other optional Environment Variables from below like this
-    -e REQUIRE_PLAY_KEY=True
-    -p 8000:8000/tcp
-    -v /path/to/unpacked/client:/app/luclient:rw \
+    -e REQUIRE_PLAY_KEY=True \
+    -p 8000:8000/tcp \
+    -v /path/to/logs:/logs:rw /
+    -v /path/to/unpacked/client:/app/luclient:ro \
     -v /path/to/cachedir:/app/cache:rw \
-    aronwk/nexus-dashboard:latest
+    ghcr.io/darkflameuniverse/nexusdashboard:latest
 ```
 
  * `/app/luclient` must be mapped to the location of an unpacked client
@@ -97,7 +98,7 @@ docker run -d \
 
 ### Environmental Variables
 
-Please Reference `app/settings_exmaple.py` to see all the variables
+Please Reference `app/settings_example.py` to see all the variables
 
   * Required:
     * APP_SECRET_KEY (Must be provided)
@@ -248,12 +249,12 @@ Run the following command to clone the repository `git clone https://github.com/
 
 You should now have a directory called `NexusDashboard` present on your desktop.
 
-### Setting up 
+### Setting up
 Now that we have the repository cloned you need to rename the example settings file, you can perform this manually in the GUI or you can use the command line, to do the latter run the following commands
   * `cd NexusDashboard\app`
   * `copy settings_example.py settings.py`
 
-Now let's open the settings file we just created and configure some of the settings with the Windows default notepad. 
+Now let's open the settings file we just created and configure some of the settings with the Windows default notepad.
 * `notepad settings.py`
 
 Inside this file is where you can change certain settings like user registration, email support and other things. In this tutorial we will only be focusing on the bare minimum to get up and running, but feel free to adjust what you would like to fit your needs.

app/__init__.py

@@ -5,7 +5,6 @@ from flask_assets import Environment
 from webassets import Bundle
 import time
 from app.models import db, migrate, PlayKey
-from app.schemas import ma
 from app.forms import CustomUserManager
 from flask_user import user_registered, current_user, user_logged_in
 from flask_wtf.csrf import CSRFProtect
@@ -19,7 +18,9 @@ from app.commands import (
     load_property,
     gen_image_cache,
     gen_model_cache,
-    fix_clone_ids
+    fix_clone_ids,
+    remove_buffs,
+    find_missing_commendation_items
 )
 from app.models import Account, AccountInvitation, AuditLog
 
@@ -95,6 +96,8 @@ def create_app():
     app.cli.add_command(gen_image_cache)
     app.cli.add_command(gen_model_cache)
     app.cli.add_command(fix_clone_ids)
+    app.cli.add_command(remove_buffs)
+    app.cli.add_command(find_missing_commendation_items)
 
     register_logging(app)
     register_settings(app)
@@ -126,8 +129,6 @@ def register_extensions(app):
     """
     db.init_app(app)
     migrate.init_app(app, db)
-    ma.init_app(app)
-
     scheduler.init_app(app)
     scheduler.start()
 
@@ -202,19 +203,23 @@ def register_settings(app):
     # Load environment specific settings
     app.config['TESTING'] = False
     app.config['DEBUG'] = False
+    app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
+        "pool_pre_ping": True,
+        "pool_size": 10,
+        "max_overflow": 2,
+        "pool_recycle": 300,
+        "pool_pre_ping": True,
+        "pool_use_lifo": True
+    }
 
-    # always pull these two from the env
     app.config['SECRET_KEY'] = os.getenv(
         'APP_SECRET_KEY',
         app.config['APP_SECRET_KEY']
-
     )
     app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv(
         'APP_DATABASE_URI',
         app.config['APP_DATABASE_URI']
     )
-
-    # try to get overides, otherwise just use what we have already
     app.config['USER_ENABLE_REGISTER'] = os.getenv(
         'USER_ENABLE_REGISTER',
         app.config['USER_ENABLE_REGISTER']
@@ -239,21 +244,15 @@ def register_settings(app):
         'USER_REQUIRE_INVITATION',
         app.config['USER_REQUIRE_INVITATION']
     )
-    app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
-        "pool_pre_ping": True,
-        "pool_size": 10,
-        "max_overflow": 2,
-        "pool_recycle": 300,
-        "pool_pre_ping": True,
-        "pool_use_lifo": True
-    }
     app.config['MAIL_SERVER'] = os.getenv(
         'MAIL_SERVER',
         app.config['MAIL_SERVER']
     )
-    app.config['MAIL_PORT'] = os.getenv(
-        'MAIL_USE_SSL',
+    app.config['MAIL_PORT'] = int(
+        os.getenv(
+        'MAIL_PORT',
         app.config['MAIL_PORT']
+        )
     )
     app.config['MAIL_USE_SSL'] = os.getenv(
         'MAIL_USE_SSL',
@@ -308,6 +307,42 @@ def register_settings(app):
         app.config['CACHE_LOCATION']
     )
 
+    # Recaptcha settings
+    if "RECAPTCHA_ENABLE" not in app.config:
+        app.config['RECAPTCHA_ENABLE'] = False
+    app.config['RECAPTCHA_ENABLE'] = os.getenv(
+        'RECAPTCHA_ENABLE',
+        app.config['RECAPTCHA_ENABLE']
+    )
+    if "RECAPTCHA_PUBLIC_KEY" not in app.config:
+        app.config['RECAPTCHA_PUBLIC_KEY'] = ''
+    app.config['RECAPTCHA_PUBLIC_KEY'] = os.getenv(
+        'RECAPTCHA_PUBLIC_KEY',
+        app.config['RECAPTCHA_PUBLIC_KEY']
+    )
+    if "RECAPTCHA_PRIVATE_KEY" not in app.config:
+        app.config['RECAPTCHA_PRIVATE_KEY'] = ''
+    app.config['RECAPTCHA_PRIVATE_KEY'] = os.getenv(
+        'RECAPTCHA_PRIVATE_KEY',
+        app.config['RECAPTCHA_PRIVATE_KEY']
+    )
+    # Optional
+    if "RECAPTCHA_API_SERVER" in app.config:
+        app.config['RECAPTCHA_API_SERVER'] = os.getenv(
+            'RECAPTCHA_API_SERVER',
+            app.config['RECAPTCHA_API_SERVER']
+        )
+    if "RECAPTCHA_PARAMETERS" in app.config:
+        app.config['RECAPTCHA_PARAMETERS'] = os.getenv(
+            'RECAPTCHA_PARAMETERS',
+            app.config['RECAPTCHA_PARAMETERS']
+        )
+    if "RECAPTCHA_DATA_ATTRS" in app.config:
+        app.config['RECAPTCHA_DATA_ATTRS'] = os.getenv(
+            'RECAPTCHA_DATA_ATTRS',
+            app.config['RECAPTCHA_DATA_ATTRS']
+        )
+
 
 
 def gm_level(gm_level):

app/accounts.py

@@ -19,16 +19,12 @@ from app.models import (
     db,
     Friends
 )
-from app.schemas import AccountSchema
 from app import gm_level, log_audit
 from app.forms import EditGMLevelForm, EditEmailForm
 from sqlalchemy import or_
 
 accounts_blueprint = Blueprint('accounts', __name__)
 
-account_schema = AccountSchema()
-
-
 @accounts_blueprint.route('/', methods=['GET'])
 @login_required
 @gm_level(3)
@@ -256,6 +252,9 @@ def get():
         # Delete
         # </a>
 
+        if not current_app.config["USER_ENABLE_EMAIL"]:
+            account["2"] = '''N/A'''
+
         if account["4"]:
             account["4"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
         else:
@@ -271,20 +270,11 @@ def get():
         else:
             account["6"] = '''<h2 class="far fa-check-square text-success"></h2>'''
 
-        if current_app.config["USER_ENABLE_EMAIL"]:
-            if account["8"]:
-                account["8"] = '''<h2 class="far fa-check-square text-success"></h2>'''
-            else:
-                account["8"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
+        if not current_app.config["USER_ENABLE_EMAIL"]:
+            account["8"] = '''<h2 class="far fa-times-circle text-muted"></h2>'''
+        elif account["8"]:
+            account["8"] = '''<h2 class="far fa-check-square text-success"></h2>'''
         else:
-            # shift columns to fill in gap of 2
-            account["2"] = account["3"]
-            account["3"] = account["4"]
-            account["4"] = account["5"]
-            account["5"] = account["6"]
-            account["6"] = account["7"]
-            # remove last two columns
-            del account["7"]
-            del account["8"]
+            account["8"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
 
     return data

app/characters.py

@@ -3,20 +3,17 @@ from flask_user import login_required, current_user
 from datatables import ColumnDT, DataTables
 import time
 from app.models import CharacterInfo, CharacterXML, Account, db
-from app.schemas import CharacterInfoSchema
 from app.forms import RescueForm, CharXMLUploadForm
 from app import gm_level, log_audit
 from app.luclient import translate_from_locale
 import xmltodict
 import xml.etree.ElementTree as ET
 import json
+from xml.dom import minidom
 
 
 character_blueprint = Blueprint('characters', __name__)
 
-character_schema = CharacterInfoSchema()
-
-
 @character_blueprint.route('/', methods=['GET'])
 @login_required
 @gm_level(3)
@@ -103,6 +100,88 @@ def view(id):
         character_json=character_json
     )
 
+@character_blueprint.route('/chardata/<id>', methods=['GET'])
+@login_required
+def chardata(id):
+
+    character_data = CharacterInfo.query.filter(CharacterInfo.id == id).first()
+
+    if character_data == {}:
+        abort(404)
+        return
+
+    if current_user.gm_level < 3:
+        if character_data.account_id and character_data.account_id != current_user.id:
+            abort(403)
+            return
+    character_json = xmltodict.parse(
+        CharacterXML.query.filter(
+            CharacterXML.id == id
+        ).first().xml_data.replace("\"stt=", "\" stt="),
+        attr_prefix="attr_"
+    )
+
+    # print json for reference
+    # with open("errorchar.json", "a") as file:
+    #     file.write(
+    #         json.dumps(character_json, indent=4)
+    #     )
+
+    # stupid fix for jinja parsing
+    character_json["obj"]["inv"]["holdings"] = character_json["obj"]["inv"].pop("items")
+    # sort by items slot index
+    if type(character_json["obj"]["inv"]["holdings"]["in"]) == list:
+        for inv in character_json["obj"]["inv"]["holdings"]["in"]:
+                if "i" in inv.keys() and type(inv["i"]) == list:
+                    inv["i"] = sorted(inv["i"], key=lambda i: int(i['attr_s']))
+
+    return render_template(
+        'partials/_charxml.html.j2',
+        character_data=character_data,
+        character_json=character_json
+    )
+
+@character_blueprint.route('/inventory/<id>/<inventory_id>', methods=['GET'])
+@login_required
+def inventory(id, inventory_id):
+    
+    character_data = CharacterInfo.query.filter(CharacterInfo.id == id).first()
+
+    if character_data == {}:
+        abort(404)
+        return
+
+    if current_user.gm_level < 3:
+        if character_data.account_id and character_data.account_id != current_user.id:
+            abort(403)
+            return
+    character_json = xmltodict.parse(
+        CharacterXML.query.filter(
+            CharacterXML.id == id
+        ).first().xml_data.replace("\"stt=", "\" stt="),
+        attr_prefix="attr_"
+    )
+
+    # print json for reference
+    # with open("errorchar.json", "a") as file:
+    #     file.write(
+    #         json.dumps(character_json, indent=4)
+    #     )
+
+    # stupid fix for jinja parsing
+    character_json["obj"]["inv"]["holdings"] = character_json["obj"]["inv"].pop("items")
+    # sort by items slot index
+    if type(character_json["obj"]["inv"]["holdings"]["in"]) == list:
+        for inv in character_json["obj"]["inv"]["holdings"]["in"]:
+                if "i" in inv.keys() and type(inv["i"]) == list:
+                    inv["i"] = sorted(inv["i"], key=lambda i: int(i['attr_s']))
+    for inventory in character_json["obj"]["inv"]["holdings"]["in"]:
+        if inventory["attr_t"] == inventory_id:
+            return render_template(
+                'partials/charxml/_inventory.html.j2',
+                inventory=inventory
+            )
+    return "No Items in Inventory", 404
 
 @character_blueprint.route('/view_xml/<id>', methods=['GET'])
 @login_required
@@ -238,7 +317,7 @@ def upload(id):
         flash("You accept all consequences from these actions", "danger")
         log_audit(f"Updated {character_data.id}'s xml data")
         return redirect(url_for('characters.view', id=id))
-    form.char_xml.data = character_data.xml_data
+    form.char_xml.data = minidom.parseString(character_data.xml_data).toprettyxml(indent="   ")
     return render_template("character/upload.html.j2", form=form)
 
 

app/commands.py

@@ -4,8 +4,8 @@ import random
 import string
 import datetime
 from flask_user import current_app
-from app import db
-from app.models import Account, PlayKey, CharacterInfo, Property, PropertyContent, UGC, Mail
+from app import db, luclient
+from app.models import Account, PlayKey, CharacterInfo, Property, PropertyContent, UGC, Mail, CharacterXML
 import pathlib
 import zlib
 from wand import image
@@ -15,7 +15,7 @@ from multiprocessing import Pool
 from functools import partial
 from sqlalchemy import func
 import time
-
+import xml.etree.ElementTree as ET
 
 @click.command("init_db")
 @click.argument('drop_tables', nargs=1)
@@ -180,6 +180,20 @@ def load_property(zone, player):
             )
             new_prop_content.save()
 
+@click.command("remove_buffs")
+@with_appcontext
+def remove_buffs():
+    """Clears all buff from all characters"""
+    chars = CharacterXML.query.all()
+    for char in chars:
+        character_xml = ET.XML(char.xml_data.replace("\"stt=", "\" stt="))
+        dest = character_xml.find(".//dest")
+        if dest:
+            buff = character_xml.find(".//buff")
+            if buff:
+                dest.remove(buff)
+        char.xml_data = ET.tostring(character_xml)
+        char.save()
 
 @click.command("gen_image_cache")
 def gen_image_cache():
@@ -267,3 +281,41 @@ def find_or_create_account(name, email, password, gm_level=9):
         db.session.add(play_key)
         db.session.commit()
     return  # account
+
+@click.command("find_missing_commendation_items")
+@with_appcontext
+def find_missing_commendation_items():
+    data = dict()
+    lots = set()
+    reward_items = luclient.query_cdclient("Select reward_item1 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item2 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item3 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item4 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    lots.remove(0)
+    lots.remove(-1)
+
+    for lot in lots:
+        itemcompid = luclient.query_cdclient(
+            "Select component_id from ComponentsRegistry where component_type = 11 and id = ?;",
+            [lot],
+            one=True
+        )[0]
+        
+        itemcomp = luclient.query_cdclient(
+            "Select commendationLOT, commendationCost from ItemComponent where id = ?;",
+            [itemcompid],
+            one=True
+        )
+        if itemcomp[0] is None or itemcomp[1] is None:
+            data[lot] = {"name": luclient.get_lot_name(lot)}
+    print(data)
+
+

app/forms.py

@@ -1,92 +1,92 @@
-from flask_wtf import FlaskForm
+from flask_wtf import FlaskForm, Recaptcha, RecaptchaField
 from flask import current_app
 
 from flask_user.forms import (
     unique_email_validator,
-    password_validator,
-    unique_username_validator
+    LoginForm,
+    RegisterForm,
+    ChangePasswordForm
 )
 from flask_user import UserManager
 from wtforms.widgets import TextArea, NumberInput
 from wtforms import (
     StringField,
-    HiddenField,
-    PasswordField,
     BooleanField,
     SubmitField,
     validators,
     IntegerField,
-    SelectField
+    SelectField,
+    PasswordField
 )
 
 from wtforms.validators import DataRequired, Optional
 from app.models import PlayKey
 
+def password_check(form, field):
+    """
+    Validates that the password does not contain a colon, is between 6 and 40 characters long and has an uppercase letter, lowercase letter and a number
+    """
+    error_msg = "Password must be between 6 and 40 characters long, contain a lowercase letter, an uppercase letter, a number, and cannot contain a colon"
+    password = field.data
+    pass_len = len(password)
+    if pass_len < 6:
+        raise validators.ValidationError(error_msg)
+    if ':' in password:
+        raise validators.ValidationError(error_msg)
+    if not any(c.islower() for c in password):
+        raise validators.ValidationError(error_msg)
+    if not any(c.isupper() for c in password):
+        raise validators.ValidationError(error_msg)
+    if not any(c.isdigit() for c in password):
+        raise validators.ValidationError(error_msg)
+    if pass_len > 40:
+        raise validators.ValidationError(error_msg)
+    return True
+    
 
 def validate_play_key(form, field):
-    """Validates a field for a valid phone number
+    """Validates a field for a valid play kyey
     Args:
         form: REQUIRED, the field's parent form
         field: REQUIRED, the field with data
     Returns:
         None, raises ValidationError if failed
     """
-    # jank to get the fireign key that we need back into the field
+    # jank to get the foreign key that we need back into the field
     if current_app.config["REQUIRE_PLAY_KEY"]:
         field.data = PlayKey.key_is_valid(key_string=field.data)
-    return
+    return True
+
+class CustomRecaptcha(Recaptcha):
+    def __call__(self, form, field):
+        if not current_app.config.get("RECAPTCHA_ENABLE", False):
+            return True
+        return super(CustomRecaptcha, self).__call__(form, field)
 
 
 class CustomUserManager(UserManager):
     def customize(self, app):
         self.RegisterFormClass = CustomRegisterForm
+        self.LoginFormClass = CustomLoginForm
+        self.ChangePasswordFormClass = ColonlessChangePasswordForm
 
-
-class CustomRegisterForm(FlaskForm):
-    """Registration form"""
-    next = HiddenField()
-    reg_next = HiddenField()
-
-    # Login Info
-    email = StringField(
-        'E-Mail',
-        validators=[
-            Optional(),
-            validators.Email('Invalid email address'),
-            unique_email_validator,
-        ]
-    )
-
-    username = StringField(
-        'Username',
-        validators=[
-            DataRequired(),
-            unique_username_validator,
-        ]
-    )
-
+class CustomRegisterForm(RegisterForm):
     play_key_id = StringField(
         'Play Key',
-        validators=[
-            Optional(),
-            validate_play_key,
-        ]
+        validators=[validate_play_key]
     )
+    recaptcha = RecaptchaField(
+        validators=[CustomRecaptcha()]
+    )
+    password=PasswordField(
+		'Password',
+		validators=[DataRequired(), password_check]
+	)
 
-    password = PasswordField('Password', validators=[
-        DataRequired(),
-        password_validator,
-        validators.length(max=40, message="The maximum length of the password is 40 characters due to game client limitations")
-    ])
-    retype_password = PasswordField('Retype Password', validators=[
-        validators.EqualTo('password', message='Passwords did not match'),
-        validators.length(max=40, message="The maximum length of the password is 40 characters due to game client limitations")
-    ])
-
-    invite_token = HiddenField('Token')
-
-    submit = SubmitField('Register')
-
+class CustomLoginForm(LoginForm):
+    recaptcha = RecaptchaField(
+        validators=[CustomRecaptcha()]
+    )
 
 class CreatePlayKeyForm(FlaskForm):
 
@@ -221,3 +221,9 @@ class CharXMLUploadForm(FlaskForm):
     )
 
     submit = SubmitField('Submit')
+
+class ColonlessChangePasswordForm(ChangePasswordForm):
+    new_password = PasswordField(
+        'New Password',
+        validators=[validators.DataRequired(), password_check]
+    )

app/luclient.py

@@ -138,14 +138,14 @@ def brick_list():
     brick_list = []
     if len(brick_list) == 0:
         suffixes = [".g", ".g1", ".g2", ".g3", ".xml"]
-        res = pathlib.Path(f"{current_app.config['CLIENT_LOCATION']}res/")
+        cache = pathlib.Path(f"{current_app.config['CACHE_LOCATION']}")
         # Load g files
-        for path in res.rglob("*.*"):
+        for path in cache.rglob("*.*"):
             if str(path.suffix) in suffixes:
                 brick_list.append(
                     {
                         "type": "file",
-                        "name": str(path.as_posix()).replace("{current_app.config['CLIENT_LOCATION']}res/", "")
+                        "name": str(path.as_posix()).replace(f"{current_app.config['CACHE_LOCATION']}", "")
                     }
                 )
     response = make_response(json.dumps(brick_list))
@@ -157,7 +157,7 @@ def brick_list():
 @luclient_blueprint.route('/ldddb/<path:req_path>')
 def dir_listing(req_path):
     # Joining the base and the requested path
-    rel_path = pathlib.Path(str(pathlib.Path(f"{current_app.config['CLIENT_LOCATION']}res/{req_path}").resolve()))
+    rel_path = pathlib.Path(str(pathlib.Path(f"{current_app.config['CACHE_LOCATION']}/{req_path}").resolve()))
     # Return 404 if path doesn't exist
     if not rel_path.exists():
         return abort(404)

app/main.py

@@ -2,16 +2,12 @@ from flask import render_template, Blueprint, send_from_directory
 from flask_user import current_user, login_required
 
 from app.models import Account, CharacterInfo, ActivityLog
-from app.schemas import AccountSchema, CharacterInfoSchema
 
 import datetime
 import time
 
 main_blueprint = Blueprint('main', __name__)
 
-account_schema = AccountSchema()
-char_info_schema = CharacterInfoSchema()
-
 
 @main_blueprint.route('/', methods=['GET'])
 def index():

app/models.py

@@ -4,10 +4,9 @@ from flask_user import UserMixin
 from wtforms import ValidationError
 
 import logging
-from flask_sqlalchemy import BaseQuery
+from flask_sqlalchemy.query import Query
 from sqlalchemy.dialects import mysql
 from sqlalchemy.exc import OperationalError, StatementError
-from sqlalchemy.types import JSON
 from time import sleep
 import random
 import string
@@ -15,7 +14,7 @@ import string
 
 # retrying query to work around python trash collector
 # killing connections of other gunicorn workers
-class RetryingQuery(BaseQuery):
+class RetryingQuery(Query):
     __retry_count__ = 3
 
     def __init__(self, *args, **kwargs):
@@ -1018,7 +1017,7 @@ class Reports(db.Model):
     __tablename__ = 'reports'
 
     data = db.Column(
-        JSON(),
+        mysql.MEDIUMBLOB(),
         nullable=False
     )
 

app/properties.py

@@ -13,7 +13,6 @@ from flask_user import login_required, current_user
 from datatables import ColumnDT, DataTables
 import time
 from app.models import Property, db, UGC, CharacterInfo, PropertyContent, Account, Mail
-from app.schemas import PropertySchema
 from app import gm_level, log_audit
 from app.luclient import query_cdclient
 from app.forms import RejectPropertyForm
@@ -24,9 +23,6 @@ import pathlib
 
 property_blueprint = Blueprint('properties', __name__)
 
-property_schema = PropertySchema()
-
-
 @property_blueprint.route('/', methods=['GET'])
 @login_required
 @gm_level(3)

app/reports.py

@@ -4,8 +4,9 @@ from app.models import CharacterInfo, Account, CharacterXML, Reports
 from app.luclient import get_lot_name
 from app import gm_level, scheduler
 from sqlalchemy.orm import load_only
-import datetime
-import xmltodict
+import xmltodict, gzip, json, datetime
+from collections import OrderedDict
+import xml.etree.ElementTree as ET
 
 reports_blueprint = Blueprint('reports', __name__)
 
@@ -44,6 +45,8 @@ def index():
 @gm_level(3)
 def items_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "items").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/items/by_date.html.j2', data=data, date=date)
 
 
@@ -62,6 +65,8 @@ def items_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
         for key in entry.data:
             items[key] = get_lot_name(key)
@@ -104,6 +109,8 @@ def items_graph(start, end):
 @gm_level(3)
 def currency_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "currency").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/currency/by_date.html.j2', data=data, date=date)
 
 
@@ -121,6 +128,8 @@ def currency_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
     for character in characters:
         data = []
@@ -155,6 +164,8 @@ def currency_graph(start, end):
 @gm_level(3)
 def uscore_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "uscore").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/uscore/by_date.html.j2', data=data, date=date)
 
 
@@ -172,6 +183,8 @@ def uscore_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
     for character in characters:
         data = []
@@ -228,27 +241,27 @@ def gen_item_report():
             for char_xml in char_xmls:
                 name = CharacterInfo.query.filter(CharacterInfo.id == char_xml.id).first().name
                 try:
-                    character_json = xmltodict.parse(
-                        char_xml.xml_data,
-                        attr_prefix="attr_"
-                    )
-                    for inv in character_json["obj"]["inv"]["items"]["in"]:
-                        if "i" in inv.keys() and type(inv["i"]) == list and (int(inv["attr_t"]) == 0 or int(inv["attr_t"]) == 1):
-                            for item in inv["i"]:
-                                if item["attr_l"] in report_data:
-                                    report_data[item["attr_l"]]["item_count"] = report_data[item["attr_l"]]["item_count"] + int(item["attr_c"])
+                    xml_data = ET.fromstring(char_xml.xml_data)
+                    inventories = xml_data.findall(".//inv/items/in")
+                    for inv in inventories:
+                        if inv.attrib["t"] in ["0", "1"]:
+                            for item in inv.findall("i"):
+                                item_attr_l = item.attrib["l"]
+                                item_attr_c = int(item.attrib["c"]) if "c" in item.attrib else 1
+                                if item_attr_l in report_data:
+                                    report_data[item_attr_l]["item_count"] = report_data[item_attr_l]["item_count"] + item_attr_c
                                 else:
-                                    report_data[item["attr_l"]] = {"item_count": int(item["attr_c"]), "chars": {}}
-                                if name in report_data[item["attr_l"]]["chars"]:
-                                    report_data[item["attr_l"]]["chars"][name] = report_data[item["attr_l"]]["chars"][name] + int(item["attr_c"])
+                                    report_data[item_attr_l] = {"item_count": item_attr_c, "chars": {}}
+                                if name in report_data[item_attr_l]["chars"]:
+                                    report_data[item_attr_l]["chars"][name] = report_data[item_attr_l]["chars"][name] + item_attr_c
                                 else:
-                                    report_data[item["attr_l"]]["chars"][name] = int(item["attr_c"])
+                                    report_data[item_attr_l]["chars"][name] = item_attr_c
                 except Exception as e:
                     current_app.logger.error(f"REPORT::ITEMS - ERROR PARSING CHARACTER {char_xml.id}")
                     current_app.logger.error(f"REPORT::ITEMS - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="items",
                 date=date
             )
@@ -286,17 +299,15 @@ def gen_currency_report():
 
             for character in characters:
                 try:
-                    character_json = xmltodict.parse(
-                        character.xml_data,
-                        attr_prefix="attr_"
-                    )
-                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(character_json["obj"]["char"]["attr_cc"])
+                    xml_data = ET.fromstring(character.xml_data)
+                    char = xml_data.find(".//char")
+                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(char.attrib.get("cc"))
                 except Exception as e:
                     current_app.logger.error(f"REPORT::CURRENCY - ERROR PARSING CHARACTER {character.id}")
                     current_app.logger.error(f"REPORT::CURRENCY - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="currency",
                 date=date
             )
@@ -334,17 +345,15 @@ def gen_uscore_report():
 
             for character in characters:
                 try:
-                    character_json = xmltodict.parse(
-                        character.xml_data,
-                        attr_prefix="attr_"
-                    )
-                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(character_json["obj"]["char"]["attr_ls"])
+                    xml_data = ET.fromstring(character.xml_data)
+                    char = xml_data.find(".//char")
+                    report_data[CharacterInfo.query.filter(CharacterInfo.id == character.id).first().name] = int(char.attrib.get("ls"))
                 except Exception as e:
                     current_app.logger.error(f"REPORT::U-SCORE - ERROR PARSING CHARACTER {character.id}")
                     current_app.logger.error(f"REPORT::U-SCORE - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="uscore",
                 date=date
             )

app/schemas.py

@@ -1,130 +0,0 @@
-from flask_marshmallow import Marshmallow
-from app.models import (
-    PlayKey,
-    PetNames,
-    Mail,
-    UGC,
-    PropertyContent,
-    Property,
-    CharacterXML,
-    CharacterInfo,
-    Account,
-    AccountInvitation,
-    ActivityLog,
-    CommandLog
-)
-ma = Marshmallow()
-
-
-class PlayKeySchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = PlayKey
-        include_relationships = False
-        load_instance = True
-        include_fk = True
-
-
-class PetNamesSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = PetNames
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-
-class MailSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = Mail
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-
-class UGCSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = UGC
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-
-class PropertyContentSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = PropertyContent
-        include_relationships = True
-        load_instance = True
-        include_fk = True
-
-    ugc = ma.Nested(UGCSchema)
-
-
-class PropertySchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = Property
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-    properties_contents = ma.Nested(PropertyContentSchema, many=True)
-
-
-class CharacterXMLSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = CharacterXML
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-
-class CharacterInfoSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = CharacterInfo
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-    charxml = ma.Nested(CharacterXMLSchema)
-    properties_owner = ma.Nested(PropertySchema, many=True)
-    pets = ma.Nested(PetNamesSchema, many=True)
-    mail = ma.Nested(MailSchema, many=True)
-
-
-class AccountSchema(ma.SQLAlchemyAutoSchema):
-    class Meta:
-        model = Account
-        include_relationships = False
-        load_instance = True
-        include_fk = False
-
-    play_key = ma.Nested(PlayKeySchema)
-    charinfo = ma.Nested(CharacterInfoSchema, many=True)
-
-
-class AccountInvitationSchema(ma.SQLAlchemyAutoSchema): #  noqa
-    class Meta:
-        model = AccountInvitation
-        include_relationships = True
-        load_instance = True
-        include_fk = True
-
-    invite_by_user = ma.Nested(AccountSchema)
-
-
-class ActivityLogSchema(ma.SQLAlchemyAutoSchema): #  noqa
-    class Meta:
-        model = ActivityLog
-        include_relationships = True
-        load_instance = True
-        include_fk = True
-
-    character = ma.Nested(CharacterInfoSchema())
-
-
-class CommandLogSchema(ma.SQLAlchemyAutoSchema): #  noqa
-    class Meta:
-        model = CommandLog
-        include_relationships = True
-        load_instance = True
-        include_fk = True
-
-    character = ma.Nested(CharacterInfoSchema())

app/settings_example.py

@@ -61,3 +61,13 @@ USER_AFTER_LOGOUT_ENDPOINT = "main.index"
 
 # Option will be removed once this feature is full implemeted
 ENABLE_CHAR_XML_UPLOAD = False
+
+# Recaptcha settings
+# See: https://flask-wtf.readthedocs.io/en/1.2.x/form/#recaptcha
+RECAPTCHA_ENABLE = False
+RECAPTCHA_PUBLIC_KEY = ''
+RECAPTCHA_PRIVATE_KEY = ''
+# Optional
+# RECAPTCHA_API_SERVER = ''
+# RECAPTCHA_PARAMETERS = ''
+RECAPTCHA_DATA_ATTRS = {'theme': 'white', 'size': 'invisible'}

app/templates/accounts/index.html.j2

@@ -14,17 +14,13 @@
         <tr>
             <th>Actions</th>
             <th>Name</th>
-            {% if config.USER_ENABLE_EMAIL %}
-              <th>Email</th>
-            {% endif %}
+            <th>Email</th>
             <th>GM Level</th>
             <th>Locked</th>
             <th>Banned</th>
             <th>Muted</th>
             <th>Registered</th>
-            {% if config.USER_ENABLE_EMAIL %}
-              <th>Email Confirmed</th>
-            {% endif %}
+            <th>Email Confirmed</th>
         </tr>
     </thead>
     <tbody></tbody>

app/templates/admin/dashboard.html.j2

@@ -1,188 +0,0 @@
-{% extends "bootstrap/base.html" %}
-{% block title %}Key Creation{% endblock %}
-
-{% block navbar %}
-<nav class="navbar navbar-default">
-  <div class="container-fluid">
-    <div class="navbar-header">
-      <a class="navbar-brand" href="{{ url_for('dashboard') }}">Dashboard</a>
-    </div>
-    <ul class="nav navbar-nav">
-    </ul>
-    <ul class="nav navbar-nav" style="float: right">
-      <li class="active"><a href="#">Welcome {{ current_user.username }}!</a></li>
-      <li><a href="{{ url_for('logout') }}">Logout</a></li>
-    </ul>
-  </div>
-</nav>
-{% endblock navbar %}}
-
-{% block content %}
-{# LOGO #}
-<div class="container" style="margin-top: 50px">
-
-    {# Display logo #}
-    <div style="margin-bottom: 50px">
-        <img
-                src="{{ url_for('static', filename=resources.LOGO) }}"
-                class="center-block img-responsive mx-auto d-block"
-                alt="Logo"
-                width="100"
-                height="100"
-        >
-    </div>
-
-</div>
-
-{# Key creation #}
-<div class="container">
-    <div class="text-center">
-        <h3>Key Creation</h3>
-    </div>
-
-    <div class="col-lg-3">
-    </div>
-    <div class="col-lg-6">
-
-    {# If the error value is set, display the error in red text #}
-    {% if error %}
-        <div class="alert alert-danger">
-            {{ error }}
-        </div>
-    {% endif %}
-
-    {# If the message value is set, display the message in green text #}
-    {% if message %}
-        <div class="alert alert-success">
-            {{ message }}
-        </div>
-    {% endif %}
-
-    {# Form which takes in Admin Username, Admin Password, and the amount of keys to create. #}
-    <form action="{{ url_for('dashboard') }}" method="post">
-        {# Key count input #}
-        <div class="form-group">
-            <label for="key_count">Generate keys</label>
-            <input type="number" class="form-control" name="key_count" placeholder="Enter number of keys...">
-            <small class="form-text text-muted">Number of keys to create.</small>
-        </div>
-        
-        {# Submit button #}
-        <div class="form-group">
-            <button type="submit" class="btn btn-primary">Generate Keys</button>
-        </div>
-    </form>
-
-    {# If the keys value is set, create a list for each key in keys #}
-    {% if keys %}
-        <div class="alert alert-success">
-            <ul>
-                {% for key in keys %}
-                    <li>{{ key }}</li>
-                {% endfor %}
-            </ul>
-        </div>
-    {% endif %}
-    </div>
-    <div class="col-lg-3">
-    </div>
-    </div>
-</div>
-
-{# Activity graphs #}
-<div class="container">
-
-    <div class="text-center">
-        <h3>Activity</h3>
-    </div>
-
-    <div class="col-lg-3">
-    </div>
-
-    <div class="col-lg-6">
-
-    <canvas id="sessions_graph" width="400" height="400"></canvas>
-    <canvas id="play_time_graph" width="400" height="400"></canvas>
-    <canvas id="zone_play_time_graph" width="400" height="400"></canvas>
-
-    </div>
-
-    <div class="col-lg-3">
-    </div>
-</div>
-{% endblock %}
-
-{% block scripts %}
-<script src="https://cdn.jsdelivr.net/npm/chart.js@3.5.1/dist/chart.min.js"></script>
-<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
-
-<script>
-// Make a get request to the server to load the activity data
-$.get("{{ url_for('load_activities') }}", function(data) {
-
-});
-
-// Make a get request to the server to get the activity data for "sessions"
-$.get("{{ url_for('activity_data', name='sessions') }}", function(data) {
-    // Load data as a json object
-    data = JSON.parse(data);
-    var ctx = document.getElementById('sessions_graph').getContext('2d');
-    var myChart = new Chart(ctx, {
-        type: 'line',
-        data: {
-            labels: data.labels,
-            datasets: data.datasets
-        },
-        options: {
-            scales: {
-                y: {
-                    beginAtZero: true
-                }
-            }
-        }
-    });
-});
-
-// Make a get request to the server to get the activity data for "play_time"
-$.get("{{ url_for('activity_data', name='play_time') }}", function(data) {
-    // Load data as a json object
-    data = JSON.parse(data);
-    var ctx = document.getElementById('play_time_graph').getContext('2d');
-    var myChart = new Chart(ctx, {
-        type: 'line',
-        data: {
-            labels: data.labels,
-            datasets: data.datasets
-        },
-        options: {
-            scales: {
-                y: {
-                    beginAtZero: true
-                }
-            }
-        }
-    });
-});
-
-// Make a get request to the server to get the activity data for "zone_play_time"
-$.get("{{ url_for('activity_data', name='zone_play_time') }}", function(data) {
-    // Load data as a json object
-    data = JSON.parse(data);
-    var ctx = document.getElementById('zone_play_time_graph').getContext('2d');
-    var myChart = new Chart(ctx, {
-        type: 'line',
-        data: {
-            labels: data.labels,
-            datasets: data.datasets
-        },
-        options: {
-            scales: {
-                y: {
-                    beginAtZero: true
-                }
-            }
-        }
-    });
-});
-</script>
-{% endblock scripts %}}

app/templates/base.html.j2

@@ -84,6 +84,35 @@
   <script type="text/javascript" src="{{ url_for('static', filename='bootstrap-4.2.1/js/bootstrap.bundle.min.js') }}"></script>
   <script type="text/javascript" src="{{ url_for('static', filename='datatables/datatables.min.js') }}"></script>
   <script type="sytylesheet" src="{{ url_for('static', filename='datatables/datatables.min.css') }}"></script>
+  <script>
+    // set the active nav-link item
+    $(function () {
+      let target_nav = '#{{request.endpoint}}'.replace('\.', '-');
+      $(target_nav).addClass('active');
+    });
+    // make tooltips with data work
+    $(function () {
+      $('[data-toggle="tooltip"]').tooltip()
+    })
+    function setInnerHTML(elm, html) {
+      elm.innerHTML = html;
+      $("body").tooltip({ selector: '[data-toggle=tooltip]' });
+      Array.from(elm.querySelectorAll("script"))
+        .forEach( oldScriptEl => {
+          const newScriptEl = document.createElement("script");
+          
+          Array.from(oldScriptEl.attributes).forEach( attr => {
+            newScriptEl.setAttribute(attr.name, attr.value) 
+          });
+          
+          const scriptText = document.createTextNode(oldScriptEl.innerHTML);
+          newScriptEl.appendChild(scriptText);
+          
+          oldScriptEl.parentNode.replaceChild(newScriptEl, oldScriptEl);
+          
+      });
+    }
+  </script>
 {% endblock %}
 
 </body>

app/templates/character/upload.html.j2

@@ -9,10 +9,22 @@
 {% endblock content_before %}
 
 {% block content %}
-  <h3 style="color: orange;">PROCEED WITH CAUTION</h3>
+  <style>
+      .blink {
+          animation: blinker .5s linear infinite;
+          color: red;
+          font-family: sans-serif;
+      }
+      @keyframes blinker {
+          50% {
+              opacity: 0;
+          }
+      }
+  </style>
+  <h3 class="text-center blink">PROCEED WITH CAUTION</h3>
   <form method=post>
     {{ form.csrf_token }}
-    <div class="card shadow-sm mx-auto pb-3 bg-dark border-primary" style="width: 20rem;">
+    <div class="card shadow-sm mx-auto pb-3 bg-dark border-primary" >
       <div class="card-body">
         {{ helper.render_field(form.char_xml) }}
         {{ helper.render_submit_field(form.submit) }}

app/templates/character/view.html.j2

@@ -15,8 +15,9 @@
         {% include 'partials/_character.html.j2' %}
       {% endwith %}
     </div>
-    <div class="col-sm">
-      {% include 'partials/_charxml.html.j2'%}
+    <div class="col-sm" id="charxml">
+      Loading Character Data 
+      {% include 'partials/_loading.html' %}
     </div>
   </div>
 {% endblock content %}
@@ -32,3 +33,14 @@
     {% endfor %}
   </div>
 {% endblock content_after %}
+
+{% block js %}
+  {{ super() }}
+  <script>
+    fetch({{ url_for("characters.chardata", id=character_data.id)|tojson }})
+        .then(response => response.text())
+        .then(text => {
+          setInnerHTML(document.getElementById("charxml"), text);
+        })
+  </script>
+{% endblock js %}

app/templates/flask_user/login.html

@@ -60,7 +60,12 @@
 
     {# Remember me #}
     {% if user_manager.USER_ENABLE_REMEMBER_ME %}
-    {{ render_checkbox_field(login_form.remember_me, tabindex=130) }}
+        {{ render_checkbox_field(login_form.remember_me, tabindex=130) }}
+    {% endif %}
+
+    {# recaptcha #}
+    {% if config.RECAPTCHA_ENABLE %}
+        {{ render_field(form.recaptcha, tabindex=250) }}
     {% endif %}
 
     {# Submit button #}

app/templates/flask_user/login_or_register.html

@@ -29,6 +29,11 @@
     {{ render_checkbox_field(login_form.remember_me, tabindex=130) }}
     {% endif %}
 
+    {# recaptcha #}
+    {% if config.RECAPTCHA_ENABLE %}
+        {{ form.recaptcha }}
+    {% endif %}
+
     {# Submit button #}
     {{ render_submit_field(login_form.submit, tabindex=180) }}
 </form>
@@ -63,6 +68,11 @@
         {{ render_field(register_form.retype_password, tabindex=240) }}
     {% endif %}
 
+    {# recaptcha #}
+    {% if config.RECAPTCHA_ENABLE %}
+        {{ register_form.recaptcha }}
+    {% endif %}
+
     {{ render_submit_field(register_form.submit, tabindex=280) }}
 </form>
 

app/templates/flask_user/register.html

@@ -47,6 +47,11 @@
         {{ render_field(form.retype_password, tabindex=240) }}
     {% endif %}
 
+    {# recaptcha #}
+    {% if config.RECAPTCHA_ENABLE %}
+        {{ render_field(form.recaptcha, tabindex=250) }}
+    {% endif %}
+
     {{ render_submit_field(form.submit, tabindex=280) }}
 </form>
 

app/templates/main/about.html.j2

@@ -82,7 +82,6 @@
         </div>
       {% endif %}
 
-
       <div class="row">
         <div class="col text-right">
           Source

app/templates/partials/_account.html.j2

@@ -116,7 +116,7 @@
       </div>
     {% endif %}
 
-    {% if account_data.play_key and current_user.gm_level > 3 and config.REQUIRE_PLAY_KEY %}
+    {% if account_data.play_key and config.REQUIRE_PLAY_KEY %}
       <hr class="bg-primary"/>
       <div class="row">
         <div class="col text-center">
@@ -133,26 +133,28 @@
           {{ account_data.play_key.key_string }}
         </div>
       </div>
-      <div class="row">
-        <div class="col text-right">
-          Uses Left:
+      {% if current_user.gm_level > 3 %}
+        <div class="row">
+          <div class="col text-right">
+            Uses Left:
+          </div>
+          <div class="col">
+            {{ account_data.play_key.key_uses }}
+          </div>
         </div>
-        <div class="col">
-          {{ account_data.play_key.key_uses }}
+        <div class="row">
+          <div class="col text-right">
+            Active:
+          </div>
+          <div class="col">
+            {% if account_data.active %}
+              <h5 class="far fa-check-square text-success"></h5>
+            {% else %}
+              <h5 class="far fa-times-circle text-danger"></h5>
+            {% endif %}
+          </div>
         </div>
-      </div>
-      <div class="row">
-        <div class="col text-right">
-          Active:
-        </div>
-        <div class="col">
-          {% if account_data.active %}
-            <h5 class="far fa-check-square text-success"></h5>
-          {% else %}
-            <h5 class="far fa-times-circle text-danger"></h5>
-          {% endif %}
-        </div>
-      </div>
+      {% endif %}
     {% endif %}
 
     {% if current_user.id != account_data.id and current_user.gm_level > 3 %}

app/templates/partials/_charxml.html.j2

@@ -11,9 +11,11 @@
       <div class="col text-right">
         U-Score: {{ character_json.obj.char.attr_ls }}
       </div>
-      <div class="col">
-        Level: {{ character_json.obj.lvl.attr_l }}
-      </div>
+      {% if "lvl" in character_json.obj %}
+        <div class="col">
+          Level: {{ character_json.obj.lvl.attr_l }}
+        </div>
+      {% endif %}
     </div>
     <br/>
     <div class="row">
@@ -112,99 +114,33 @@
         <div class="tab-content mt-3" id="nav-invContent">
           <div class="tab-pane fade show active" id="nav-items" role="tabpanel" aria-labelledby="nav-items-tab">
             {# Inv ID 0 - Index: 0 #}
-            {% for item in character_json.obj.inv.holdings.in %}
-              {% if item.attr_t == "0" %}
-                {% if item.i is iterable and (item.i is not string and item.i is not mapping) %}
-                  {% for inv_item in item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endfor %}
-                {% else %}
-                  {% with inv_item=item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endwith %}
-                {% endif %}
-              {% endif %}
-            {% endfor %}
+            Loading Inventory 
+            {% include 'partials/_loading.html' %}
           </div>
           <div class="tab-pane fade" id="nav-vault" role="tabpanel" aria-labelledby="nav-vault-tab">
             {# Inv ID 1 - Index: 1 #}
-            {% for item in character_json.obj.inv.holdings.in %}
-              {% if item.attr_t == "1" %}
-                {% if item.i is iterable and (item.i is not string and item.i is not mapping) %}
-                  {% for inv_item in item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endfor %}
-                {% else %}
-                  {% with inv_item=item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endwith %}
-                {% endif %}
-              {% endif %}
-            {% endfor %}
+            Loading Inventory 
+            {% include 'partials/_loading.html' %}
           </div>
           <div class="tab-pane fade" id="nav-vault-models" role="tabpanel" aria-labelledby="nav-vault-models-tab">
             {# Inv ID 14 - Index: 10 #}
-            {% for item in character_json.obj.inv.holdings.in %}
-              {% if item.attr_t == "14" %}
-                {% if item.i is iterable and (item.i is not string and item.i is not mapping) %}
-                  {% for inv_item in item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endfor %}
-                {% else %}
-                  {% with inv_item=item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endwith %}
-                {% endif %}
-              {% endif %}
-            {% endfor %}
+            Loading Inventory 
+            {% include 'partials/_loading.html' %}
           </div>
           <div class="tab-pane fade" id="nav-bricks" role="tabpanel" aria-labelledby="nav-bricks-tab">
             {# Inv ID 2 - Index: 2 #}
-            {% for item in character_json.obj.inv.holdings.in %}
-              {% if item.attr_t == "2" %}
-                {% if item.i is iterable and (item.i is not string and item.i is not mapping) %}
-                  {% for inv_item in item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endfor %}
-                {% else %}
-                  {% with inv_item=item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endwith %}
-                {% endif %}
-              {% endif %}
-            {% endfor %}
+            Loading Inventory 
+            {% include 'partials/_loading.html' %}
           </div>
           <div class="tab-pane fade" id="nav-models" role="tabpanel" aria-labelledby="nav-models-tab">
             {# Inv ID 5 - Index: 6 #}
-            {% for item in character_json.obj.inv.holdings.in %}
-              {% if item.attr_t == "5" %}
-                {% if item.i is iterable and (item.i is not string and item.i is not mapping) %}
-                  {% for inv_item in item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endfor %}
-                {% else %}
-                  {% with inv_item=item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endwith %}
-                {% endif %}
-              {% endif %}
-            {% endfor %}
+            Loading Inventory 
+            {% include 'partials/_loading.html' %}
           </div>
           <div class="tab-pane fade" id="nav-behaviors" role="tabpanel" aria-labelledby="nav-behaviors-tab">
             {# Inv ID 7 - Index: 8 #}
-            {% for item in character_json.obj.inv.holdings.in %}
-              {% if item.attr_t == "7" %}
-                {% if item.i is iterable and (item.i is not string and item.i is not mapping) %}
-                  {% for inv_item in item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endfor %}
-                {% else %}
-                  {% with inv_item=item.i %}
-                    {% include 'partials/charxml/_inv_grid.html.j2' %}
-                  {% endwith %}
-                {% endif %}
-              {% endif %}
-            {% endfor %}
+            Loading Inventory 
+            {% include 'partials/_loading.html' %}
           </div>
         </div>
       </div>
@@ -263,3 +199,40 @@
     </div>
   </div>
 </div>
+<script>
+  fetch({{ url_for("characters.inventory", id=character_data.id, inventory_id=0)|tojson }})
+      .then(response => response.text())
+      .then(text => {
+        setInnerHTML(document.getElementById("nav-items"), text);
+      })
+
+  fetch({{ url_for("characters.inventory", id=character_data.id, inventory_id=1)|tojson }})
+      .then(response => response.text())
+      .then(text => {
+        setInnerHTML(document.getElementById("nav-vault"), text);
+      })
+
+  fetch({{ url_for("characters.inventory", id=character_data.id, inventory_id=14)|tojson }})
+      .then(response => response.text())
+      .then(text => {
+        setInnerHTML(document.getElementById("nav-vault-models"), text);
+      })
+
+  fetch({{ url_for("characters.inventory", id=character_data.id, inventory_id=2)|tojson }})
+      .then(response => response.text())
+      .then(text => {
+        setInnerHTML(document.getElementById("nav-bricks"), text);
+      })
+
+  fetch({{ url_for("characters.inventory", id=character_data.id, inventory_id=5)|tojson }})
+      .then(response => response.text())
+      .then(text => {
+        setInnerHTML(document.getElementById("nav-models"), text);
+      })
+
+  fetch({{ url_for("characters.inventory", id=character_data.id, inventory_id=7)|tojson }})
+      .then(response => response.text())
+      .then(text => {
+        setInnerHTML(document.getElementById("nav-behaviors"), text);
+      })
+</script>

app/templates/partials/_loading.html

@@ -0,0 +1,21 @@
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s;">
+    <span class="sr-only">Loading 0</span>
+</div>
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s; animation-delay: 1s;">
+    <span class="sr-only">Loading 1</span>
+</div>
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s; animation-delay: 2s;">
+    <span class="sr-only">Loading 2</span>
+</div>
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s; animation-delay: 3s;">
+    <span class="sr-only">Loading 3</span>
+</div>
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s; animation-delay: 4s;">
+    <span class="sr-only">Loading 4</span>
+</div>
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s; animation-delay: 5s;">
+    <span class="sr-only">Loading 5</span>
+</div>
+<div class="spinner-grow text-light" role="status" style="animation-duration: 6s; animation-delay: 6s;">
+    <span class="sr-only">Loading 6</span>
+</div>

app/templates/partials/_property.html.j2

@@ -77,6 +77,14 @@
           {{ property.performance_cost }}
       </div>
     </div>
+    <div class="row">
+      <div class="col text-right">
+        Clone ID:
+      </div>
+      <div class="col">
+          {{ property.clone_id }}
+      </div>
+    </div>
     {% if request.endpoint != "properties.view" %}
       <br/>
       <div class="row">

app/templates/partials/charxml/_inv_grid.html.j2

@@ -4,25 +4,29 @@
     alt="{{ inv_item.attr_l|get_lot_name }}"
     class="border p-1 border-primary rounded m-1"
     width="60"
-    {% if inv_item.attr_eq == "true" %}style="background-color:#d16f05;"{% endif %}
+    {% if 'attr_eq' in inv_item %}
+      {% if inv_item.attr_eq == "true" %}style="background-color:#d16f05;"{% endif %}
+    {% endif %}
     height="60"
     data-html="true"
     data-toggle="tooltip"
     data-placement="left"
     title="{% include 'partials/charxml/_item_tooltip.html.j2' %}"
   >
-  {% if inv_item.attr_c != "1" %}
+  {% if 'attr_c' in inv_item %}
     <span class="inventory-count text-bold">
       {%if inv_item.attr_c|int > 999 %}
         +999
-      {% else %}
+      {% elif inv_item.attr_c|int > 1 %}
         {{ inv_item.attr_c }}
       {% endif %}
     </span>
   {% endif %}
-  {% if inv_item.attr_b == "true" %}
-    <span class="inventory-lock">
-      <i class='fas fa-lock'></i>
-    </span>
+  {% if 'attr_b' in inv_item %}
+    {% if inv_item.attr_b == "true" %}
+      <span class="inventory-lock">
+        <i class='fas fa-lock'></i>
+      </span>
+    {% endif %}
   {% endif %}
 </div>

app/templates/partials/charxml/_inventory.html.j2

@@ -0,0 +1,9 @@
+{% if inventory.i is iterable and (inventory.i is not string and inventory.i is not mapping) %}
+    {% for inv_item in inventory.i %}
+        {% include 'partials/charxml/_inv_grid.html.j2' %}
+    {% endfor %}
+{% else %}
+    {% with inv_item=inventory.i %}
+        {% include 'partials/charxml/_inv_grid.html.j2' %}
+    {% endwith %}
+{% endif %}

app/templates/partials/charxml/_item_tooltip.html.j2

@@ -66,7 +66,8 @@
   {% endwith %}
 {% endif %}
 
-{%if inv_item.attr_c|int > 999 %}
-  <br />Count: {{ inv_item.attr_c|numberFormat }}
+{% if 'attr_c' in inv_item %}
+  {%if inv_item.attr_c|int > 999 %}
+    <br />Count: {{ inv_item.attr_c|numberFormat }}
+  {% endif %}
 {% endif %}
-

migrations/env.py

@@ -22,8 +22,8 @@ logger = logging.getLogger('alembic.env')
 # target_metadata = mymodel.Base.metadata
 config.set_main_option(
     'sqlalchemy.url',
-    str(current_app.extensions['migrate'].db.get_engine().url).replace(
-        '%', '%%'))
+    current_app.config.get('SQLALCHEMY_DATABASE_URI')
+)
 target_metadata = current_app.extensions['migrate'].db.metadata
 
 # other values from the config, defined by the needs of env.py,

migrations/versions/1164e037907f_compressss_reports.py

@@ -0,0 +1,164 @@
+"""compressss reports
+
+Revision ID: 1164e037907f
+Revises: a6e42ef03da7
+Create Date: 2023-11-18 01:38:00.127472
+
+"""
+from alembic import op
+from sqlalchemy.orm.session import Session
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+from sqlalchemy.types import JSON
+from sqlalchemy.ext.declarative import declarative_base
+import gzip
+import json
+
+
+Base = declarative_base()
+
+
+# revision identifiers, used by Alembic.
+revision = '1164e037907f'
+down_revision = 'a6e42ef03da7'
+branch_labels = None
+depends_on = None
+
+class ReportsUpgradeNew(Base):
+    __tablename__ = 'reports'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        mysql.MEDIUMBLOB(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+    def save(self):
+        sa.session.add(self)
+        sa.session.commit()
+        sa.session.refresh(self)
+
+
+class ReportsUpgradeOld(Base):
+    __tablename__ = 'reports_old'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        JSON(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+class ReportsDowngradeOld(Base):
+    __tablename__ = 'reports'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        mysql.MEDIUMBLOB(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+    def save(self):
+        sa.session.add(self)
+        sa.session.commit()
+        sa.session.refresh(self)
+
+
+class ReportsDowngradeNew(Base):
+    __tablename__ = 'reports_old'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        JSON(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.rename_table('reports', 'reports_old')
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    reports = session.query(ReportsUpgradeOld)
+    op.create_table('reports',
+        sa.Column('data', mysql.MEDIUMBLOB(), nullable=False),
+        sa.Column('report_type', sa.VARCHAR(length=35), autoincrement=False, nullable=False),
+        sa.Column('date', sa.Date(), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('report_type', 'date')
+    )
+    # insert records
+    new_reports = []
+    # insert records
+    for report in reports:
+        new_reports.append({
+            "data":gzip.compress(json.dumps(report.data).encode('utf-8')), 
+            "report_type":report.report_type, 
+            "date":report.date
+        })
+    op.bulk_insert(ReportsUpgradeNew.__table__, new_reports)
+    op.drop_table('reports_old')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('reports')
+    op.create_table('reports',
+        sa.Column('data', JSON(), nullable=False),
+        sa.Column('report_type', sa.VARCHAR(length=35), autoincrement=False, nullable=False),
+        sa.Column('date', sa.Date(), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('report_type', 'date')
+    )
+    # ### end Alembic commands ###

migrations/versions/712d42956a47_initial_migration.py

@@ -77,11 +77,11 @@ def upgrade():
             sa.Column('created_at', mysql.TIMESTAMP(), server_default=sa.text('now()'), nullable=False),
             sa.Column('play_key_id', mysql.INTEGER(), nullable=True),
             sa.Column('mute_expire', mysql.BIGINT(unsigned=True), server_default='0', nullable=False),
-            sa.ForeignKeyConstraint(['play_key_id'], ['play_keys.id'], ondelete='CASCADE'),
             sa.PrimaryKeyConstraint('id'),
             sa.UniqueConstraint('name')
         )
 
+    op.create_foreign_key(None, 'accounts', 'play_keys', ['play_key_id'], ['id'], ondelete='CASCADE')
     op.add_column('accounts', sa.Column('active', sa.BOOLEAN(), server_default='1', nullable=False))
     op.add_column('accounts', sa.Column('email_confirmed_at', sa.DateTime(), nullable=True))
     op.add_column('accounts', sa.Column('email', sa.Unicode(length=255), server_default='', nullable=True))

requirements.txt

@@ -1,61 +1,21 @@
 alembic==1.7.5
-APScheduler==3.8.1
-astroid==2.9.1
 autopep8==1.6.0
-bcrypt==3.2.0
-blinker==1.4
-cffi==1.14.6
-click==8.0.1
-colorama==0.4.4
-cryptography==36.0.0
-dnspython==2.1.0
-dominate==2.6.0
+bcrypt==4.0.1
 email-validator==1.1.3
-Flask==2.0.1
+Flask==3.0.0
 Flask-APScheduler==1.12.3
-Flask-Assets==2.0
-Flask-Login==0.5.0
-Flask-Mail==0.9.1
-flask-marshmallow==0.14.0
+Flask-Assets==2.1.0
 Flask-Migrate==3.1.0
-Flask-SQLAlchemy==2.5.1
+Flask-SQLAlchemy==3.1.1
 Flask-User==1.0.2.2
-Flask-WTF==1.0.0
-greenlet==1.1.0
-idna==3.3
-isort==5.10.1
-itsdangerous==2.0.1
-Jinja2==3.0.1
-lazy-object-proxy==1.7.1
+Flask-WTF==1.2.1
+gunicorn==21.2.0
 libsass==0.21.0
-Mako==1.2.2
-MarkupSafe==2.0.1
-marshmallow==3.14.1
-marshmallow-sqlalchemy==0.26.1
-mccabe==0.6.1
-passlib==1.7.4
-platformdirs==2.4.1
-pycodestyle==2.8.0
-pycparser==2.20
-pydocstyle==6.1.1
-pyflakes==2.4.0
-pylama==8.3.3
-pylint==2.12.2
 PyMySQL==1.0.2
-python-dateutil==2.8.2
-pytz==2021.3
-pytz-deprecation-shim==0.1.0.post0
-six==1.16.0
-snowballstemmer==2.2.0
-SQLAlchemy==1.4.22
+SQLAlchemy==2.0.40
 sqlalchemy-datatables==2.0.1
-toml==0.10.2
-tzdata==2021.5
-tzlocal==4.1
-visitor==0.1.3
 Wand==0.6.7
 webassets==2.0
-Werkzeug==2.0.1
-wrapt==1.13.3
+Werkzeug==3.0.1
 WTForms==3.0.0
 xmltodict==0.12.0

wsgi.py

@@ -1,23 +1,19 @@
-from sys import platform
+
 from app import create_app
 
 app = create_app()
 
+
 @app.shell_context_processor
 def make_shell_context():
     """Extend the Flask shell context."""
     return {'app': app}
 
-running_directly = __name__ == "wsgi" or __name__ == "__main__"
-running_under_gunicorn = not running_directly and 'gunicorn' in __name__ and 'linux' in platform
 
-# Configure development running
-if running_directly:
+if __name__ == '__main__':
     with app.app_context():
         app.run(host='0.0.0.0')
-
-# Configure production running
-if running_under_gunicorn:
+else:
     import logging
     from logging.handlers import RotatingFileHandler
     gunicorn_logger = logging.getLogger('gunicorn.error')
@@ -27,7 +23,3 @@ if running_under_gunicorn:
     file_handler.setFormatter(formatter)
     app.logger.addHandler(file_handler)
     app.logger.setLevel(gunicorn_logger.level)
-
-# Error out if nothing has been setup
-if not running_directly and not running_under_gunicorn:
-    raise RuntimeError('Unsupported WSGI server')