update gitignore

Resolves #40

.gitignore

@@ -4,7 +4,7 @@ resources.py
 __pycache__/
 venv/
 static/policy/
-app/static/site.css
+app/static/css/site.css
 app/static/.webassets-cache/**/*
 app/static/brickdb/*
 locale.json
@@ -17,3 +17,5 @@ property_files/*
 *.log
 app/settings.py
 *.exe
+*.csv
+*.sql

README.md

@@ -132,6 +132,7 @@ Now let's open the settings file we just created and configure some of the setti
 
 <br>
 Inside this file is where you can change certain settings like user registration, email support and other things. In this tutorial I will only be focusing on the bare minimum to get up and running, but feel free to adjust what you would like
+
 >*Note: Enabling the email option will require further setup that is outside the scope of this tutorial*
 
 The two important settings to configure are `APP_SECRET_KEY` and `APP_DATABASE_URI`
@@ -204,17 +205,19 @@ res
 We will also need to copy the `CDServer.sqlite` database file from the server to the `~/NexusDashboard/app/luclient/res` folder
 
 Once the file is moved over, you will need to rename it to `cdclient.sqlite`
-`mv ~/NexusDashboard/app/luclient/res/CDServer.sqlite ~/NexusDashboard/app/luclient/res/cdclient.sqlite`
+```bash
+mv ~/NexusDashboard/app/luclient/res/CDServer.sqlite ~/NexusDashboard/app/luclient/res/cdclient.sqlite
+```
 
 
 ##### Remaining Setup
 Run the following commands one at a time
-
-`cd ~/NexusDashboard`
-`pip install -r requirements.txt`
-`pip install gunicorn`
-`flask db upgrade`
-
+```bash
+cd ~/NexusDashboard
+pip install -r requirements.txt
+pip install gunicorn
+flask db upgrade
+```
 ##### Running the site
 You can run the site with
 `gunicorn -b :8000 -w 4 wsgi:app`

app/__init__.py

@@ -1,5 +1,5 @@
 import os
-from flask import Flask, url_for, g, redirect
+from flask import Flask, url_for, redirect
 from functools import wraps
 from flask_assets import Environment
 from webassets import Bundle
@@ -18,15 +18,17 @@ from app.commands import (
     load_property,
     gen_image_cache,
     gen_model_cache,
-    fix_clone_ids
+    fix_clone_ids,
+    parse_lucache,
+    makeup_unlisted_objects,
+    gen_new_locales,
+    xref_scripts
 )
 from app.models import Account, AccountInvitation, AuditLog
 
 import logging
 from logging.handlers import RotatingFileHandler
 
-from werkzeug.exceptions import HTTPException
-
 # Instantiate Flask extensions
 csrf_protect = CSRFProtect()
 scheduler = APScheduler()
@@ -76,12 +78,6 @@ def create_app():
     def debug(text):
         print(text)
 
-    @app.teardown_appcontext
-    def close_connection(exception):
-        cdclient = getattr(g, '_cdclient', None)
-        if cdclient is not None:
-            cdclient.close()
-
     # add the commands to flask cli
     app.cli.add_command(init_db)
     app.cli.add_command(init_accounts)
@@ -89,6 +85,10 @@ def create_app():
     app.cli.add_command(gen_image_cache)
     app.cli.add_command(gen_model_cache)
     app.cli.add_command(fix_clone_ids)
+    app.cli.add_command(parse_lucache)
+    app.cli.add_command(makeup_unlisted_objects)
+    app.cli.add_command(gen_new_locales)
+    app.cli.add_command(xref_scripts)
 
     register_logging(app)
     register_settings(app)
@@ -194,6 +194,9 @@ def register_settings(app):
         'APP_DATABASE_URI',
         app.config['APP_DATABASE_URI']
     )
+    app.config['SQLALCHEMY_BINDS'] = {
+        'cdclient': 'sqlite:///luclient/res/cdclient.sqlite'
+    }
 
     # try to get overides, otherwise just use what we have already
     app.config['USER_ENABLE_REGISTER'] = os.getenv(
@@ -224,14 +227,6 @@ def register_settings(app):
         'ALLOW_ANALYTICS',
         app.config['ALLOW_ANALYTICS']
     )
-    app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
-        "pool_pre_ping": True,
-        "pool_size": 10,
-        "max_overflow": 2,
-        "pool_recycle": 300,
-        "pool_pre_ping": True,
-        "pool_use_lifo": True
-    }
     app.config['MAIL_SERVER'] = os.getenv(
         'MAIL_SERVER',
         app.config['MAIL_SERVER']

app/accounts.py

@@ -169,7 +169,7 @@ def delete(id):
             prop.delete()
         char.delete()
     # This is for GM stuff, it will be permnently delete logs
-    bugs = BugReport.query.filter(BugReport.resolve_by_id == id).all()
+    bugs = BugReport.query.filter(BugReport.resoleved_by_id == id).all()
     for bug in bugs:
         bug.delete()
     audits = AuditLog.query.filter(AuditLog.account_id == id).all()

app/characters.py

@@ -30,14 +30,20 @@ def approve_name(id, action):
     character = CharacterInfo.query.filter(CharacterInfo.id == id).first()
 
     if action == "approve":
-        log_audit(f"Approved ({character.id}){character.pending_name} from {character.name}")
-        flash(
-            f"Approved ({character.id}){character.pending_name} from {character.name}",
-            "success"
-        )
         if character.pending_name:
             character.name = character.pending_name
             character.pending_name = ""
+            log_audit(f"Approved ({character.id}){character.pending_name} from {character.name}")
+            flash(
+                f"Approved ({character.id}){character.pending_name} from {character.name}",
+                "success"
+            )
+        else:
+            log_audit("Cannot make character name empty")
+            flash(
+                "Cannot make character name empty",
+                "danger"
+            )
         character.needs_rename = False
 
     elif action == "rename":

app/commands.py

@@ -15,7 +15,17 @@ from multiprocessing import Pool
 from functools import partial
 from sqlalchemy import func
 import time
+import csv
+import json
+from app.cdclient import (
+    ComponentsRegistry,
+    RenderComponent,
+    ItemComponent,
+    Objects,
+    ScriptComponent,
+)
 
+from app.luclient import translate_from_locale
 
 @click.command("init_db")
 @click.argument('drop_tables', nargs=1)
@@ -180,6 +190,187 @@ def load_property(zone, player):
             )
             new_prop_content.save()
 
+@click.command("parse_lucache")
+@with_appcontext
+def parse_lucache():
+    """Parses lucache csv file dump from nexus hq"""
+    unlisted_ids = [146, 147, 938, 1180, 1692, 1715, 1797, 1799, 1824, 1846, 1847, 1848, 1849, 1850, 1872, 1877, 1887, 1928, 1937, 1968, 1970, 1971, 1972, 1974, 1976, 1977, 1978, 1979, 1980, 1981, 1983, 1984, 2189, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2416, 2417, 2418, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2429, 2430, 2431, 2432, 2433, 2434, 2435, 2436, 2529, 2530, 2553, 2583, 2655, 2656, 2669, 2947, 2948, 3009, 3058, 3068, 3078, 3807, 3812, 3937, 4828, 4874, 4875, 4876, 4877, 4943, 4954, 5839, 5840, 6196, 6218, 6219, 6221, 6433, 6471, 6696, 6821, 6877, 6888, 6889, 6891, 6892, 6893, 6894, 6896, 6897, 6983, 7277, 7551, 7552, 7553, 7554, 7609, 7701, 7713, 7723, 7753, 7754, 7755, 7756, 7760, 7777, 7791, 7824, 7872, 8046, 8053, 8146, 9865, 9866, 9867, 9868, 10126, 10291, 10292, 10293, 10294, 10518, 10630, 10631, 10987, 11511, 11512, 11513, 11514, 11515, 11516, 11517, 11518, 11519, 11520, 11521, 11522, 11523, 11524, 11525, 12096, 12097, 12099, 12100, 12104, 12105, 12111, 12112, 12113, 12324, 12325, 12326, 12553, 12666, 12668, 12670, 12671, 12673, 12674, 12676, 12679, 12680, 12683, 12684, 12685, 12687, 12692, 12694, 12697, 12699, 12701, 12703, 12704, 12713, 12716, 12717, 12727, 12736, 12738, 12739, 12745, 12746, 12750, 12751, 12752, 12757, 12787, 12790, 12791, 12794, 12795, 12799, 12800, 12803, 12887, 12888, 12902, 12904, 12905, 12906, 12907, 12941, 13060, 13061, 13071, 13075, 13076, 13077, 13092, 13093, 13094, 13106, 13118, 13121, 13126, 13127, 13150, 13191, 13192, 13275, 13276, 13277, 13278, 13280, 13295, 13410, 13411, 13510, 13638, 13740, 13742, 13776, 13782, 13905, 13925, 13926, 13927, 13928, 13929, 13930, 13931, 13932, 13953, 13958, 13974, 13996, 13997, 13998, 13999, 14000, 14001, 14002, 14056, 14057, 14058, 14059, 14060, 14061, 14062, 14063, 14064, 14065, 14066, 14067, 14068, 14069, 14070, 14071, 14072, 14073, 14074, 14075, 14076, 14077, 14078, 14079, 14080, 14081, 14090, 14094, 14111, 14135, 14140, 14170, 14171, 14188, 14200, 14202, 14206, 14207, 14208, 14209, 14210, 14211, 14212, 14213, 14228, 14229, 14314, 14428, 14483, 14515, 14522, 14531, 14535, 14536, 14538, 14548, 14554, 14587, 14588, 14589, 14597, 14598, 14599, 14605, 14607, 14608, 14609, 14610, 14611, 14612, 14613, 14614, 14615, 14616, 14617, 14618, 14619, 14620, 14621, 14622, 14623, 14624, 14625, 14626, 14627, 14628, 14629, 14630, 14631, 14632, 14633, 14634, 14635, 14636, 14637, 14638, 14639, 14640, 14641, 14642, 14643, 14644, 14645, 14646, 14647, 14648, 14649, 14650, 14651, 14652, 14653, 14654, 14655, 14656, 14657, 14658, 14659, 14660, 14661, 14662, 14663, 14664, 14665, 14666, 14667, 14668, 14686, 14687, 14688, 14689, 14690, 14704, 14706, 14707, 14716, 14717, 14721, 14722, 14727, 14728, 14729, 14779, 14795, 14799, 14800, 14803, 14815, 14820, 14821, 14822, 14823, 14824, 14825, 14826, 14827, 14831, 14832, 14838, 14839, 15852, 15853, 15854, 15855, 15856, 15857, 15858, 15859, 15860, 15861, 15862, 15863, 15864, 15865, 15885, 15886, 15887, 15888, 15889, 15893, 15894, 15898, 15921, 15923, 15925, 15928, 15930, 15931, 15932, 15933, 15934, 15938, 15939, 15940, 15941, 15942, 15945, 15958, 15962, 15963, 15964, 15965, 15966, 15967, 15968, 15969, 15970, 15971, 15972, 15973, 15981, 15984, 15985, 15986, 15987, 15988, 15989, 15996, 15997, 15998, 15999, 16000, 16001, 16002, 16003, 16004, 16005, 16007, 16008, 16009, 16010, 16011, 16025, 16026, 16027, 16028, 16036, 16039, 16042, 16046, 16051, 16056, 16071, 16072, 16073, 16074, 16075, 16077, 16078, 16079, 16080, 16081, 16089, 16090, 16091, 16092, 16108, 16109, 16110, 16111, 16112, 16113, 16114, 16115, 16116, 16117, 16124, 16125, 16126, 16127, 16128, 16129, 16130, 16137, 16138, 16139, 16140, 16142, 16145, 16167, 16168, 16169, 16170, 16171, 16172, 16173, 16174, 16175, 16176, 16177, 16200, 16201, 16202, 16204, 16212, 16253, 16254, 16418, 16437, 16469, 16479, 16489, 16505, 16641, 16645, 16646, 16648, 16655, 16658, 16659, 16660, 16661, 16662, 16665, 16666, 16667, 16668, 16669, 16670, 16671, 16672, 16673, 16674, 16675, 16676, 16677, 16678, 16679, 16680, 16681, 16685, 16686, 16687, 16688, 16689, 16690, 16691, 16692, 16693, 16694, 16695, 16696, 16697, 16698, 16699, 16700, 16701, 16702, 16703, 16704, 16705, 16706, 16707, 16708, 16709, 16712, 16714, 16717, 16718, 16719, 16720, 16721, 16722, 16724, 16725, 16726, 16727, 16732, 16733, 16734, 16735] # noqa
+    with open("lucache.csv") as cache_file:
+        csv_reader = csv.reader(cache_file, delimiter=',')
+        line_count = 0
+        for row in csv_reader:
+            if row[0] == "id":
+                continue
+            if int(row[0]) in unlisted_ids:
+                json_data = json.loads(row[2])
+                components = ComponentsRegistry.query.filter(ComponentsRegistry.id == int(row[0])).all()
+                obj_type = "Environmental"
+                desc = json_data["Description"]
+                if desc in ["None", None, ""]:
+                    desc = row[1]
+                nametag = 0
+                npcTemplateID = "null"
+                for comp in components:
+                    if comp.component_type == 7:  # Item
+                        obj_type = "Smashable"
+                    if comp.component_type == 11:  # Item
+                        obj_type = "Loot"
+                    if comp.component_type == 35:  # minifig
+                        obj_type = "NPC"
+                        npcTemplateID = comp.component_id
+                        nametag = 1
+                    if comp.component_type == 42:  # b3
+                        obj_type = "Behavior"
+                    if comp.component_type == 60:  # base combat ai
+                        obj_type = "Enemy"
+                    if comp.component_type == 73:  # mission giver
+                        if obj_type != "NPC":
+                            obj_type = "Structure"
+                        desc = f"__MG__{desc}"
+                if "vendor" in row[1].lower():
+                    obj_type = "NPC"
+                print(f"""INSERT INTO "Objects" ("id","name","placeable","type","description","localize","npcTemplateID","displayName","interactionDistance","nametag","_internalNotes","locStatus","gate_version","HQ_valid") VALUES ("{row[0]}", "{row[1].replace("_", " ")}", 1, "{obj_type}", "{desc}", 1, {npcTemplateID} , "{json_data["DisplayName"]}", null , {nametag}, "Unlisted Object", 0, null, 1);""")
+                if obj_type in ["NPC", "Smashable", "Loot"]:
+                    print(f"""        <phrase id="Objects_{row[0]}_name">
+            <translation locale="en_US">{row[1]}</translation>
+            <translation locale="de_DE">TRASNSLATE UNLISTED</translation>
+            <translation locale="en_GB">{row[1]}</translation>
+        </phrase>
+        <phrase id="Objects_{row[0]}_description">
+            <translation locale="en_US">{desc}</translation>
+            <translation locale="de_DE">TRASNSLATE UNLISTED</translation>
+            <translation locale="en_GB">{desc}</translation>
+        </phrase>""")
+                # print(f'{row[0]}: {json_data["DisplayName"]}')
+                line_count += 1
+        # print(f'Processed {line_count} lines.')
+
+
+@click.command("makeup_unlisted_objects")
+@with_appcontext
+def makeup_unlisted_objects():
+    objs_left = []
+    for obj in objs_left:
+        obj_type = "Environmental"
+        nametag = 0
+        name = "Name Missing"
+        desc = "null"
+        npcTemplateID = "null"
+        components = ComponentsRegistry.query.filter(ComponentsRegistry.id == obj).all()
+        for comp in components:
+            if comp.component_type == 2:  # render
+                render = RenderComponent.query.filter(RenderComponent.id == comp.component_id).first()
+                if render is not None:
+
+                    if render.render_asset not in [None, ""]:
+                        name = render.render_asset.replace("_", " ").split('\\')[-1].split('/')[-1].split('.')[0].lower()
+                    if name == "Name Missing":
+                        if render.icon_asset not in [None, ""]:
+                            name = render.icon_asset.replace("_", " ").split('\\')[-1].split('/')[-1].split('.')[0].lower()
+                    name  = name.replace("env ", "").replace("obj ", "").replace("minifig accessory ", "").replace("", "").replace("mf ", "").replace("cre ", "")
+                    # print(f"{obj}: {name} :  {alt_name}")
+                    obj_type = "Smashable"
+                # else:
+                    # print(f"{obj}: No Render")
+            if comp.component_type == 7:  # destroyable
+                obj_type = "Smashable"
+            if comp.component_type == 11:  # Item
+                item = ItemComponent.query.filter(ItemComponent.id == comp.component_id).first()
+                if item.itemType == 24:
+                    obj_type = "Mount"
+                else:
+                    obj_type = "Loot"
+            if comp.component_type == 35:  # minifig
+                obj_type = "NPC"
+                npcTemplateID = comp.component_id
+                nametag = 1
+            if comp.component_type == 42:  # b3
+                obj_type = "Behavior"
+            if comp.component_type == 60:  # base combat ai
+                obj_type = "Enemy"
+            if comp.component_type == 73:  # mission giver
+                if obj_type != "NPC":
+                    obj_type = "Structure"
+                desc = f"__MG__{name}"
+        # print(f"""INSERT INTO "Objects" ("id","name","placeable","type","description","localize","npcTemplateID","displayName","interactionDistance","nametag","_internalNotes","locStatus","gate_version","HQ_valid") VALUES ("{obj}", "{name}", 1, "{obj_type}", "{desc}", 1, {npcTemplateID} , "{name}", null , {nametag}, "Unlisted Object", 0, null, 1);""")
+        if name != "Name Missing" and obj_type in ["Mount"]:
+            print(f"""        <phrase id="Objects_{obj}_name">
+            <translation locale="en_US">{name}</translation>
+            <translation locale="de_DE">TRASNSLATE UNLISTED</translation>
+            <translation locale="en_GB">{name}</translation>
+        </phrase>""")
+
+
+@click.command("gen_new_locales")
+@with_appcontext
+def gen_new_locales():
+    objects = Objects.query.order_by(Objects.id).all()
+    for obj in objects:
+        if obj.type == "Loot":
+            if obj.name not in ["Name Missing", None, "None"] and obj.name[:1] != "m":
+                name_to_trans = f"Object_{obj.id}_name"
+                name_transed = translate_from_locale(name_to_trans)
+                if name_to_trans == name_transed:
+                    print(f"""        <phrase id="Objects_{obj.id}_name">
+            <translation locale="en_US">{obj.name}</translation>
+            <translation locale="de_DE">TRASNSLATE OLD</translation>
+            <translation locale="en_GB">{obj.name}</translation>
+        </phrase>""")
+            if obj.description not in ["None", None, ""]:
+                description_to_trans = f"Object_{obj.id}_description"
+                description_transed = translate_from_locale(description_to_trans)
+                if description_to_trans == description_transed:
+                    print(f"""        <phrase id="Objects_{obj.id}_description">
+            <translation locale="en_US">{obj.description}</translation>
+            <translation locale="de_DE">TRASNSLATE OLD</translation>
+            <translation locale="en_GB">{obj.description}</translation>
+        </phrase>""")
+
+
+@click.command("xref_scripts")
+@with_appcontext
+def xref_scripts():
+    """cross refernce scripts dir with script component table"""
+    scripts = ScriptComponent.query.all()
+    base = 'app/luclient/res/'
+    server = 0
+    server_total = 0
+    client = 0
+    client_total = 0
+    server_used = 0
+    client_used = 0
+    used_total = 0
+    disk_scripts = [path for path in pathlib.Path('app/luclient/res/scripts').rglob("*.lua") if path.is_file()]
+
+    for script in scripts:
+        script_comps = ComponentsRegistry.query.filter(ComponentsRegistry.component_type == 5).filter(ComponentsRegistry.component_id == script.id).all()
+        if len(script_comps) > 0:
+            used_total += 1
+        if script.client_script_name not in [None, ""]:
+            cleaned_name = script.client_script_name.replace('\\', '/').lower()
+            client_script = pathlib.Path(f"{base}{cleaned_name}")
+            client_total += 1
+            if not client_script.is_file():
+                print(f"Missing Server Script: {client_script.as_posix()}")
+                client += 1
+                if len(script_comps) > 0:
+                    client_used += 1
+        if script.script_name not in [None, ""]:
+            cleaned_name = script.script_name.replace('\\', '/').lower()
+            server_script = pathlib.Path(f"{base}{cleaned_name}")
+            server_total += 1
+            if not server_script.is_file():
+                print(f"Missing Client Script: {server_script.as_posix()}")
+                server += 1
+                if len(script_comps) > 0:
+                    server_used += 1
+
+    print(f"Missing {server}/{server_total} server scripts")
+    print(f"Missing {client}/{client_total} client scripts")
+    print(f"Missing {server_used}/{used_total} used server scripts")
+    print(f"Missing {client_used}/{used_total} used client scripts")
+    print(f"Total cdclient scripts {server_total + client_total}\nTotal disk scripts {len(disk_scripts)}")
+
 
 @click.command("gen_image_cache")
 def gen_image_cache():

app/forms.py

@@ -119,7 +119,7 @@ class EditPlayKeyForm(FlaskForm):
 
 class EditGMLevelForm(FlaskForm):
 
-    email = IntegerField(
+    gm_level = IntegerField(
         'GM Level',
         widget=NumberInput(min=0, max=9)
     )

app/luclient.py

@@ -9,6 +9,17 @@ from flask import (
 )
 from flask_user import login_required
 from app.models import CharacterInfo
+from app.cdclient import (
+    Objects,
+    Icons,
+    ItemSets,
+    ComponentsRegistry,
+    ComponentType,
+    RenderComponent,
+    ItemComponent,
+    ObjectSkills,
+    SkillBehavior
+)
 import glob
 import os
 from wand import image
@@ -16,8 +27,8 @@ from wand.exceptions import BlobError as BE
 import pathlib
 import json
 
-import sqlite3
 import xml.etree.ElementTree as ET
+from sqlalchemy import or_
 
 luclient_blueprint = Blueprint('luclient', __name__)
 locale = {}
@@ -65,32 +76,24 @@ def get_dds(filename):
 @luclient_blueprint.route('/get_icon_lot/<id>')
 @login_required
 def get_icon_lot(id):
+    icon_path = RenderComponent.query.filter(
+        RenderComponent.id == ComponentsRegistry.query.filter(
+            ComponentsRegistry.component_type == ComponentType.COMPONENT_TYPE_RENDER
+        ).filter(ComponentsRegistry.id == id).first().component_id
+    ).first().icon_asset
 
-    render_component_id = query_cdclient(
-        'select component_id from ComponentsRegistry where component_type = 2 and id = ?',
-        [id],
-        one=True
-    )[0]
-
-    # find the asset from rendercomponent given the  component id
-    filename = query_cdclient(
-        'select icon_asset from RenderComponent where id = ?',
-        [render_component_id],
-        one=True
-    )[0]
-
-    if filename:
-        filename = filename.replace("..\\", "").replace("\\", "/")
+    if icon_path:
+        icon_path = icon_path.replace("..\\", "").replace("\\", "/")
     else:
         return redirect(url_for('luclient.unknown'))
 
-    cache = f'app/cache/{filename.split(".")[0]}.png'
+    cache = f'app/cache/{icon_path.split(".")[0]}.png'
 
     if not os.path.exists(cache):
         root = 'app/luclient/res/'
         try:
             pathlib.Path(os.path.dirname(cache)).resolve().mkdir(parents=True, exist_ok=True)
-            with image.Image(filename=f'{root}{filename}'.lower()) as img:
+            with image.Image(filename=f'{root}{icon_path}'.lower()) as img:
                 img.compression = "no"
                 img.save(filename=cache)
         except BE:
@@ -103,11 +106,7 @@ def get_icon_lot(id):
 @login_required
 def get_icon_iconid(id):
 
-    filename = query_cdclient(
-        'select IconPath from Icons where IconID = ?',
-        [id],
-        one=True
-    )[0]
+    filename = Icons.query.filter(Icons.IconID == id).first().IconPath
 
     filename = filename.replace("..\\", "").replace("\\", "/")
 
@@ -183,32 +182,6 @@ def unknown():
     return send_file(pathlib.Path(cache).resolve())
 
 
-def get_cdclient():
-    """Connect to CDClient from file system Relative Path
-
-    Args:
-        None
-    """
-    cdclient = getattr(g, '_cdclient', None)
-    if cdclient is None:
-        cdclient = g._database = sqlite3.connect('app/luclient/res/cdclient.sqlite')
-    return cdclient
-
-
-def query_cdclient(query, args=(), one=False):
-    """Run sql queries on CDClient
-
-    Args:
-        query   (string)    : SQL query
-        args    (list)      : List of args to place in query
-        one     (bool)      : Return only on result or all results
-    """
-    cur = get_cdclient().execute(query, args)
-    rv = cur.fetchall()
-    cur.close()
-    return (rv[0] if rv else None) if one else rv
-
-
 def translate_from_locale(trans_string):
     """Finds the string translation from locale.xml
 
@@ -247,13 +220,11 @@ def get_lot_name(lot_id):
         return "Missing"
     name = translate_from_locale(f'Objects_{lot_id}_name')
     if name == f'Objects_{lot_id}_name':
-        intermed = query_cdclient(
-            'select * from Objects where id = ?',
-            [lot_id],
-            one=True
-        )
+        intermed = Objects.query.filter(Objects.id == lot_id).first()
         if intermed:
-            name = intermed[7] if (intermed[7] != "None" and intermed[7] != "" and intermed[7] is None) else intermed[1]
+            name = intermed.displayName if (intermed.displayName != "None" or intermed.displayName != "" or intermed.displayName == None) else intermed.name
+    if not name:
+        name = f'Objects_{lot_id}_name'
     return name
 
 
@@ -304,19 +275,11 @@ def register_luclient_jinja_helpers(app):
     def get_lot_rarity(lot_id):
         if not lot_id:
             return "Missing"
-        render_component_id = query_cdclient(
-            'select component_id from ComponentsRegistry where component_type = 11 and id = ?',
-            [lot_id],
-            one=True
-        )[0]
-
-        rarity = query_cdclient(
-            'select rarity from ItemComponent where id = ?',
-            [render_component_id],
-            one=True
-        )
-        if rarity:
-            rarity = rarity[0]
+        rarity = ItemComponent.query.filter(
+            ItemComponent.id == ComponentsRegistry.query.filter(
+                ComponentsRegistry.component_type == ComponentType.COMPONENT_TYPE_ITEM
+            ).filter(ComponentsRegistry.id == id).first().component_id
+        ).first().rarity
         return rarity
 
     @app.template_filter('get_lot_desc')
@@ -325,15 +288,12 @@ def register_luclient_jinja_helpers(app):
             return "Missing"
         desc = translate_from_locale(f'Objects_{lot_id}_description')
         if desc == f'Objects_{lot_id}_description':
-            desc = query_cdclient(
-                'select description from Objects where id = ?',
-                [lot_id],
-                one=True
-            )
+            desc = Objects.query.filter(Objects.id == lot_id).first()
+
             if desc in ("", None):
                 desc = None
             else:
-                desc = desc[0]
+                desc = desc.description
                 if desc in ("", None):
                     desc = None
         if desc:
@@ -344,11 +304,14 @@ def register_luclient_jinja_helpers(app):
     def check_if_in_set(lot_id):
         if not lot_id:
             return None
-        item_set = query_cdclient(
-            'select * from ItemSets where itemIDs like ? or itemIDs like ? or itemIDs like ?',
-            [f'{lot_id}%', f'%, {lot_id}%', f'%,{lot_id}%'],
-            one=True
-        )
+        item_set = ItemSets.query.filter(
+            or_(
+                ItemSets.itemIDs.like(f'{lot_id}%'),
+                ItemSets.itemIDs.like(f'%, {lot_id}%'),
+                ItemSets.itemIDs.like(f'%,{lot_id}%')
+            )
+        ).first()
+
         if item_set in ("", None):
             return None
         else:
@@ -358,12 +321,19 @@ def register_luclient_jinja_helpers(app):
     def get_lot_stats(lot_id):
         if not lot_id:
             return None
-        stats = query_cdclient(
-            'SELECT imBonusUI, lifeBonusUI, armorBonusUI, skillID, skillIcon FROM SkillBehavior WHERE skillID IN (\
-                SELECT skillID FROM ObjectSkills WHERE objectTemplate=?\
-                )',
-            [lot_id]
-        )
+        stats = SkillBehavior.query.with_entities(
+            SkillBehavior.imBonusUI,
+            SkillBehavior.lifeBonusUI,
+            SkillBehavior.armorBonusUI,
+            SkillBehavior.skillID,
+            SkillBehavior.skillIcon
+        ).filter(
+            SkillBehavior.skillID in ObjectSkills.query.with_entities(
+                ObjectSkills.skillID
+            ).filter(
+                ObjectSkills.objectTemplate == lot_id
+            ).all()
+        ).all()
 
         return consolidate_stats(stats)
 
@@ -371,23 +341,22 @@ def register_luclient_jinja_helpers(app):
     def get_set_stats(lot_id):
         if not lot_id:
             return "Missing"
-        stats = query_cdclient(
-            'SELECT imBonusUI, lifeBonusUI, armorBonusUI, skillID, skillIcon FROM SkillBehavior WHERE skillID IN (\
-                SELECT skillID FROM ItemSetSkills WHERE SkillSetID=?\
-                )',
-            [lot_id]
-        )
+        stats = SkillBehavior.query.with_entities(
+            SkillBehavior.imBonusUI,
+            SkillBehavior.lifeBonusUI,
+            SkillBehavior.armorBonusUI,
+            SkillBehavior.skillID,
+            SkillBehavior.skillIcon
+        ).filter(
+            SkillBehavior.skillID == ObjectSkills.query.with_entities(
+                ObjectSkills.skillID
+            ).filter(
+                ObjectSkills.objectTemplate == lot_id
+            ).all()
+        ).all()
 
         return consolidate_stats(stats)
 
-    @app.template_filter('query_cdclient')
-    def jinja_query_cdclient(query, items):
-        print(query, items)
-        return query_cdclient(
-            query,
-            items,
-            one=True
-        )[0]
 
     @app.template_filter('lu_translate')
     def lu_translate(to_translate):
@@ -396,26 +365,18 @@ def register_luclient_jinja_helpers(app):
 
 def consolidate_stats(stats):
 
-    if len(stats) > 1:
+    if stats:
         consolidated_stats = {"im": 0, "life": 0, "armor": 0, "skill": []}
         for stat in stats:
-            if stat[0]:
-                consolidated_stats["im"] += stat[0]
-            if stat[1]:
-                consolidated_stats["life"] += stat[1]
-            if stat[2]:
-                consolidated_stats["armor"] += stat[2]
-            if stat[3]:
-                consolidated_stats["skill"].append([stat[3], stat[4]])
-
+            if stat.imBonusUI:
+                consolidated_stats["im"] += stat.imBonusUI
+            if stat.lifeBonusUI:
+                consolidated_stats["life"] += stat.lifeBonusUI
+            if stat.armorBonusUI:
+                consolidated_stats["armor"] += stat.armorBonusUI
+            if stat.skillID:
+                consolidated_stats["skill"].append([stat.skillID, stat.skillIcon])
         stats = consolidated_stats
-    elif len(stats) == 1:
-        stats = {
-            "im": stats[0][0] if stats[0][0] else 0,
-            "life": stats[0][1] if stats[0][1] else 0,
-            "armor": stats[0][2] if stats[0][2] else 0,
-            "skill": [[stats[0][3], stats[0][4]]] if stats[0][3] else None,
-        }
     else:
         stats = None
     return stats

app/mail.py

@@ -3,7 +3,8 @@ from flask_user import login_required, current_user
 from app.models import Mail, CharacterInfo
 from app.forms import SendMailForm
 from app import gm_level, log_audit
-from app.luclient import translate_from_locale, query_cdclient
+from app.luclient import translate_from_locale
+from app.cdclient import Objects
 import time
 
 mail_blueprint = Blueprint('mail', __name__)
@@ -68,15 +69,12 @@ def send():
     for character in recipients:
         form.recipient.choices.append((character.id, character.name))
 
-    items = query_cdclient(
-        'Select id, name, displayName from Objects where type = ?',
-        ["Loot"]
-    )
+    items = Objects.query.filter(Objects.type == "Loot").all()
 
     for item in items:
         name = translate_from_locale(f'Objects_{item[0]}_name')
-        if name == f'Objects_{item[0]}_name':
-            name = (item[2] if (item[2] != "None" and item[2] != "" and item[2] is not None) else item[1])
+        if name == f'Objects_{item.id}_name':
+            name = (item.displayName if (item.displayName != "None" and item.displayName != "" and item.displayName is not None) else item.name)
         form.attachment.choices.append(
             (
                 item[0],

app/models.py

@@ -114,7 +114,7 @@ class PlayKey(db.Model):
             )
             db.session.add(new_key)
             db.session.commit()
-            return key
+        return key
 
     def delete(self):
         db.session.delete(self)

app/properties.py

@@ -13,9 +13,10 @@ from flask_user import login_required, current_user
 from datatables import ColumnDT, DataTables
 import time
 from app.models import Property, db, UGC, CharacterInfo, PropertyContent, Account, Mail
+from app.cdclient import ComponentsRegistry, ComponentType, RenderComponent
 from app.schemas import PropertySchema
 from app import gm_level, log_audit
-from app.luclient import query_cdclient
+from app.cdclient import ZoneTable
 from app.forms import RejectPropertyForm
 
 import zlib
@@ -49,11 +50,9 @@ def approve(id):
 
     if property_data.mod_approved:
         message = f"""Approved Property
-            {property_data.name if property_data.name else query_cdclient(
-                'select DisplayDescription from ZoneTable where zoneID = ?',
-                [property_data.zone_id],
-                one=True
-            )[0]}
+            {property_data.name if property_data.name else ZoneTable.query.filter(
+                ZoneTable.zoneID == property_data.zone_id
+            ).first().DisplayDescription}
             from {CharacterInfo.query.filter(CharacterInfo.id==property_data.owner_id).first().name}"""
         log_audit(message)
         flash(
@@ -62,11 +61,9 @@ def approve(id):
         )
     else:
         message = f"""Unapproved Property
-            {property_data.name if property_data.name else query_cdclient(
-                'select DisplayDescription from ZoneTable where zoneID = ?',
-                [property_data.zone_id],
-                one=True
-            )[0]}
+            {property_data.name if property_data.name else ZoneTable.query.filter(
+                ZoneTable.zoneID == property_data.zone_id
+            ).first().DisplayDescription}
             from {CharacterInfo.query.filter(CharacterInfo.id==property_data.owner_id).first().name}"""
         log_audit(message)
         flash(
@@ -100,11 +97,9 @@ def reject(id):
 
     if form.validate_on_submit():
         char_name = CharacterInfo.query.filter(CharacterInfo.id == property_data.owner_id).first().name
-        zone_name = query_cdclient(
-            'select DisplayDescription from ZoneTable where zoneID = ?',
-            [property_data.zone_id],
-            one=True
-        )[0]
+        zone_name = ZoneTable.query.filter(
+            ZoneTable.zoneID == property_data.zone_id
+        ).first().DisplayDescription
         property_data.mod_approved = False
         property_data.rejection_reason = form.rejection_reason.data
         message = f"""Rejected Property
@@ -210,7 +205,7 @@ def get(status="all"):
     rowTable = DataTables(params, query, columns)
 
     data = rowTable.output_result()
-    print(data)
+
     for property_data in data["data"]:
         id = property_data["0"]
 
@@ -251,11 +246,9 @@ def get(status="all"):
         """
 
         if property_data["4"] == "":
-            property_data["4"] = query_cdclient(
-                'select DisplayDescription from ZoneTable where zoneID = ?',
-                [property_data["13"]],
-                one=True
-            )
+            property_data["4"] = ZoneTable.query.filter(
+                ZoneTable.zoneID == property_data["13"]
+            ).first().DisplayDescription
 
         if property_data["6"] == 0:
             property_data["6"] = "Private"
@@ -272,11 +265,9 @@ def get(status="all"):
         else:
             property_data["7"] = '''<h2 class="far fa-check-square text-success"></h2>'''
 
-        property_data["13"] = query_cdclient(
-            'select DisplayDescription from ZoneTable where zoneID = ?',
-            [property_data["13"]],
-            one=True
-        )
+        property_data["13"] = ZoneTable.query.filter(
+            ZoneTable.zoneID == property_data["13"]
+        ).first().DisplayDescription
 
     return data
 
@@ -419,17 +410,12 @@ def ugc(content):
 def prebuilt(content, file_format, lod):
     # translate LOT to component id
     # we need to get a type of 2 because reasons
-    render_component_id = query_cdclient(
-        'select component_id from ComponentsRegistry where component_type = 2 and id = ?',
-        [content.lot],
-        one=True
-    )[0]
-    # find the asset from rendercomponent given the  component id
-    filename = query_cdclient(
-        'select render_asset from RenderComponent where id = ?',
-        [render_component_id],
-        one=True
-    )
+    filename = RenderComponent.query.filter(
+        RenderComponent.id == ComponentsRegistry.query.filter(
+            ComponentsRegistry.component_type == ComponentType.COMPONENT_TYPE_RENDER
+        ).filter(ComponentsRegistry.id == id).first().component_id
+    ).first().render_asset
+
     if filename:
         filename = filename[0].split("\\\\")[-1].lower().split(".")[0]
         if "/" in filename:

app/reports.py

@@ -6,7 +6,6 @@ from app import gm_level, scheduler
 from sqlalchemy.orm import load_only
 import datetime
 import xmltodict
-import random
 
 reports_blueprint = Blueprint('reports', __name__)
 
@@ -32,7 +31,6 @@ def index():
     ).filter(
         Reports.report_type == "uscore"
     ).group_by(Reports.date).options(load_only(Reports.date)).all()
-
     return render_template(
         'reports/index.html.j2',
         reports_items=reports_items,
@@ -69,20 +67,24 @@ def items_graph(start, end):
             items[key] = get_lot_name(key)
     # make it
     for key, value in items.items():
-        data = []
-        for entry in entries:
-            if key in entry.data.keys():
-                data.append(entry.data[key])
-            else:
-                data.append(0)
-        color = "#" + value.encode("utf-8").hex()[1:7]
-        if max(data) > 10:
-            datasets.append({
-                "label": value,
-                "data": data,
-                "backgroundColor": color,
-                "borderColor": color
-            })
+        if value:
+            data = []
+            for entry in entries:
+                if key in entry.data.keys():
+                    if not isinstance(entry.data[key], int):
+                        data.append(entry.data[key]["item_count"])
+                    else:
+                        data.append(entry.data[key])
+                else:
+                    data.append(0)
+            color = "#" + value.encode("utf-8").hex()[1:7]
+            if max(data) > 10:
+                datasets.append({
+                    "label": value,
+                    "data": data,
+                    "backgroundColor": color,
+                    "borderColor": color
+                })
 
     return render_template(
         'reports/graph.html.j2',
@@ -224,6 +226,7 @@ def gen_item_report():
             report_data = {}
 
             for char_xml in char_xmls:
+                name = CharacterInfo.query.filter(CharacterInfo.id == char_xml.id).first().name
                 try:
                     character_json = xmltodict.parse(
                         char_xml.xml_data,
@@ -233,9 +236,13 @@ def gen_item_report():
                         if "i" in inv.keys() and type(inv["i"]) == list and (int(inv["attr_t"]) == 0 or int(inv["attr_t"]) == 1):
                             for item in inv["i"]:
                                 if item["attr_l"] in report_data:
-                                    report_data[item["attr_l"]] = report_data[item["attr_l"]] + int(item["attr_c"])
+                                    report_data[item["attr_l"]]["item_count"] = report_data[item["attr_l"]]["item_count"] + int(item["attr_c"])
                                 else:
-                                    report_data[item["attr_l"]] = int(item["attr_c"])
+                                    report_data[item["attr_l"]] = {"item_count": int(item["attr_c"]), "chars": {}}
+                                if name in report_data[item["attr_l"]]["chars"]:
+                                    report_data[item["attr_l"]]["chars"][name] = report_data[item["attr_l"]]["chars"][name] + int(item["attr_c"])
+                                else:
+                                    report_data[item["attr_l"]]["chars"][name] = int(item["attr_c"])
                 except Exception as e:
                     current_app.logger.error(f"REPORT::ITEMS - ERROR PARSING CHARACTER {char_xml.id}")
                     current_app.logger.error(f"REPORT::ITEMS - {e}")

app/settings_example.py

@@ -7,6 +7,11 @@ APP_SYSTEM_ERROR_SUBJECT_LINE = APP_NAME + " system error"
 APP_SECRET_KEY = ""
 APP_DATABASE_URI = "mysql+pymysql://<username>:<password>@<host>:<port>/<database>"
 
+CONFIG_LINK = False
+CONFIG_LINK_TITLE = ""
+CONFIG_LINK_HREF = ""
+CONFIG_LINK_TEXT = ""
+
 # Send Analytics for Developers to better fix issues
 ALLOW_ANALYTICS = False
 

app/templates/main/about.html.j2

@@ -24,7 +24,27 @@
         </div>
       {% endfor %}
 
-      <hr>
+    </div>
+  </div>
+
+  <div class='card mx-auto mt-5 shadow-sm bg-dark border-primary'>
+    <div class="card-body">
+      <h4 class="text-center">Links</h4>
+
+      {% if config.CONFIG_LINK %}
+        <div class="row">
+          <div class="col text-right">
+            {{ config.CONFIG_LINK_TITLE }}
+          </div>
+          <div class="col">
+            <a href="{{ url_for('static', filename=config.CONFIG_LINK_HREF) }}">
+              {{ config.CONFIG_LINK_TEXT }}
+            </a>
+          </div>
+        </div>
+      {% endif %}
+
+
       <div class="row">
         <div class="col text-right">
           Source

app/templates/reports/items/by_date.html.j2

@@ -20,18 +20,34 @@
         <th scope="col">
           Count
         </th>
+        <th scope="col">
+          Breakdown
+        </th>
         <th scope="col">
           Rarity
         </th>
       </thead>
       <tbody>
-        {% for lot, count in data.items() %}
+        {% for lot, details in data.items() %}
           <tr>
             <td>
               {{ lot|get_lot_name }}
             </td>
             <td>
-              {{ count }}
+              {% if details.chars %}
+                {{ details.item_count }}
+              {% else %}
+                {{ details }}
+              {% endif %}
+            </td>
+            <td>
+              {% if details.chars %}
+                {% for char, value in details.chars|dictsort(false, 'value')|reverse %}
+                  {{char}}: {{value}}<br/>
+                {% endfor %}
+              {% else %}
+                Missing
+              {% endif %}
             </td>
             <td>
               {{ lot|get_lot_rarity }}

requirements.txt

@@ -49,6 +49,7 @@ six==1.16.0
 snowballstemmer==2.2.0
 SQLAlchemy==1.4.22
 sqlalchemy-datatables==2.0.1
+SQLAlchemy-Utils==0.38.2
 toml==0.10.2
 tzdata==2021.5
 tzlocal==4.1