util command to get mission/ach awards that aren't in honor accollade

Reintroduce libsass

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -34,7 +34,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           # generate Docker tags based on the following events/attributes
@@ -46,7 +46,7 @@ jobs:
             type=semver,pattern={{major}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM python:3.8-slim-buster
+FROM python:3.11-slim-bookworm
 
 RUN apt update
 RUN apt -y install zip

README.md

@@ -98,7 +98,7 @@ docker run -d \
 
 ### Environmental Variables
 
-Please Reference `app/settings_exmaple.py` to see all the variables
+Please Reference `app/settings_example.py` to see all the variables
 
   * Required:
     * APP_SECRET_KEY (Must be provided)

app/__init__.py

@@ -19,7 +19,8 @@ from app.commands import (
     gen_image_cache,
     gen_model_cache,
     fix_clone_ids,
-    remove_buffs
+    remove_buffs,
+    find_missing_commendation_items
 )
 from app.models import Account, AccountInvitation, AuditLog
 
@@ -96,6 +97,7 @@ def create_app():
     app.cli.add_command(gen_model_cache)
     app.cli.add_command(fix_clone_ids)
     app.cli.add_command(remove_buffs)
+    app.cli.add_command(find_missing_commendation_items)
 
     register_logging(app)
     register_settings(app)

app/accounts.py

@@ -252,6 +252,9 @@ def get():
         # Delete
         # </a>
 
+        if not current_app.config["USER_ENABLE_EMAIL"]:
+            account["2"] = '''N/A'''
+
         if account["4"]:
             account["4"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
         else:
@@ -267,20 +270,11 @@ def get():
         else:
             account["6"] = '''<h2 class="far fa-check-square text-success"></h2>'''
 
-        if current_app.config["USER_ENABLE_EMAIL"]:
-            if account["8"]:
-                account["8"] = '''<h2 class="far fa-check-square text-success"></h2>'''
-            else:
-                account["8"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
+        if not current_app.config["USER_ENABLE_EMAIL"]:
+            account["8"] = '''<h2 class="far fa-times-circle text-muted"></h2>'''
+        elif account["8"]:
+            account["8"] = '''<h2 class="far fa-check-square text-success"></h2>'''
         else:
-            # shift columns to fill in gap of 2
-            account["2"] = account["3"]
-            account["3"] = account["4"]
-            account["4"] = account["5"]
-            account["5"] = account["6"]
-            account["6"] = account["7"]
-            # remove last two columns
-            del account["7"]
-            del account["8"]
+            account["8"] = '''<h2 class="far fa-times-circle text-danger"></h2>'''
 
     return data

app/commands.py

@@ -4,7 +4,7 @@ import random
 import string
 import datetime
 from flask_user import current_app
-from app import db
+from app import db, luclient
 from app.models import Account, PlayKey, CharacterInfo, Property, PropertyContent, UGC, Mail, CharacterXML
 import pathlib
 import zlib
@@ -281,3 +281,41 @@ def find_or_create_account(name, email, password, gm_level=9):
         db.session.add(play_key)
         db.session.commit()
     return  # account
+
+@click.command("find_missing_commendation_items")
+@with_appcontext
+def find_missing_commendation_items():
+    data = dict()
+    lots = set()
+    reward_items = luclient.query_cdclient("Select reward_item1 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item2 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item3 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    reward_items = luclient.query_cdclient("Select reward_item4 from Missions;")
+    for reward_item in reward_items:
+        lots.add(reward_item[0])
+    lots.remove(0)
+    lots.remove(-1)
+
+    for lot in lots:
+        itemcompid = luclient.query_cdclient(
+            "Select component_id from ComponentsRegistry where component_type = 11 and id = ?;",
+            [lot],
+            one=True
+        )[0]
+        
+        itemcomp = luclient.query_cdclient(
+            "Select commendationLOT, commendationCost from ItemComponent where id = ?;",
+            [itemcompid],
+            one=True
+        )
+        if itemcomp[0] is None or itemcomp[1] is None:
+            data[lot] = {"name": luclient.get_lot_name(lot)}
+    print(data)
+
+

app/forms.py

@@ -22,17 +22,17 @@ from app.models import PlayKey
 
 
 def validate_play_key(form, field):
-    """Validates a field for a valid phone number
+    """Validates a field for a valid play kyey
     Args:
         form: REQUIRED, the field's parent form
         field: REQUIRED, the field with data
     Returns:
         None, raises ValidationError if failed
     """
-    # jank to get the fireign key that we need back into the field
+    # jank to get the foreign key that we need back into the field
     if current_app.config["REQUIRE_PLAY_KEY"]:
         field.data = PlayKey.key_is_valid(key_string=field.data)
-    return
+    return True
 
 class CustomRecaptcha(Recaptcha):
     def __call__(self, form, field):
@@ -49,10 +49,7 @@ class CustomUserManager(UserManager):
 class CustomRegisterForm(RegisterForm):
     play_key_id = StringField(
         'Play Key',
-        validators=[
-            Optional(),
-            validate_play_key,
-        ]
+        validators=[validate_play_key]
     )
     recaptcha = RecaptchaField(
         validators=[CustomRecaptcha()]

app/models.py

@@ -7,7 +7,6 @@ import logging
 from flask_sqlalchemy.query import Query
 from sqlalchemy.dialects import mysql
 from sqlalchemy.exc import OperationalError, StatementError
-from sqlalchemy.types import JSON
 from time import sleep
 import random
 import string
@@ -1018,7 +1017,7 @@ class Reports(db.Model):
     __tablename__ = 'reports'
 
     data = db.Column(
-        JSON(),
+        mysql.MEDIUMBLOB(),
         nullable=False
     )
 

app/reports.py

@@ -4,8 +4,7 @@ from app.models import CharacterInfo, Account, CharacterXML, Reports
 from app.luclient import get_lot_name
 from app import gm_level, scheduler
 from sqlalchemy.orm import load_only
-import datetime
-import xmltodict
+import xmltodict, gzip, json, datetime
 
 reports_blueprint = Blueprint('reports', __name__)
 
@@ -44,6 +43,8 @@ def index():
 @gm_level(3)
 def items_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "items").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/items/by_date.html.j2', data=data, date=date)
 
 
@@ -62,6 +63,8 @@ def items_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
         for key in entry.data:
             items[key] = get_lot_name(key)
@@ -104,6 +107,8 @@ def items_graph(start, end):
 @gm_level(3)
 def currency_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "currency").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/currency/by_date.html.j2', data=data, date=date)
 
 
@@ -121,6 +126,8 @@ def currency_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
     for character in characters:
         data = []
@@ -155,6 +162,8 @@ def currency_graph(start, end):
 @gm_level(3)
 def uscore_by_date(date):
     data = Reports.query.filter(Reports.date == date).filter(Reports.report_type == "uscore").first().data
+    data = gzip.decompress(data)
+    data = json.loads(data.decode('utf-8'))
     return render_template('reports/uscore/by_date.html.j2', data=data, date=date)
 
 
@@ -172,6 +181,8 @@ def uscore_graph(start, end):
     datasets = []
     # get stuff ready
     for entry in entries:
+        entry.data = gzip.decompress(entry.data)
+        entry.data = json.loads(entry.data.decode('utf-8'))
         labels.append(entry.date.strftime("%m/%d/%Y"))
     for character in characters:
         data = []
@@ -260,7 +271,7 @@ def gen_item_report():
                     current_app.logger.error(f"REPORT::ITEMS - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="items",
                 date=date
             )
@@ -308,7 +319,7 @@ def gen_currency_report():
                     current_app.logger.error(f"REPORT::CURRENCY - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="currency",
                 date=date
             )
@@ -356,7 +367,7 @@ def gen_uscore_report():
                     current_app.logger.error(f"REPORT::U-SCORE - {e}")
 
             new_report = Reports(
-                data=report_data,
+                data=gzip.compress(json.dumps(report_data).encode('utf-8')),
                 report_type="uscore",
                 date=date
             )

app/templates/accounts/index.html.j2

@@ -14,17 +14,13 @@
         <tr>
             <th>Actions</th>
             <th>Name</th>
-            {% if config.USER_ENABLE_EMAIL %}
-              <th>Email</th>
-            {% endif %}
+            <th>Email</th>
             <th>GM Level</th>
             <th>Locked</th>
             <th>Banned</th>
             <th>Muted</th>
             <th>Registered</th>
-            {% if config.USER_ENABLE_EMAIL %}
-              <th>Email Confirmed</th>
-            {% endif %}
+            <th>Email Confirmed</th>
         </tr>
     </thead>
     <tbody></tbody>

migrations/env.py

@@ -22,8 +22,8 @@ logger = logging.getLogger('alembic.env')
 # target_metadata = mymodel.Base.metadata
 config.set_main_option(
     'sqlalchemy.url',
-    str(current_app.extensions['migrate'].db.get_engine().url).replace(
-        '%', '%%'))
+    current_app.config.get('SQLALCHEMY_DATABASE_URI')
+)
 target_metadata = current_app.extensions['migrate'].db.metadata
 
 # other values from the config, defined by the needs of env.py,

migrations/versions/1164e037907f_compressss_reports.py

@@ -0,0 +1,164 @@
+"""compressss reports
+
+Revision ID: 1164e037907f
+Revises: a6e42ef03da7
+Create Date: 2023-11-18 01:38:00.127472
+
+"""
+from alembic import op
+from sqlalchemy.orm.session import Session
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+from sqlalchemy.types import JSON
+from sqlalchemy.ext.declarative import declarative_base
+import gzip
+import json
+
+
+Base = declarative_base()
+
+
+# revision identifiers, used by Alembic.
+revision = '1164e037907f'
+down_revision = 'a6e42ef03da7'
+branch_labels = None
+depends_on = None
+
+class ReportsUpgradeNew(Base):
+    __tablename__ = 'reports'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        mysql.MEDIUMBLOB(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+    def save(self):
+        sa.session.add(self)
+        sa.session.commit()
+        sa.session.refresh(self)
+
+
+class ReportsUpgradeOld(Base):
+    __tablename__ = 'reports_old'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        JSON(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+class ReportsDowngradeOld(Base):
+    __tablename__ = 'reports'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        mysql.MEDIUMBLOB(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+    def save(self):
+        sa.session.add(self)
+        sa.session.commit()
+        sa.session.refresh(self)
+
+
+class ReportsDowngradeNew(Base):
+    __tablename__ = 'reports_old'
+    __table_args__ = {'extend_existing': True}
+
+    data = sa.Column(
+        JSON(),
+        nullable=False
+    )
+
+    report_type = sa.Column(
+        sa.VARCHAR(35),
+        nullable=False,
+        primary_key=True,
+        autoincrement=False
+    )
+
+    date = sa.Column(
+        sa.Date(),
+        primary_key=True,
+        autoincrement=False
+    )
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.rename_table('reports', 'reports_old')
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    reports = session.query(ReportsUpgradeOld)
+    op.create_table('reports',
+        sa.Column('data', mysql.MEDIUMBLOB(), nullable=False),
+        sa.Column('report_type', sa.VARCHAR(length=35), autoincrement=False, nullable=False),
+        sa.Column('date', sa.Date(), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('report_type', 'date')
+    )
+    # insert records
+    new_reports = []
+    # insert records
+    for report in reports:
+        new_reports.append({
+            "data":gzip.compress(json.dumps(report.data).encode('utf-8')), 
+            "report_type":report.report_type, 
+            "date":report.date
+        })
+    op.bulk_insert(ReportsUpgradeNew.__table__, new_reports)
+    op.drop_table('reports_old')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('reports')
+    op.create_table('reports',
+        sa.Column('data', JSON(), nullable=False),
+        sa.Column('report_type', sa.VARCHAR(length=35), autoincrement=False, nullable=False),
+        sa.Column('date', sa.Date(), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('report_type', 'date')
+    )
+    # ### end Alembic commands ###

requirements.txt

@@ -1,68 +1,20 @@
 alembic==1.7.5
-APScheduler==3.8.1
-astroid==2.9.1
 autopep8==1.6.0
-backports.zoneinfo==0.2.1
-bcrypt==3.2.0
-blinker==1.7.0
-cffi==1.14.6
-click==8.1.7
-colorama==0.4.4
-cryptography==36.0.0
-dnspython==2.1.0
-dominate==2.6.0
 email-validator==1.1.3
 Flask==3.0.0
 Flask-APScheduler==1.12.3
 Flask-Assets==2.1.0
-Flask-Login==0.6.3
-Flask-Mail==0.9.1
-flask-marshmallow==0.15.0
 Flask-Migrate==3.1.0
 Flask-SQLAlchemy==3.1.1
 Flask-User==1.0.2.2
 Flask-WTF==1.2.1
-greenlet==1.1.0
 gunicorn==21.2.0
-idna==3.3
-importlib-metadata==6.8.0
-importlib-resources==6.1.1
-isort==5.10.1
-itsdangerous==2.1.2
-Jinja2==3.1.2
-lazy-object-proxy==1.7.1
 libsass==0.21.0
-Mako==1.2.2
-MarkupSafe==2.1.3
-marshmallow==3.14.1
-marshmallow-sqlalchemy==0.26.1
-mccabe==0.6.1
-packaging==23.2
-passlib==1.7.4
-platformdirs==2.4.1
-pycodestyle==2.8.0
-pycparser==2.20
-pydocstyle==6.1.1
-pyflakes==2.4.0
-pylama==8.3.3
-pylint==2.12.2
 PyMySQL==1.0.2
-python-dateutil==2.8.2
-pytz==2021.3
-pytz-deprecation-shim==0.1.0.post0
-six==1.16.0
-snowballstemmer==2.2.0
 SQLAlchemy==2.0.23
 sqlalchemy-datatables==2.0.1
-toml==0.10.2
-typing_extensions==4.8.0
-tzdata==2021.5
-tzlocal==4.1
-visitor==0.1.3
 Wand==0.6.7
 webassets==2.0
 Werkzeug==3.0.1
-wrapt==1.13.3
 WTForms==3.0.0
 xmltodict==0.12.0
-zipp==3.17.0