mirror of
https://github.com/DarkflameUniverse/NexusDashboard.git
synced 2026-11-06 02:27:14 +00:00
Compare commits
1 Commits
main
...
fix/ugc-mo
| Author | SHA1 | Date | |
|---|---|---|---|
|
18bab8d57a |
@@ -104,11 +104,6 @@ Please Reference `app/settings_example.py` to see all the variables
|
||||
* APP_SECRET_KEY (Must be provided)
|
||||
* APP_DATABASE_URI (Must be provided)
|
||||
* Everything else is optional and has defaults
|
||||
* Property model viewer:
|
||||
* COMPRESS_MODEL_RESPONSES (default `True`): gzip model and brick database files for browsers that support it
|
||||
* CACHE_COMPRESSED_MODELS (default `False`): store a `.gz` copy next to each cached file instead of compressing on every request (uses more disk space, saves CPU)
|
||||
* PROPERTY_VIEWER_LOAD_REDRAW (default `none`): property viewer redraw while models load; `none` draws once when all models have loaded, `interval` redraws periodically so models appear progressively
|
||||
* PROPERTY_VIEWER_REDRAW_INTERVAL_MS (default `500`): minimum time between redraws while loading when PROPERTY_VIEWER_LOAD_REDRAW is `interval`
|
||||
|
||||
## Manual Linux Installation
|
||||
|
||||
|
||||
123
app/__init__.py
123
app/__init__.py
@@ -10,18 +10,7 @@ from flask_user import user_registered, current_user, user_logged_in
|
||||
from flask_wtf.csrf import CSRFProtect
|
||||
from flask_apscheduler import APScheduler
|
||||
from app.luclient import register_luclient_jinja_helpers
|
||||
from app.compressed_files import register_compressed_files
|
||||
import pathlib
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from zipfile import ZipFile
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError: # Windows
|
||||
fcntl = None
|
||||
import msvcrt
|
||||
|
||||
from app.commands import (
|
||||
init_db,
|
||||
@@ -112,96 +101,26 @@ def create_app():
|
||||
|
||||
register_logging(app)
|
||||
register_settings(app)
|
||||
register_compressed_files(app)
|
||||
register_extensions(app)
|
||||
register_blueprints(app)
|
||||
register_luclient_jinja_helpers(app)
|
||||
|
||||
# Extract the brickdb if it's not already extracted
|
||||
unpack_brick_cache(app.config["CLIENT_LOCATION"], app.config["CACHE_LOCATION"])
|
||||
materials = pathlib.Path(f'{app.config["CACHE_LOCATION"]}Materials.xml')
|
||||
if not materials.is_file():
|
||||
# unzip the brickdb, and remove the import after
|
||||
from zipfile import ZipFile
|
||||
with ZipFile(f"{app.config['CLIENT_LOCATION']}res/brickdb.zip","r") as zip_ref:
|
||||
zip_ref.extractall(app.config["CACHE_LOCATION"])
|
||||
del ZipFile
|
||||
# copy over the brick primitives, and remove the import after
|
||||
from shutil import copytree
|
||||
copytree(f"{app.config['CLIENT_LOCATION']}res/brickprimitives", f"{app.config['CACHE_LOCATION']}brickprimitives")
|
||||
del copytree
|
||||
|
||||
return app
|
||||
|
||||
|
||||
BRICK_CACHE_MARKER = ".brickdb_unpacked"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclusive_lock(lock_path):
|
||||
"""Cross-process exclusive lock, held for the duration of the block."""
|
||||
with open(lock_path, "a+b") as lock_file:
|
||||
if fcntl is not None:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
else: # Windows
|
||||
lock_file.seek(0)
|
||||
while True:
|
||||
try:
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if fcntl is not None:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||
else:
|
||||
lock_file.seek(0)
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
|
||||
|
||||
def _make_owner_writable(path):
|
||||
"""Add owner write permission to a directory tree (copytree keeps source modes)."""
|
||||
for root, dirs, files in os.walk(path):
|
||||
for name in [root] + [os.path.join(root, d) for d in dirs]:
|
||||
os.chmod(name, os.stat(name).st_mode | stat.S_IWUSR | stat.S_IXUSR)
|
||||
|
||||
|
||||
def unpack_brick_cache(client_location, cache_location):
|
||||
"""Unpack brickdb.zip and brickprimitives into the cache exactly once.
|
||||
|
||||
Several gunicorn workers run create_app() at the same time, so the
|
||||
unpack is serialized with a file lock, extracted into a temporary
|
||||
directory and moved into place, and only marked complete at the end.
|
||||
A crashed or partial unpack is redone on the next start.
|
||||
"""
|
||||
cache = pathlib.Path(cache_location)
|
||||
marker = cache / BRICK_CACHE_MARKER
|
||||
if marker.is_file():
|
||||
return
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
with _exclusive_lock(cache / ".brickdb_unpack.lock"):
|
||||
# another process may have finished while we waited for the lock
|
||||
if marker.is_file():
|
||||
return
|
||||
tmp = pathlib.Path(tempfile.mkdtemp(prefix=".brickdb_tmp_", dir=cache))
|
||||
try:
|
||||
with ZipFile(f"{client_location}res/brickdb.zip", "r") as zip_ref:
|
||||
zip_ref.extractall(tmp)
|
||||
# copy file contents only: the client files are often read-only,
|
||||
# and read-only directories can't be moved into place or cleaned up
|
||||
shutil.copytree(
|
||||
f"{client_location}res/brickprimitives",
|
||||
tmp / "brickprimitives",
|
||||
copy_function=shutil.copyfile
|
||||
)
|
||||
_make_owner_writable(tmp)
|
||||
for entry in tmp.iterdir():
|
||||
target = cache / entry.name
|
||||
# leftovers from an older or interrupted unpack get replaced
|
||||
if target.is_dir() and not target.is_symlink():
|
||||
_make_owner_writable(target)
|
||||
shutil.rmtree(target)
|
||||
elif target.exists() or target.is_symlink():
|
||||
target.unlink()
|
||||
os.replace(entry, target)
|
||||
finally:
|
||||
if tmp.exists():
|
||||
_make_owner_writable(tmp)
|
||||
shutil.rmtree(tmp)
|
||||
marker.touch()
|
||||
|
||||
|
||||
def register_extensions(app):
|
||||
"""Register extensions for Flask app
|
||||
|
||||
@@ -367,26 +286,6 @@ def register_settings(app):
|
||||
app.config['ENABLE_CHAR_XML_UPLOAD']
|
||||
)
|
||||
|
||||
# Property model viewer compression (see settings_example.py)
|
||||
for key, default in (
|
||||
("COMPRESS_MODEL_RESPONSES", True), ("CACHE_COMPRESSED_MODELS", False)
|
||||
):
|
||||
value = os.getenv(key, app.config.get(key, default))
|
||||
if isinstance(value, str):
|
||||
value = value.strip().lower() in ("1", "true", "yes", "on")
|
||||
app.config[key] = bool(value)
|
||||
|
||||
# Property model viewer redraw while models load (see settings_example.py)
|
||||
key = "PROPERTY_VIEWER_LOAD_REDRAW"
|
||||
redraw = str(os.getenv(key, app.config.get(key, "none"))).strip().lower()
|
||||
app.config[key] = redraw if redraw in ("none", "interval") else "none"
|
||||
key = "PROPERTY_VIEWER_REDRAW_INTERVAL_MS"
|
||||
try:
|
||||
interval = int(os.getenv(key, app.config.get(key, 500)))
|
||||
except (TypeError, ValueError):
|
||||
interval = 500
|
||||
app.config[key] = max(interval, 16)
|
||||
|
||||
if "CLIENT_LOCATION" not in app.config:
|
||||
app.config['CLIENT_LOCATION'] = 'app/luclient/'
|
||||
app.config['CLIENT_LOCATION'] = os.getenv(
|
||||
|
||||
151
app/accounts.py
151
app/accounts.py
@@ -17,13 +17,11 @@ from app.models import (
|
||||
BugReport,
|
||||
AccountInvitation,
|
||||
db,
|
||||
Friends,
|
||||
CommandLog,
|
||||
CharacterXML
|
||||
Friends
|
||||
)
|
||||
from app import gm_level, log_audit
|
||||
from app.forms import EditGMLevelForm, EditEmailForm
|
||||
from sqlalchemy import or_, inspect, text, bindparam
|
||||
from sqlalchemy import or_
|
||||
|
||||
accounts_blueprint = Blueprint('accounts', __name__)
|
||||
|
||||
@@ -149,114 +147,55 @@ def mute(id, days=0):
|
||||
@gm_level(9)
|
||||
def delete(id):
|
||||
account = Account.query.filter(Account.id == id).first()
|
||||
if account is None:
|
||||
flash("Account not found", "danger")
|
||||
return redirect(url_for("main.index"))
|
||||
message = f"Deleted Account ({account.id}){account.username}"
|
||||
try:
|
||||
_delete_account_data(account)
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
current_app.logger.exception(f"Failed to delete account {id}")
|
||||
flash(f"Failed to delete Account ({account.id}){account.username}", "danger")
|
||||
return redirect(url_for("accounts.view", id=id))
|
||||
chars = CharacterInfo.query.filter(CharacterInfo.account_id == id).all()
|
||||
for char in chars:
|
||||
activities = ActivityLog.query.filter(
|
||||
ActivityLog.character_id == char.id
|
||||
).all()
|
||||
for activity in activities:
|
||||
activity.delete()
|
||||
lb_entries = Leaderboard.query.filter(
|
||||
Leaderboard.character_id == char.id
|
||||
).all()
|
||||
for lb_entry in lb_entries:
|
||||
lb_entry.delete()
|
||||
mails = Mail.query.filter(Mail.receiver_id == char.id).all()
|
||||
for mail in mails:
|
||||
mail.delete()
|
||||
props = Property.query.filter(Property.owner_id == char.id).all()
|
||||
for prop in props:
|
||||
prop_contents = PropertyContent.query.filter(
|
||||
PropertyContent.property_id == prop.id
|
||||
).all()
|
||||
for prop_content in prop_contents:
|
||||
if prop_content.lot == "14":
|
||||
UGC.query.filter(UGC.id == prop.ugc_id).first().delete()
|
||||
prop_content.delete()
|
||||
prop.delete()
|
||||
friends = Friends.query.filter(
|
||||
or_(Friends.player_id == char.id, Friends.friend_id == char.id)
|
||||
).all()
|
||||
for friend in friends:
|
||||
friend.delete()
|
||||
char.delete()
|
||||
# This is for GM stuff, it will be permnently delete logs
|
||||
bugs = BugReport.query.filter(BugReport.resoleved_by_id == id).all()
|
||||
for bug in bugs:
|
||||
bug.delete()
|
||||
audits = AuditLog.query.filter(AuditLog.account_id == id).all()
|
||||
for audit in audits:
|
||||
audit.delete()
|
||||
invites = AccountInvitation.query.filter(
|
||||
AccountInvitation.invited_by_user_id == id).all()
|
||||
for invite in invites:
|
||||
invite.delete()
|
||||
account.delete()
|
||||
flash(message, "danger")
|
||||
log_audit(message)
|
||||
return redirect(url_for("main.index"))
|
||||
|
||||
|
||||
def _bulk_delete(query):
|
||||
# Bulk deletes bypass the ORM relationship handling, which would otherwise
|
||||
# try to NULL out non-nullable foreign keys (e.g. ugc.character_id)
|
||||
# instead of deleting the dependent rows.
|
||||
query.delete(synchronize_session=False)
|
||||
|
||||
|
||||
def _delete_account_data(account):
|
||||
"""Delete an account and every row that references it or its characters.
|
||||
|
||||
The DLU schema uses ON DELETE RESTRICT for most of these foreign keys,
|
||||
so dependents have to be removed explicitly, children first, and all
|
||||
in one transaction so a failure leaves nothing half-deleted.
|
||||
"""
|
||||
char_ids = [
|
||||
c.id for c in CharacterInfo.query.with_entities(CharacterInfo.id)
|
||||
.filter(CharacterInfo.account_id == account.id).all()
|
||||
]
|
||||
clone_ids = [
|
||||
c.prop_clone_id for c in CharacterInfo.query.with_entities(CharacterInfo.prop_clone_id)
|
||||
.filter(CharacterInfo.account_id == account.id).all()
|
||||
]
|
||||
existing_tables = set(inspect(db.engine).get_table_names())
|
||||
|
||||
if char_ids:
|
||||
prop_ids = [
|
||||
p.id for p in Property.query.with_entities(Property.id).filter(
|
||||
or_(Property.owner_id.in_(char_ids), Property.clone_id.in_(clone_ids))
|
||||
).all()
|
||||
]
|
||||
ugc_ids = [
|
||||
u.id for u in UGC.query.with_entities(UGC.id).filter(
|
||||
or_(UGC.account_id == account.id, UGC.character_id.in_(char_ids))
|
||||
).all()
|
||||
]
|
||||
# property contents placed on this account's properties, or using its UGC models
|
||||
content_filter = []
|
||||
if prop_ids:
|
||||
content_filter.append(PropertyContent.property_id.in_(prop_ids))
|
||||
if ugc_ids:
|
||||
content_filter.append(PropertyContent.ugc_id.in_(ugc_ids))
|
||||
if content_filter:
|
||||
_bulk_delete(PropertyContent.query.filter(or_(*content_filter)))
|
||||
if prop_ids:
|
||||
_bulk_delete(Property.query.filter(Property.id.in_(prop_ids)))
|
||||
if "ugc_modular_build" in existing_tables:
|
||||
db.session.execute(
|
||||
text("DELETE FROM ugc_modular_build WHERE character_id IN :ids")
|
||||
.bindparams(bindparam("ids", expanding=True)),
|
||||
{"ids": char_ids}
|
||||
)
|
||||
if "ignore_list" in existing_tables:
|
||||
db.session.execute(
|
||||
text("DELETE FROM ignore_list WHERE player_id IN :ids OR ignored_player_id IN :ids")
|
||||
.bindparams(bindparam("ids", expanding=True)),
|
||||
{"ids": char_ids}
|
||||
)
|
||||
_bulk_delete(ActivityLog.query.filter(ActivityLog.character_id.in_(char_ids)))
|
||||
_bulk_delete(Leaderboard.query.filter(Leaderboard.character_id.in_(char_ids)))
|
||||
_bulk_delete(Mail.query.filter(Mail.receiver_id.in_(char_ids)))
|
||||
_bulk_delete(CommandLog.query.filter(CommandLog.character_id.in_(char_ids)))
|
||||
_bulk_delete(Friends.query.filter(
|
||||
or_(Friends.player_id.in_(char_ids), Friends.friend_id.in_(char_ids))
|
||||
))
|
||||
_bulk_delete(CharacterXML.query.filter(CharacterXML.id.in_(char_ids)))
|
||||
|
||||
# UGC can reference the account even without a (remaining) character
|
||||
_bulk_delete(UGC.query.filter(
|
||||
or_(UGC.account_id == account.id, UGC.character_id.in_(char_ids))
|
||||
if char_ids else UGC.account_id == account.id
|
||||
))
|
||||
if char_ids:
|
||||
_bulk_delete(CharacterInfo.query.filter(CharacterInfo.id.in_(char_ids)))
|
||||
|
||||
if "player_cheat_detections" in existing_tables:
|
||||
db.session.execute(
|
||||
text("DELETE FROM player_cheat_detections WHERE account_id = :id"),
|
||||
{"id": account.id}
|
||||
)
|
||||
if "accounts_rewardcodes" in existing_tables:
|
||||
db.session.execute(
|
||||
text("DELETE FROM accounts_rewardcodes WHERE account_id = :id"),
|
||||
{"id": account.id}
|
||||
)
|
||||
# This is for GM stuff, it will be permnently delete logs
|
||||
_bulk_delete(BugReport.query.filter(BugReport.resoleved_by_id == account.id))
|
||||
_bulk_delete(AuditLog.query.filter(AuditLog.account_id == account.id))
|
||||
_bulk_delete(AccountInvitation.query.filter(AccountInvitation.invited_by_user_id == account.id))
|
||||
_bulk_delete(Account.query.filter(Account.id == account.id))
|
||||
db.session.commit()
|
||||
|
||||
|
||||
@accounts_blueprint.route('/pass_reset/<id>', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@gm_level(9)
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
"""Helpers for serving model / brick database files with validators and gzip.
|
||||
|
||||
Responses carry a strong ETag (derived from the source file's mtime and size,
|
||||
or from a content hash for in-memory data) plus Last-Modified where available,
|
||||
and honour conditional requests so repeat visits get cheap 304 responses.
|
||||
|
||||
When the client accepts gzip and ``COMPRESS_MODEL_RESPONSES`` is enabled the
|
||||
body is gzip encoded. With ``CACHE_COMPRESSED_MODELS`` enabled a ``.gz``
|
||||
copy is
|
||||
stored on disk (regenerated whenever the source is newer) so no CPU is spent
|
||||
per request; otherwise the body is compressed in memory at a fast level.
|
||||
"""
|
||||
import gzip
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from flask import Response, current_app, g, request, send_file
|
||||
from flask.sessions import SecureCookieSessionInterface
|
||||
|
||||
|
||||
# Fast level for on-the-fly compression, better level for stored copies
|
||||
# (stored copies are only written once).
|
||||
ON_THE_FLY_LEVEL = 1
|
||||
STORED_LEVEL = 6
|
||||
|
||||
|
||||
def _compression_enabled():
|
||||
return bool(current_app.config.get("COMPRESS_MODEL_RESPONSES", True))
|
||||
|
||||
|
||||
def _store_compressed():
|
||||
return bool(current_app.config.get("CACHE_COMPRESSED_MODELS", False))
|
||||
|
||||
|
||||
def _client_accepts_gzip():
|
||||
return _compression_enabled() and request.accept_encodings["gzip"] > 0
|
||||
|
||||
|
||||
def _gz_path_for(path):
|
||||
"""Location of the stored gzip copy for ``path``.
|
||||
|
||||
Files inside the cache directory get a sibling ``.gz``; files that live
|
||||
elsewhere (e.g. the client's res/ folder, which may be read-only or shared)
|
||||
get a copy under ``<CACHE_LOCATION>/compressed/``.
|
||||
"""
|
||||
cache_root = pathlib.Path(current_app.config["CACHE_LOCATION"]).resolve()
|
||||
path = path.resolve()
|
||||
try:
|
||||
path.relative_to(cache_root)
|
||||
return path.with_name(path.name + ".gz")
|
||||
except ValueError:
|
||||
digest = hashlib.sha1(str(path).encode()).hexdigest()
|
||||
return cache_root / "compressed" / f"{digest}-{path.name}.gz"
|
||||
|
||||
|
||||
def _ensure_gz(path, src_stat):
|
||||
"""Create/refresh the stored gzip copy of ``path`` and return its path."""
|
||||
gz_path = _gz_path_for(path)
|
||||
try:
|
||||
if gz_path.stat().st_mtime_ns >= src_stat.st_mtime_ns:
|
||||
return gz_path
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
gz_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "rb") as src:
|
||||
data = gzip.compress(src.read(), compresslevel=STORED_LEVEL, mtime=0)
|
||||
# write atomically so concurrent workers never serve a partial file
|
||||
fd, tmp = tempfile.mkstemp(
|
||||
dir=gz_path.parent, prefix=".tmp-", suffix=".gz"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
out.write(data)
|
||||
os.chmod(tmp, 0o644) # mkstemp creates 0600, match other cache files
|
||||
os.replace(tmp, gz_path)
|
||||
except BaseException:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
raise
|
||||
return gz_path
|
||||
|
||||
|
||||
def _skip_session_cookie():
|
||||
g.compressed_files_user_independent = True
|
||||
|
||||
|
||||
class _SessionInterface(SecureCookieSessionInterface):
|
||||
"""Skip refreshing the session cookie for user independent files.
|
||||
|
||||
Flask-User refreshes the session cookie on every request, and Flask adds
|
||||
"Vary: Cookie" whenever it sets the cookie. As the cookie value changes on
|
||||
every response, the browser then has to revalidate every cached copy,
|
||||
defeating max-age. These files don't depend on the session, so there is
|
||||
nothing to save for them.
|
||||
"""
|
||||
|
||||
def save_session(self, app, session, response):
|
||||
if g.get("compressed_files_user_independent"):
|
||||
return None
|
||||
return super().save_session(app, session, response)
|
||||
|
||||
|
||||
def register_compressed_files(app):
|
||||
if type(app.session_interface) is SecureCookieSessionInterface:
|
||||
app.session_interface = _SessionInterface()
|
||||
|
||||
|
||||
def _finish(response, etag, last_modified, cache_control, vary):
|
||||
response.set_etag(etag)
|
||||
if last_modified is not None:
|
||||
response.last_modified = last_modified
|
||||
response.headers["Cache-Control"] = cache_control
|
||||
if vary:
|
||||
response.vary.add("Accept-Encoding")
|
||||
return response.make_conditional(request)
|
||||
|
||||
|
||||
def _not_modified(etag, last_modified, cache_control, vary, mimetype):
|
||||
"""Answer a matching conditional request without reading/compressing."""
|
||||
if etag in request.if_none_match:
|
||||
return _finish(
|
||||
Response(status=304, mimetype=mimetype),
|
||||
etag, last_modified, cache_control, vary
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def serve_file(path, mimetype, cache_control, user_independent=False):
|
||||
"""Serve a file from disk with validators and optional gzip encoding.
|
||||
|
||||
``user_independent`` marks content that does not depend on the logged in
|
||||
user; the session cookie is not refreshed on these responses so no
|
||||
``Vary: Cookie`` is added (see ``_SessionInterface``).
|
||||
"""
|
||||
if user_independent:
|
||||
_skip_session_cookie()
|
||||
# absolute: send_file resolves relative paths against the app package
|
||||
path = pathlib.Path(path).resolve()
|
||||
if mimetype is None:
|
||||
mimetype = (
|
||||
mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||||
)
|
||||
st = path.stat()
|
||||
base_etag = f"{st.st_mtime_ns:x}-{st.st_size:x}"
|
||||
last_modified = datetime.fromtimestamp(int(st.st_mtime), tz=timezone.utc)
|
||||
vary = _compression_enabled()
|
||||
|
||||
if not _client_accepts_gzip():
|
||||
response = send_file(
|
||||
path, mimetype=mimetype, etag=base_etag,
|
||||
last_modified=last_modified, conditional=True
|
||||
)
|
||||
response.headers["Cache-Control"] = cache_control
|
||||
if vary:
|
||||
response.vary.add("Accept-Encoding")
|
||||
return response
|
||||
|
||||
etag = base_etag + "-gz"
|
||||
early = _not_modified(etag, last_modified, cache_control, vary, mimetype)
|
||||
if early is not None:
|
||||
return early
|
||||
|
||||
if _store_compressed():
|
||||
gz_path = _ensure_gz(path, st)
|
||||
response = send_file(
|
||||
gz_path, mimetype=mimetype, etag=etag,
|
||||
last_modified=last_modified, conditional=True
|
||||
)
|
||||
response.headers["Cache-Control"] = cache_control
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
response.vary.add("Accept-Encoding")
|
||||
return response
|
||||
|
||||
with open(path, "rb") as src:
|
||||
data = gzip.compress(
|
||||
src.read(), compresslevel=ON_THE_FLY_LEVEL, mtime=0
|
||||
)
|
||||
response = Response(data, mimetype=mimetype)
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
return _finish(response, etag, last_modified, cache_control, vary)
|
||||
|
||||
|
||||
def serve_bytes(data, mimetype, cache_control, etag_seed=None):
|
||||
"""Serve in-memory data (e.g. UGC lxfml from the database).
|
||||
|
||||
``etag_seed`` should be cheap, stable bytes identifying the content (for
|
||||
example the stored compressed blob); defaults to ``data``. ``data`` may
|
||||
be a callable producing the body, so a 304 can skip building it (this
|
||||
requires ``etag_seed``).
|
||||
"""
|
||||
if callable(data):
|
||||
if etag_seed is None:
|
||||
data = data()
|
||||
if isinstance(data, str):
|
||||
data = data.encode()
|
||||
base_etag = hashlib.sha1(
|
||||
etag_seed if etag_seed is not None else data
|
||||
).hexdigest()
|
||||
vary = _compression_enabled()
|
||||
gz = _client_accepts_gzip()
|
||||
etag = base_etag + ("-gz" if gz else "")
|
||||
|
||||
early = _not_modified(etag, None, cache_control, vary, mimetype)
|
||||
if early is not None:
|
||||
return early
|
||||
|
||||
if callable(data):
|
||||
data = data()
|
||||
if isinstance(data, str):
|
||||
data = data.encode()
|
||||
if gz:
|
||||
data = gzip.compress(data, compresslevel=ON_THE_FLY_LEVEL, mtime=0)
|
||||
response = Response(data, mimetype=mimetype)
|
||||
if gz:
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
return _finish(response, etag, None, cache_control, vary)
|
||||
@@ -10,7 +10,6 @@ from flask import (
|
||||
)
|
||||
from flask_user import login_required
|
||||
from app.models import CharacterInfo
|
||||
from app.compressed_files import serve_file
|
||||
import glob
|
||||
import os
|
||||
from wand import image
|
||||
@@ -165,12 +164,7 @@ def dir_listing(req_path):
|
||||
|
||||
# Check if path is a file and serve
|
||||
if rel_path.is_file():
|
||||
# Thousands of small, effectively immutable brick primitives are
|
||||
# fetched
|
||||
# per viewer load; a modest max-age avoids a 304 round trip for each.
|
||||
return serve_file(
|
||||
rel_path, None, "private, max-age=86400", user_independent=True
|
||||
)
|
||||
return send_file(rel_path)
|
||||
else:
|
||||
return abort(404)
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import re
|
||||
from flask import (
|
||||
render_template,
|
||||
Blueprint,
|
||||
@@ -16,12 +15,9 @@ import time
|
||||
from app.models import Property, db, UGC, CharacterInfo, PropertyContent, Account, Mail
|
||||
from app import gm_level, log_audit
|
||||
from app.luclient import query_cdclient
|
||||
from app.compressed_files import serve_bytes, serve_file
|
||||
from app.forms import RejectPropertyForm
|
||||
|
||||
import zlib
|
||||
import json
|
||||
import hashlib
|
||||
import app.pylddlib as ldd
|
||||
import pathlib
|
||||
|
||||
@@ -308,30 +304,10 @@ def view_model(id, lod):
|
||||
return render_template(
|
||||
'ldd/ldd.html.j2',
|
||||
content=formatted_data,
|
||||
lod=lod,
|
||||
**viewer_redraw_options()
|
||||
lod=lod
|
||||
)
|
||||
|
||||
|
||||
def viewer_redraw_options():
|
||||
"""Redraw-while-loading settings for the model viewer template.
|
||||
|
||||
?redraw=none|interval and ?redraw_ms=N override the config for testing.
|
||||
"""
|
||||
config = current_app.config
|
||||
mode = request.args.get(
|
||||
"redraw", config.get("PROPERTY_VIEWER_LOAD_REDRAW", "none")
|
||||
)
|
||||
if mode not in ("none", "interval"):
|
||||
mode = "none"
|
||||
interval = config.get("PROPERTY_VIEWER_REDRAW_INTERVAL_MS", 500)
|
||||
try:
|
||||
interval = max(int(request.args.get("redraw_ms", interval)), 16)
|
||||
except ValueError:
|
||||
pass
|
||||
return {"load_redraw": mode, "redraw_interval_ms": interval}
|
||||
|
||||
|
||||
property_center = {
|
||||
1150: "(-17, 432, -60)",
|
||||
1151: "(0, 455, -110)",
|
||||
@@ -391,9 +367,7 @@ def view_models(id, lod):
|
||||
property_data=property_data,
|
||||
content=consolidated_list,
|
||||
center=property_center[property_data.zone_id],
|
||||
lod=lod,
|
||||
ugc_batch_url=url_for('properties.get_ugc_models', id=id),
|
||||
**viewer_redraw_options()
|
||||
lod=lod
|
||||
)
|
||||
|
||||
|
||||
@@ -403,53 +377,13 @@ def get_model(id, file_format, lod):
|
||||
content = PropertyContent.query.filter(PropertyContent.id == id).first()
|
||||
if not(0 <= int(lod) <= 2):
|
||||
abort(404)
|
||||
# Large model files: let the browser keep a copy but always revalidate it
|
||||
# (cheap 304 via the ETag) instead of storing long-lived entries.
|
||||
cache_control = "private, no-cache"
|
||||
if content.lot == 14: # ugc model
|
||||
ugc_data = UGC.query.filter(UGC.id == content.ugc_id).first()
|
||||
# decompress lazily so a matching If-None-Match skips the work
|
||||
return serve_bytes(
|
||||
lambda: decompress(ugc_data.lxfml), 'text/xml', cache_control,
|
||||
etag_seed=ugc_data.lxfml
|
||||
)
|
||||
response = ugc(content)[0]
|
||||
else: # prebuilt model
|
||||
path = prebuilt_path(content, file_format, lod)
|
||||
if not isinstance(path, pathlib.Path):
|
||||
return path
|
||||
return serve_file(path, 'text/xml', cache_control)
|
||||
response = prebuilt(content, file_format, lod)[0]
|
||||
|
||||
|
||||
@property_blueprint.route('/get_ugc_models/<id>', methods=['GET'])
|
||||
@login_required
|
||||
def get_ugc_models(id):
|
||||
"""All UGC (brick-built) LXFML of a property in one JSON response.
|
||||
|
||||
Returns ``{"<property content id>": "<lxfml>", ...}`` so the viewer does
|
||||
not need one request per model (a property can have hundreds).
|
||||
"""
|
||||
rows = db.session.query(PropertyContent.id, UGC.lxfml).join(
|
||||
UGC, UGC.id == PropertyContent.ugc_id
|
||||
).filter(
|
||||
PropertyContent.property_id == id, PropertyContent.lot == 14
|
||||
).order_by(PropertyContent.id).all()
|
||||
|
||||
# ETag over the whole set: content ids plus the stored (compressed) blobs
|
||||
seed = hashlib.sha1()
|
||||
for content_id, lxfml in rows:
|
||||
seed.update(f"{content_id}:{len(lxfml)}:".encode())
|
||||
seed.update(lxfml)
|
||||
|
||||
def body():
|
||||
return json.dumps({
|
||||
str(content_id): decompress(lxfml).decode("utf-8", "replace")
|
||||
for content_id, lxfml in rows
|
||||
})
|
||||
|
||||
return serve_bytes(
|
||||
body, 'application/json', "private, no-cache",
|
||||
etag_seed=seed.digest()
|
||||
)
|
||||
response.headers.set('Content-Type', 'text/xml')
|
||||
return response
|
||||
|
||||
|
||||
@property_blueprint.route('/download_model/<id>', methods=['GET'])
|
||||
@@ -460,7 +394,7 @@ def download_model(id):
|
||||
if content.lot == 14: # ugc model
|
||||
response, filename = ugc(content)
|
||||
else: # prebuilt model
|
||||
response, filename = prebuilt(content, "lxfml", 0)
|
||||
response, filename = prebuilt(content, "lxfml")
|
||||
|
||||
response.headers.set('Content-Type', 'attachment/xml')
|
||||
response.headers.set(
|
||||
@@ -488,18 +422,7 @@ def decompress(data):
|
||||
pos += length
|
||||
return out
|
||||
|
||||
def render_asset_basename(render_asset):
|
||||
"""Return the lowercase file name without extension of a client asset path.
|
||||
|
||||
render_asset paths in the cdclient mix separators: single backslashes
|
||||
(mesh\\ftcommunityitems\\x.nif), escaped double backslashes, forward
|
||||
slashes, or combinations of those, and mixed case.
|
||||
"""
|
||||
name = re.split(r"[\\/]+", render_asset.strip())[-1]
|
||||
return name.split(".")[0].lower()
|
||||
|
||||
|
||||
def prebuilt_path(content, file_format, lod):
|
||||
def prebuilt(content, file_format, lod):
|
||||
# translate LOT to component id
|
||||
# we need to get a type of 2 for the render component to find the filename
|
||||
render_component_id = query_cdclient(
|
||||
@@ -515,15 +438,21 @@ def prebuilt_path(content, file_format, lod):
|
||||
)
|
||||
|
||||
# if we have a valie filename, coerce it
|
||||
if filename and filename[0]:
|
||||
filename = render_asset_basename(filename[0])
|
||||
if not filename:
|
||||
abort(404)
|
||||
if filename:
|
||||
filename = filename[0].split("\\\\")[-1].lower().split(".")[0]
|
||||
if "/" in filename:
|
||||
filename = filename.split("/")[-1].lower()
|
||||
else:
|
||||
return f"No filename for LOT {content.lot}"
|
||||
|
||||
# if we just want the lxfml, fine t and return it
|
||||
lxfml = pathlib.Path(f'{current_app.config["CLIENT_LOCATION"]}res/BrickModels/{filename.split(".")[0]}.lxfml')
|
||||
if file_format == "lxfml":
|
||||
return lxfml
|
||||
|
||||
with open(lxfml, 'r') as file:
|
||||
lxfml_data = file.read()
|
||||
response = make_response(lxfml_data)
|
||||
|
||||
# else we handle getting the files for lddviewer
|
||||
elif file_format in ["obj", "mtl"]:
|
||||
# check to see if the file exists
|
||||
@@ -535,16 +464,13 @@ def prebuilt_path(content, file_format, lod):
|
||||
ldd.main(str(lxfml.as_posix()), str(cache.with_suffix("").as_posix()), lod) # convert to OBJ
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"ERROR on {cache}:\n {e}")
|
||||
return cache
|
||||
# then just read it
|
||||
with open(str(cache.as_posix()), 'r') as file:
|
||||
cache_data = file.read()
|
||||
# and serve it
|
||||
response = make_response(cache_data)
|
||||
|
||||
else:
|
||||
raise(Exception("INVALID FILE FORMAT"))
|
||||
|
||||
|
||||
def prebuilt(content, file_format, lod):
|
||||
path = prebuilt_path(content, file_format, lod)
|
||||
if not isinstance(path, pathlib.Path):
|
||||
return path
|
||||
with open(path, 'r') as file:
|
||||
response = make_response(file.read())
|
||||
|
||||
return response, path.name
|
||||
return response, f"{filename}.{file_format}"
|
||||
|
||||
@@ -139,7 +139,7 @@ def currency_graph(start, end):
|
||||
else:
|
||||
data.append(0)
|
||||
color = "#" + character.name.encode("utf-8").hex()[1:7]
|
||||
if data and max(data) > 10000:
|
||||
if max(data) > 10000:
|
||||
datasets.append({
|
||||
"label": character.name,
|
||||
"data": data,
|
||||
@@ -194,7 +194,7 @@ def uscore_graph(start, end):
|
||||
else:
|
||||
data.append(0)
|
||||
color = "#" + character.name.encode("utf-8").hex()[1:7]
|
||||
if data and max(data) > 1000:
|
||||
if max(data) > 1000:
|
||||
datasets.append({
|
||||
"label": character.name,
|
||||
"data": data,
|
||||
|
||||
@@ -71,23 +71,3 @@ RECAPTCHA_PRIVATE_KEY = ''
|
||||
# RECAPTCHA_API_SERVER = ''
|
||||
# RECAPTCHA_PARAMETERS = ''
|
||||
RECAPTCHA_DATA_ATTRS = {'theme': 'white', 'size': 'invisible'}
|
||||
|
||||
# Property model viewer response compression
|
||||
# Gzip model (OBJ/MTL/LXFML) and LDD brick database responses for clients that
|
||||
# accept it. Set to False to always send uncompressed files.
|
||||
COMPRESS_MODEL_RESPONSES = True
|
||||
# When True, a gzip copy is written to disk next to each cached file the first
|
||||
# time it is served (regenerated if the source file is newer) and served
|
||||
# directly: uses more disk space, but no CPU per request.
|
||||
# When False, responses are compressed in memory on every request at a fast
|
||||
# compression level and nothing extra is stored.
|
||||
CACHE_COMPRESSED_MODELS = False
|
||||
|
||||
# Property model viewer redraw while models are loading
|
||||
# 'none': do not redraw as individual models finish loading; draw once when
|
||||
# all loads have completed (or failed), then only on camera move/resize.
|
||||
# 'interval': redraw at most every PROPERTY_VIEWER_REDRAW_INTERVAL_MS while
|
||||
# loading so models appear progressively, then a final draw.
|
||||
# Can be overridden per page for testing with ?redraw=none|interval&redraw_ms=N
|
||||
PROPERTY_VIEWER_LOAD_REDRAW = 'none'
|
||||
PROPERTY_VIEWER_REDRAW_INTERVAL_MS = 500
|
||||
|
||||
@@ -277,13 +277,11 @@
|
||||
this.xmldata = ''
|
||||
}
|
||||
|
||||
AddScene(file){
|
||||
// placement: optional THREE.Matrix4 that places this model in the property. DLU stores player-built
|
||||
// (UGC) models normalized around their own origin, with the position and rotation in properties_contents.
|
||||
AddScene(file, placement = null){
|
||||
let lxfmlfile = new DBURLFile(file,file)
|
||||
this.AddSceneData(lxfmlfile.read())
|
||||
}
|
||||
|
||||
AddSceneData(xmldata){
|
||||
this.xmldata = xmldata
|
||||
this.xmldata = lxfmlfile.read()
|
||||
|
||||
if (this.xmldata === ""){
|
||||
alert(this.xmldata)
|
||||
@@ -309,7 +307,9 @@
|
||||
for (let j = 0; j < childnodes.length ;j++) {
|
||||
let childnode = childnodes[j]
|
||||
if (childnode.nodeName == 'Brick'){
|
||||
this.Bricks.push(new Brick(childnode))
|
||||
let brick = new Brick(childnode)
|
||||
brick.placement = placement
|
||||
this.Bricks.push(brick)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -525,8 +525,6 @@
|
||||
|
||||
constructor(){
|
||||
this.allMaterials = ''
|
||||
// brick geometry is shared by every UGC model converted by this converter
|
||||
this.geometryCache = {}
|
||||
this.scene = new Scene()
|
||||
}
|
||||
|
||||
@@ -537,33 +535,21 @@
|
||||
}
|
||||
}
|
||||
|
||||
LoadScene(filename){
|
||||
LoadScene(filename, placement = null){
|
||||
if(this.database.initok){
|
||||
// Each LXFML file is its own model with its own placement,
|
||||
// so start from a fresh scene instead of accumulating bricks
|
||||
this.scene = new Scene()
|
||||
this.scene.AddScene(filename)
|
||||
this.scene.AddScene(filename, placement)
|
||||
}
|
||||
}
|
||||
|
||||
// Builds the meshes for the currently loaded LXFML scene.
|
||||
// The brick transforms in a UGC LXFML are relative to the model's own origin,
|
||||
// so the meshes are collected into a group which is then placed in the world
|
||||
// using the position/rotation stored for the model (properties_contents).
|
||||
Export(filename, positions = []){
|
||||
let modelGroup = new THREE.Group()
|
||||
// reuse the Materials.xml parsed once in LoadDBURL
|
||||
if (!this.allMaterials) {
|
||||
this.allMaterials = new Materials(this.database.filelist['Materials.xml'].read())
|
||||
}
|
||||
let allMaterials = this.allMaterials
|
||||
Export(filename){
|
||||
let allMaterials = new Materials(this.database.filelist['Materials.xml'].read())
|
||||
let invert = new Matrix3D()
|
||||
//invert.n33 = -1 //uncomment to invert the Z-Axis
|
||||
|
||||
let indexOffset = 1
|
||||
let textOffset = 1
|
||||
let usedmaterials = []
|
||||
let geometriecache = this.geometryCache
|
||||
let geometriecache = {}
|
||||
|
||||
let total = this.scene.Bricks.length
|
||||
let current = 0
|
||||
@@ -626,6 +612,11 @@
|
||||
n14, n24 ,n34, n44);
|
||||
}
|
||||
|
||||
// Move the brick from model space to its place on the property
|
||||
if (bri.placement){
|
||||
m.premultiply(bri.placement)
|
||||
}
|
||||
|
||||
let decoCount = 0
|
||||
|
||||
for (const [partindex, part] of geo.Parts.entries()){
|
||||
@@ -735,26 +726,15 @@
|
||||
let mesh = new THREE.Mesh(geometry, materials);
|
||||
mesh.matrixAutoUpdate = false
|
||||
mesh.matrix = m
|
||||
modelGroup.add(mesh);
|
||||
{% if not center %}
|
||||
brick_pos.setFromMatrixPosition( m );
|
||||
{% endif %}
|
||||
scene.add(mesh);
|
||||
// let vnh = new VertexNormalsHelper( mesh, 5 );
|
||||
// scene.add( vnh );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (positions.length === 0) {
|
||||
positions = [{x: 0, y: 0, z: 0, rx: 0, ry: 0, rz: 0, rw: 1}]
|
||||
}
|
||||
for (const [j, pos] of positions.entries()) {
|
||||
let model = (j === 0) ? modelGroup : modelGroup.clone()
|
||||
model.position.set(pos.x, pos.y, pos.z)
|
||||
model.quaternion.set(pos.rx, pos.ry, pos.rz, pos.rw)
|
||||
model.updateMatrixWorld(true)
|
||||
scene.add(model)
|
||||
{% if not center %}
|
||||
new THREE.Box3().setFromObject(model).getCenter(brick_pos)
|
||||
{% endif %}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -965,25 +945,7 @@
|
||||
this.name = name
|
||||
}
|
||||
|
||||
// Fetch the file asynchronously (once) and keep it for read().
|
||||
load() {
|
||||
if (!this.loading) {
|
||||
this.loading = fetch(this.urlHandle).then(function (response) {
|
||||
if (!response.ok) throw new Error('HTTP ' + response.status + ' for ' + response.url)
|
||||
return response.arrayBuffer()
|
||||
}).then((buffer) => {
|
||||
this.content = bytesToUserDefinedString(buffer)
|
||||
}).catch((error) => {
|
||||
console.log('Error loading', this.urlHandle, error)
|
||||
this.content = ''
|
||||
})
|
||||
}
|
||||
return this.loading
|
||||
}
|
||||
|
||||
read() {
|
||||
// Prefetched by load(): no request needed
|
||||
if (this.content !== undefined) return this.content
|
||||
let fileContent
|
||||
let self = this;
|
||||
let xhr = new XMLHttpRequest();
|
||||
@@ -1015,72 +977,6 @@
|
||||
}
|
||||
|
||||
|
||||
// Same string the synchronous XHR with the 'x-user-defined' charset hack
|
||||
// produced: ASCII bytes as-is, bytes >= 0x80 mapped to U+F780..U+F7FF.
|
||||
function bytesToUserDefinedString(buffer) {
|
||||
const bytes = new Uint8Array(buffer)
|
||||
const chunk = 8192
|
||||
let out = ''
|
||||
for (let i = 0; i < bytes.length; i += chunk) {
|
||||
const part = bytes.subarray(i, i + chunk)
|
||||
const codes = new Array(part.length)
|
||||
for (let j = 0; j < part.length; j++) {
|
||||
const b = part[j]
|
||||
codes[j] = b < 0x80 ? b : 0xF700 + b
|
||||
}
|
||||
out += String.fromCharCode.apply(null, codes)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Run fn over items with at most `limit` in flight at a time.
|
||||
async function runLimited(items, limit, fn) {
|
||||
let next = 0
|
||||
async function worker() {
|
||||
while (next < items.length) {
|
||||
const item = items[next++]
|
||||
await fn(item)
|
||||
}
|
||||
}
|
||||
const workers = []
|
||||
for (let i = 0; i < Math.min(limit, items.length); i++) workers.push(worker())
|
||||
await Promise.all(workers)
|
||||
}
|
||||
|
||||
const FETCH_CONCURRENCY = 8
|
||||
|
||||
// Asynchronous version of DBURLReader: lists the brick database and
|
||||
// prefetches info.xml / Materials.xml without blocking the main thread.
|
||||
async function loadDBURLReader(dburl) {
|
||||
const reader = Object.create(DBURLReader.prototype)
|
||||
reader.filelist = {}
|
||||
reader.initok = false
|
||||
reader.location = dburl
|
||||
reader.dbinfo = ''
|
||||
async function parse(url) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
console.log('HTTP error', response.status, response.statusText)
|
||||
return
|
||||
}
|
||||
const data = await response.json()
|
||||
const subdirs = []
|
||||
for (const obj of data) {
|
||||
if (obj.type == 'directory') subdirs.push(url + obj.name + '/')
|
||||
else if (obj.type == 'file') reader.filelist[obj.name] = new DBURLFile(url + obj.name, obj.name)
|
||||
else console.log('Strange object parsed: ' + obj.type)
|
||||
}
|
||||
await Promise.all(subdirs.map(parse))
|
||||
}
|
||||
await parse(dburl)
|
||||
if (reader.fileexist('Materials.xml') && reader.fileexist('info.xml')) {
|
||||
await Promise.all([reader.filelist['Materials.xml'].load(), reader.filelist['info.xml'].load()])
|
||||
reader.dbinfo = new DBinfo(reader.filelist['info.xml'].read())
|
||||
reader.initok = true
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
class DBURLReader {
|
||||
constructor(dburl) {
|
||||
this.filelist = {};
|
||||
@@ -1146,130 +1042,38 @@
|
||||
|
||||
|
||||
//Start
|
||||
// Player-built models with the position and rotation stored for them in properties_contents
|
||||
let lxfml_file_list = [
|
||||
{% for model in content %}
|
||||
{% if model.lot == 14 %}
|
||||
{
|
||||
"id": {{ model.id|string|tojson }},
|
||||
"url": "{{url_for('properties.get_model', id=model.id, file_format='lxfml', lod=lod)}}",
|
||||
"pos": {{ model.pos|tojson }}
|
||||
},
|
||||
"pos": {{ model.pos[0]|tojson }}
|
||||
}{{ ", " if not loop.last else "" }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
]
|
||||
// All UGC LXFML of a property in one request (view_models); otherwise per model
|
||||
const ugcBatchUrl = {{ ugc_batch_url|default(none)|tojson }}
|
||||
if (lxfml_file_list.length > 0) {
|
||||
let ldddburl = FindDBURL()
|
||||
|
||||
// Load the brick-built (UGC) models without blocking the page: fetch the
|
||||
// LXFML (batched), then each needed brick primitive once with bounded
|
||||
// concurrency, then build the meshes in small slices so the progress badge
|
||||
// keeps updating.
|
||||
async function loadUGCModels() {
|
||||
if (lxfml_file_list.length === 0) return
|
||||
let built = 0
|
||||
try {
|
||||
setProgressDetail('brick database')
|
||||
const database = await loadDBURLReader('/luclient/ldddb/')
|
||||
if (!database.initok) {
|
||||
alert("LDD database not available. Please look for LEGO-Digital-Designer database.")
|
||||
throw new Error('LDD database not available')
|
||||
if (ldddburl) {
|
||||
let converter = new Converter()
|
||||
converter.LoadDBURL(ldddburl)
|
||||
for (let i = 0; i < lxfml_file_list.length; i++) {
|
||||
let pos = lxfml_file_list[i].pos
|
||||
let placement = new THREE.Matrix4().compose(
|
||||
new THREE.Vector3(pos.x, pos.y, pos.z),
|
||||
new THREE.Quaternion(pos.rx, pos.ry, pos.rz, pos.rw),
|
||||
new THREE.Vector3(1, 1, 1)
|
||||
)
|
||||
converter.LoadScene(lxfml_file_list[i].url, placement)
|
||||
}
|
||||
const converter = new Converter()
|
||||
converter.database = database
|
||||
converter.allMaterials = new Materials(database.filelist['Materials.xml'].read())
|
||||
converter.Export('test.webgl')
|
||||
|
||||
setProgressDetail('model files')
|
||||
let texts
|
||||
if (ugcBatchUrl) {
|
||||
const response = await fetch(ugcBatchUrl)
|
||||
if (!response.ok) throw new Error('HTTP ' + response.status + ' for ' + ugcBatchUrl)
|
||||
const data = await response.json()
|
||||
texts = lxfml_file_list.map(function (m) { return data[m.id] })
|
||||
} else {
|
||||
texts = new Array(lxfml_file_list.length)
|
||||
await runLimited(lxfml_file_list.map(function (m, i) { return i }), FETCH_CONCURRENCY, async function (i) {
|
||||
try {
|
||||
const response = await fetch(lxfml_file_list[i].url)
|
||||
if (response.ok) texts[i] = bytesToUserDefinedString(await response.arrayBuffer())
|
||||
} catch (error) { console.log(error) }
|
||||
})
|
||||
}
|
||||
|
||||
let lastYield = performance.now()
|
||||
async function maybeYield() {
|
||||
if (performance.now() - lastYield > 30) {
|
||||
await new Promise(function (resolve) { setTimeout(resolve) })
|
||||
lastYield = performance.now()
|
||||
}
|
||||
}
|
||||
|
||||
// parse the scenes and collect the bricks they use
|
||||
const scenes = []
|
||||
for (let i = 0; i < texts.length; i++) {
|
||||
let parsed = null
|
||||
if (texts[i]) {
|
||||
try { parsed = new Scene(); parsed.AddSceneData(texts[i]) } catch (error) { console.log(error); parsed = null }
|
||||
}
|
||||
scenes.push(parsed)
|
||||
await maybeYield()
|
||||
}
|
||||
const files = new Set()
|
||||
const lod = {{ lod }}
|
||||
for (const parsed of scenes) {
|
||||
if (!parsed) continue
|
||||
for (const brick of parsed.Bricks) {
|
||||
for (const part of brick.Parts) {
|
||||
let n = 0
|
||||
let location = `brickprimitives/lod${lod}/${part.designID}.g`
|
||||
while (location in database.filelist) {
|
||||
files.add(database.filelist[location])
|
||||
n += 1
|
||||
location = `brickprimitives/lod${lod}/${part.designID}.g${n}`
|
||||
}
|
||||
const primitive = database.filelist[`Primitives/${part.designID}.xml`]
|
||||
if (primitive) files.add(primitive)
|
||||
}
|
||||
}
|
||||
}
|
||||
// each primitive is fetched once (Set + DBURLFile.load() caching)
|
||||
const fileList = Array.from(files)
|
||||
let fetched = 0
|
||||
setProgressDetail('bricks 0/' + fileList.length)
|
||||
await runLimited(fileList, FETCH_CONCURRENCY, async function (file) {
|
||||
await file.load()
|
||||
fetched += 1
|
||||
setProgressDetail('bricks ' + fetched + '/' + fileList.length)
|
||||
})
|
||||
setProgressDetail('')
|
||||
|
||||
// build the meshes
|
||||
for (let i = 0; i < scenes.length; i++) {
|
||||
built += 1
|
||||
if (!scenes[i]) { modelDone(true); continue }
|
||||
try {
|
||||
converter.scene = scenes[i]
|
||||
converter.Export('test.webgl', lxfml_file_list[i].pos)
|
||||
modelDone(false)
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
modelDone(true)
|
||||
}
|
||||
if (performance.now() - lastYield > 30) {
|
||||
loadProgressRender()
|
||||
await maybeYield()
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
// whatever was not built counts as failed
|
||||
for (; built < lxfml_file_list.length; built++) modelDone(true)
|
||||
}
|
||||
setProgressDetail('')
|
||||
{% if not center and not content|length > 1 %}
|
||||
// the camera follows the brick-built model, now that its position is known
|
||||
camera.position.set( brick_pos.x + 200, brick_pos.y + 300, brick_pos.z + 200 );
|
||||
controls.update()
|
||||
{% endif %}
|
||||
else {
|
||||
alert("LDD database not available. Please look for LEGO-Digital-Designer database.")
|
||||
}
|
||||
}
|
||||
|
||||
const onProgress = function ( xhr ) {
|
||||
@@ -1285,120 +1089,8 @@
|
||||
|
||||
const onError = function (error) {
|
||||
console.log(error)
|
||||
loadProgressRender()
|
||||
};
|
||||
|
||||
// Render on demand: only draw when something changed (a load finished,
|
||||
// the camera moved or the window resized), at most once per animation frame.
|
||||
let renderPending = false
|
||||
function requestRender() {
|
||||
if (renderPending) return
|
||||
renderPending = true
|
||||
requestAnimationFrame(function () {
|
||||
renderPending = false
|
||||
renderer.render(scene, camera)
|
||||
})
|
||||
}
|
||||
|
||||
// Redraw behaviour while models are still loading (PROPERTY_VIEWER_LOAD_REDRAW):
|
||||
// 'none' - do not redraw for finished loads; draw once when everything is done
|
||||
// 'interval' - redraw at most every PROPERTY_VIEWER_REDRAW_INTERVAL_MS while loading
|
||||
// Camera moves and resizes always redraw. The final draw also happens when loads fail.
|
||||
const loadRedraw = {{ load_redraw|default('none')|tojson }}
|
||||
const redrawIntervalMs = {{ redraw_interval_ms|default(500)|int }}
|
||||
let modelsLoading = true
|
||||
let lastLoadRender = 0
|
||||
let loadRenderTimer = null
|
||||
// Progress indicator (a DOM element, so updating it never redraws the canvas).
|
||||
// Counts models: brick-built (lxfml) models count once their meshes are built,
|
||||
// pre-built models when their OBJ loads or their MTL/OBJ fails.
|
||||
const loadingIndicator = document.createElement('div')
|
||||
loadingIndicator.id = 'loading-progress'
|
||||
loadingIndicator.style.cssText = 'position:fixed;top:8px;left:8px;padding:4px 8px;background:rgba(20,20,20,0.93);color:#fff;font:bold 14px sans-serif;border:1px solid rgba(255,255,255,0.35);border-radius:4px;text-shadow:0 1px 2px rgba(0,0,0,0.8);box-shadow:0 1px 4px rgba(0,0,0,0.5);z-index:10;pointer-events:none;transition:opacity 1s'
|
||||
document.body.appendChild(loadingIndicator)
|
||||
const spinnerStyle = document.createElement('style')
|
||||
spinnerStyle.textContent = '@keyframes loading-spin{to{transform:rotate(360deg)}}#loading-progress .loading-spinner{display:inline-block;width:11px;height:11px;margin-right:7px;vertical-align:-1px;border:2px solid rgba(255,255,255,0.3);border-top-color:#fff;border-radius:50%;animation:loading-spin 0.8s linear infinite;will-change:transform}'
|
||||
document.head.appendChild(spinnerStyle)
|
||||
let finalRendering = false
|
||||
let modelsTotal = 0, modelsDone = 0, modelsFailed = 0
|
||||
let progressPending = false
|
||||
let progressDetail = ''
|
||||
let prebuiltDetail = ''
|
||||
function progressText() {
|
||||
let pct = modelsTotal ? Math.floor(modelsDone / modelsTotal * 100) : 100
|
||||
let failed = modelsFailed ? ' (' + modelsFailed + ' failed)' : ''
|
||||
let parts = [prebuiltDetail, progressDetail].filter(Boolean).join(', ')
|
||||
let detail = (modelsLoading && parts) ? ' \u2013 ' + parts : ''
|
||||
return (modelsLoading ? 'Loading models: ' : 'Loaded ') + pct + '% (' + modelsDone + '/' + modelsTotal + ')' + failed + detail
|
||||
}
|
||||
function setProgressDetail(text) {
|
||||
progressDetail = text
|
||||
updateProgress()
|
||||
}
|
||||
function updateProgress() {
|
||||
// Batch DOM updates to one per animation frame
|
||||
if (progressPending) return
|
||||
progressPending = true
|
||||
requestAnimationFrame(function () { progressPending = false; if (!finalRendering) loadingIndicator.textContent = progressText() })
|
||||
}
|
||||
function modelDone(failed) {
|
||||
modelsDone++
|
||||
if (failed) modelsFailed++
|
||||
updateProgress()
|
||||
}
|
||||
|
||||
function loadProgressRender() {
|
||||
if (!modelsLoading) return requestRender()
|
||||
if (loadRedraw !== 'interval' || loadRenderTimer) return
|
||||
let wait = Math.max(0, lastLoadRender + redrawIntervalMs - performance.now())
|
||||
loadRenderTimer = setTimeout(function () {
|
||||
loadRenderTimer = null
|
||||
lastLoadRender = performance.now()
|
||||
requestRender()
|
||||
}, wait)
|
||||
}
|
||||
|
||||
// Done once both the loading manager (pre-built models, ground texture) and
|
||||
// the brick-built models have finished.
|
||||
let managerDone = false
|
||||
let ugcDone = lxfml_file_list.length === 0
|
||||
function partFinished() {
|
||||
if (managerDone && ugcDone && modelsLoading) loadsFinished()
|
||||
}
|
||||
|
||||
function loadsFinished() {
|
||||
modelsLoading = false
|
||||
if (loadRenderTimer) { clearTimeout(loadRenderTimer); loadRenderTimer = null }
|
||||
// The final full render can block the main thread for a while on large
|
||||
// properties. Show a 'Rendering' state with a CSS spinner (a transform
|
||||
// animation, which the compositor keeps running while the main thread is
|
||||
// busy), let it paint, then render and show the result.
|
||||
let parts = 0
|
||||
scene.traverse(function (o) { if (o.isMesh) parts++ })
|
||||
finalRendering = true
|
||||
loadingIndicator.textContent = ''
|
||||
const spinner = document.createElement('span')
|
||||
spinner.className = 'loading-spinner'
|
||||
loadingIndicator.appendChild(spinner)
|
||||
loadingIndicator.appendChild(document.createTextNode(parts ? 'Rendering ' + parts + ' parts\u2026' : 'Rendering\u2026'))
|
||||
requestAnimationFrame(function () {
|
||||
requestAnimationFrame(function () {
|
||||
renderer.render(scene, camera)
|
||||
finalRendering = false
|
||||
loadingIndicator.textContent = progressText()
|
||||
loadingIndicator.style.background = modelsFailed ? 'rgba(170,70,0,0.94)' : 'rgba(0,105,40,0.94)'
|
||||
// Keep the completion state visible briefly, then fade to a faint badge
|
||||
setTimeout(function () { loadingIndicator.style.opacity = '0.7' }, 3000)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Shared by the model/texture loaders; onLoad fires once all queued items
|
||||
// (successful or failed) have finished.
|
||||
let loadsStarted = false
|
||||
const loadingManager = new THREE.LoadingManager(function () { managerDone = true; partFinished() }, loadProgressRender, loadProgressRender)
|
||||
loadingManager.onStart = function () { loadsStarted = true; managerDone = false; loadingIndicator.style.opacity = '1' }
|
||||
|
||||
// Load in pre-built models
|
||||
let obj_file_list = [
|
||||
{% for model in content %}
|
||||
@@ -1408,25 +1100,12 @@
|
||||
{% endfor %}
|
||||
]
|
||||
|
||||
modelsTotal = obj_file_list.length + lxfml_file_list.length
|
||||
modelsDone = 0
|
||||
loadingIndicator.textContent = progressText()
|
||||
|
||||
// Pre-built stage on the badge: materials arrive before any OBJ can load
|
||||
let materialsLoaded = 0
|
||||
function materialLoaded() {
|
||||
materialsLoaded++
|
||||
prebuiltDetail = materialsLoaded < obj_file_list.length ? 'materials ' + materialsLoaded + '/' + obj_file_list.length : ''
|
||||
updateProgress()
|
||||
}
|
||||
if (obj_file_list.length) { prebuiltDetail = 'materials 0/' + obj_file_list.length; updateProgress() }
|
||||
|
||||
for (let i = 0; i < obj_file_list.length; i++) {
|
||||
let mtlLoader = new MTLLoader(loadingManager);
|
||||
let mtlLoader = new MTLLoader();
|
||||
mtlLoader.load( obj_file_list[i].mtl, function( materials ) {
|
||||
materialLoaded()
|
||||
|
||||
materials.preload();
|
||||
let objLoader = new OBJLoader(loadingManager);
|
||||
let objLoader = new OBJLoader();
|
||||
objLoader.setMaterials( materials );
|
||||
|
||||
objLoader.load( obj_file_list[i].obj, function ( object ) {
|
||||
@@ -1448,10 +1127,8 @@
|
||||
|
||||
scene.add( newModel );
|
||||
};
|
||||
modelDone(false)
|
||||
loadProgressRender()
|
||||
}, onProgress, function (error) { modelDone(true); onError(error) } );
|
||||
}, undefined, function (error) { materialLoaded(); modelDone(true); onError(error) });
|
||||
}, onProgress, onError );
|
||||
});
|
||||
}
|
||||
|
||||
// Three.JS stuff
|
||||
@@ -1469,7 +1146,7 @@
|
||||
|
||||
{% if center %}
|
||||
let center = new THREE.Vector3{{ center }};
|
||||
let groundTexture = new THREE.TextureLoader(loadingManager).load( "{{url_for('luclient.get_dds_as_png', filename='env_nim_ag_grass.dds')}}");
|
||||
let groundTexture = new THREE.TextureLoader().load( "{{url_for('luclient.get_dds_as_png', filename='env_nim_ag_grass.dds')}}");
|
||||
groundTexture.wrapS = groundTexture.wrapT = THREE.RepeatWrapping;
|
||||
groundTexture.repeat.set( 10, 10 );
|
||||
groundTexture.anisotropy = 16;
|
||||
@@ -1512,20 +1189,13 @@
|
||||
controls.update()
|
||||
// if ( vnh ) vnh.update();
|
||||
|
||||
controls.addEventListener('change', requestRender)
|
||||
let render = function () {
|
||||
requestAnimationFrame(render);
|
||||
// camera.rotation.z += 0.01;
|
||||
renderer.render(scene, camera);
|
||||
};
|
||||
|
||||
window.addEventListener('resize', function () {
|
||||
camera.aspect = window.innerWidth / window.innerHeight
|
||||
camera.updateProjectionMatrix()
|
||||
renderer.setSize(window.innerWidth, window.innerHeight)
|
||||
requestRender()
|
||||
})
|
||||
|
||||
// Nothing queued on the loading manager (e.g. only brick-built models)
|
||||
if (!loadsStarted) managerDone = true
|
||||
loadUGCModels().then(function () { ugcDone = true; partFinished() })
|
||||
partFinished()
|
||||
requestRender();
|
||||
render();
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
Reference in New Issue
Block a user