From 69480cfa3e97f852314b928dedf21c49def370e4 Mon Sep 17 00:00:00 2001 From: Bob G Date: Sun, 1 Nov 2026 05:20:48 -0800 Subject: [PATCH] Speed up property model viewer: on-demand rendering, shared UGC caches, validators and gzip (#134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Speed up property model viewer loading - Render the viewer on demand (load completion, camera change, resize) instead of every animation frame - Share the UGC brick geometry cache and parsed Materials.xml across all UGC models on the converter - Serve model and ldddb files with strong ETag/Last-Modified validators and conditional requests so repeat visits get 304s - Gzip model and ldddb responses (COMPRESS_MODEL_RESPONSES), optionally storing .gz copies on disk (CACHE_COMPRESSED_MODELS) Co-authored-by: HailStorm32 * Add configurable redraw behaviour while property models load PROPERTY_VIEWER_LOAD_REDRAW ('none' or 'interval') and PROPERTY_VIEWER_REDRAW_INTERVAL_MS control whether the property viewer redraws as individual models finish loading. A final draw always happens once all loads complete or fail, and a loading indicator is shown meanwhile. Also wrap long lines added by this branch for flake8. Co-authored-by: HailStorm32 * Show model loading progress in the property viewer Replace the static 'Loading models...' badge in the upper-left corner with a progress indicator, e.g. 'Loading models: 42% (89/203)', counting loaded and failed models out of the total. When everything is done it shows 'Loaded 100% (203/203)' (with '(N failed)' if any), turns green (orange on failures) and fades to a faint badge after a few seconds. The indicator is a DOM element updated at most once per animation frame, so it does not trigger canvas redraws in either redraw mode. Co-authored-by: HailStorm32 * Make the loading progress badge more opaque and readable Use a ~93% opaque background for all states (dark while loading, darker green when done, orange on failures), bold white text with a subtle text shadow, border and drop shadow, and only fade to 70% opacity after completion so it stays legible over colourful models. Co-authored-by: HailStorm32 * Batch UGC models and load the viewer's brick data asynchronously Brick-built (UGC) models used to be loaded with synchronous XHRs on the main thread: one request per model's LXFML plus one per brick primitive file, strictly one after another, with the page frozen meanwhile. On a remote server every request costs a network round trip, so a property with 654 UGC models took about two minutes to load. - New route properties.get_ugc_models/ (login required, like get_model) returns all UGC LXFML of a property as one JSON object ({content id: lxfml}) from a single DB query. It is gzip-encoded per COMPRESS_MODEL_RESPONSES and has an ETag over the whole set, so repeat loads get a 304. The per-model get_model route keeps working. - The viewer lists the brick database, fetches the LXFML (batched on view_models, per model with bounded concurrency otherwise) and then each needed brick primitive exactly once with at most 8 requests in flight, all with async fetch. Meshes are then built in short slices so the page stays responsive. The progress badge counts brick-built models as they are built and shows the fetch stage (e.g. 'bricks 120/385'). - Loading finishes once both the loading manager (pre-built models, ground texture) and the brick-built models are done; redraw modes are unchanged ('none' draws once at the end, 'interval' at most every N ms). - get_model for a UGC model only decompresses the LXFML when it has to send a body (a matching If-None-Match skips it). Co-authored-by: HailStorm32 * Show the prebuilt material stage on the loading badge While prebuilt models load, the badge now shows how many MTL files have arrived (e.g. 'materials 120/203'), so progress moves from the start on prebuilt-heavy properties instead of sitting at 0% until the first OBJ loads. Co-authored-by: HailStorm32 * Show a rendering state on the badge before the final render The final full render can block the main thread for a second or more on large properties, which made the badge look frozen. Once all loads are done the badge now shows 'Rendering N parts…' with a CSS spinner (a transform animation the compositor keeps running), waits two animation frames so it is painted, then renders and shows the Loaded state. Co-authored-by: HailStorm32 --------- Co-authored-by: bob-g-code <346622190+bob-g-code@users.noreply.github.com> Co-authored-by: HailStorm32 --- README.md | 5 + app/__init__.py | 22 ++ app/compressed_files.py | 220 ++++++++++++++++++++ app/luclient.py | 8 +- app/properties.py | 102 +++++++-- app/settings_example.py | 20 ++ app/templates/ldd/ldd.html.j2 | 381 +++++++++++++++++++++++++++++++--- 7 files changed, 711 insertions(+), 47 deletions(-) create mode 100644 app/compressed_files.py diff --git a/README.md b/README.md index 88a5eb0..a732272 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,11 @@ Please Reference `app/settings_example.py` to see all the variables * APP_SECRET_KEY (Must be provided) * APP_DATABASE_URI (Must be provided) * Everything else is optional and has defaults + * Property model viewer: + * COMPRESS_MODEL_RESPONSES (default `True`): gzip model and brick database files for browsers that support it + * CACHE_COMPRESSED_MODELS (default `False`): store a `.gz` copy next to each cached file instead of compressing on every request (uses more disk space, saves CPU) + * PROPERTY_VIEWER_LOAD_REDRAW (default `none`): property viewer redraw while models load; `none` draws once when all models have loaded, `interval` redraws periodically so models appear progressively + * PROPERTY_VIEWER_REDRAW_INTERVAL_MS (default `500`): minimum time between redraws while loading when PROPERTY_VIEWER_LOAD_REDRAW is `interval` ## Manual Linux Installation diff --git a/app/__init__.py b/app/__init__.py index 9b0f3f6..f082bdd 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -10,6 +10,7 @@ from flask_user import user_registered, current_user, user_logged_in from flask_wtf.csrf import CSRFProtect from flask_apscheduler import APScheduler from app.luclient import register_luclient_jinja_helpers +from app.compressed_files import register_compressed_files import pathlib import shutil import stat @@ -111,6 +112,7 @@ def create_app(): register_logging(app) register_settings(app) + register_compressed_files(app) register_extensions(app) register_blueprints(app) register_luclient_jinja_helpers(app) @@ -365,6 +367,26 @@ def register_settings(app): app.config['ENABLE_CHAR_XML_UPLOAD'] ) + # Property model viewer compression (see settings_example.py) + for key, default in ( + ("COMPRESS_MODEL_RESPONSES", True), ("CACHE_COMPRESSED_MODELS", False) + ): + value = os.getenv(key, app.config.get(key, default)) + if isinstance(value, str): + value = value.strip().lower() in ("1", "true", "yes", "on") + app.config[key] = bool(value) + + # Property model viewer redraw while models load (see settings_example.py) + key = "PROPERTY_VIEWER_LOAD_REDRAW" + redraw = str(os.getenv(key, app.config.get(key, "none"))).strip().lower() + app.config[key] = redraw if redraw in ("none", "interval") else "none" + key = "PROPERTY_VIEWER_REDRAW_INTERVAL_MS" + try: + interval = int(os.getenv(key, app.config.get(key, 500))) + except (TypeError, ValueError): + interval = 500 + app.config[key] = max(interval, 16) + if "CLIENT_LOCATION" not in app.config: app.config['CLIENT_LOCATION'] = 'app/luclient/' app.config['CLIENT_LOCATION'] = os.getenv( diff --git a/app/compressed_files.py b/app/compressed_files.py new file mode 100644 index 0000000..f8f70da --- /dev/null +++ b/app/compressed_files.py @@ -0,0 +1,220 @@ +"""Helpers for serving model / brick database files with validators and gzip. + +Responses carry a strong ETag (derived from the source file's mtime and size, +or from a content hash for in-memory data) plus Last-Modified where available, +and honour conditional requests so repeat visits get cheap 304 responses. + +When the client accepts gzip and ``COMPRESS_MODEL_RESPONSES`` is enabled the +body is gzip encoded. With ``CACHE_COMPRESSED_MODELS`` enabled a ``.gz`` +copy is +stored on disk (regenerated whenever the source is newer) so no CPU is spent +per request; otherwise the body is compressed in memory at a fast level. +""" +import gzip +import hashlib +import mimetypes +import os +import pathlib +import tempfile +from datetime import datetime, timezone + +from flask import Response, current_app, g, request, send_file +from flask.sessions import SecureCookieSessionInterface + + +# Fast level for on-the-fly compression, better level for stored copies +# (stored copies are only written once). +ON_THE_FLY_LEVEL = 1 +STORED_LEVEL = 6 + + +def _compression_enabled(): + return bool(current_app.config.get("COMPRESS_MODEL_RESPONSES", True)) + + +def _store_compressed(): + return bool(current_app.config.get("CACHE_COMPRESSED_MODELS", False)) + + +def _client_accepts_gzip(): + return _compression_enabled() and request.accept_encodings["gzip"] > 0 + + +def _gz_path_for(path): + """Location of the stored gzip copy for ``path``. + + Files inside the cache directory get a sibling ``.gz``; files that live + elsewhere (e.g. the client's res/ folder, which may be read-only or shared) + get a copy under ``/compressed/``. + """ + cache_root = pathlib.Path(current_app.config["CACHE_LOCATION"]).resolve() + path = path.resolve() + try: + path.relative_to(cache_root) + return path.with_name(path.name + ".gz") + except ValueError: + digest = hashlib.sha1(str(path).encode()).hexdigest() + return cache_root / "compressed" / f"{digest}-{path.name}.gz" + + +def _ensure_gz(path, src_stat): + """Create/refresh the stored gzip copy of ``path`` and return its path.""" + gz_path = _gz_path_for(path) + try: + if gz_path.stat().st_mtime_ns >= src_stat.st_mtime_ns: + return gz_path + except FileNotFoundError: + pass + gz_path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "rb") as src: + data = gzip.compress(src.read(), compresslevel=STORED_LEVEL, mtime=0) + # write atomically so concurrent workers never serve a partial file + fd, tmp = tempfile.mkstemp( + dir=gz_path.parent, prefix=".tmp-", suffix=".gz" + ) + try: + with os.fdopen(fd, "wb") as out: + out.write(data) + os.chmod(tmp, 0o644) # mkstemp creates 0600, match other cache files + os.replace(tmp, gz_path) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + return gz_path + + +def _skip_session_cookie(): + g.compressed_files_user_independent = True + + +class _SessionInterface(SecureCookieSessionInterface): + """Skip refreshing the session cookie for user independent files. + + Flask-User refreshes the session cookie on every request, and Flask adds + "Vary: Cookie" whenever it sets the cookie. As the cookie value changes on + every response, the browser then has to revalidate every cached copy, + defeating max-age. These files don't depend on the session, so there is + nothing to save for them. + """ + + def save_session(self, app, session, response): + if g.get("compressed_files_user_independent"): + return None + return super().save_session(app, session, response) + + +def register_compressed_files(app): + if type(app.session_interface) is SecureCookieSessionInterface: + app.session_interface = _SessionInterface() + + +def _finish(response, etag, last_modified, cache_control, vary): + response.set_etag(etag) + if last_modified is not None: + response.last_modified = last_modified + response.headers["Cache-Control"] = cache_control + if vary: + response.vary.add("Accept-Encoding") + return response.make_conditional(request) + + +def _not_modified(etag, last_modified, cache_control, vary, mimetype): + """Answer a matching conditional request without reading/compressing.""" + if etag in request.if_none_match: + return _finish( + Response(status=304, mimetype=mimetype), + etag, last_modified, cache_control, vary + ) + return None + + +def serve_file(path, mimetype, cache_control, user_independent=False): + """Serve a file from disk with validators and optional gzip encoding. + + ``user_independent`` marks content that does not depend on the logged in + user; the session cookie is not refreshed on these responses so no + ``Vary: Cookie`` is added (see ``_SessionInterface``). + """ + if user_independent: + _skip_session_cookie() + # absolute: send_file resolves relative paths against the app package + path = pathlib.Path(path).resolve() + if mimetype is None: + mimetype = ( + mimetypes.guess_type(path.name)[0] or "application/octet-stream" + ) + st = path.stat() + base_etag = f"{st.st_mtime_ns:x}-{st.st_size:x}" + last_modified = datetime.fromtimestamp(int(st.st_mtime), tz=timezone.utc) + vary = _compression_enabled() + + if not _client_accepts_gzip(): + response = send_file( + path, mimetype=mimetype, etag=base_etag, + last_modified=last_modified, conditional=True + ) + response.headers["Cache-Control"] = cache_control + if vary: + response.vary.add("Accept-Encoding") + return response + + etag = base_etag + "-gz" + early = _not_modified(etag, last_modified, cache_control, vary, mimetype) + if early is not None: + return early + + if _store_compressed(): + gz_path = _ensure_gz(path, st) + response = send_file( + gz_path, mimetype=mimetype, etag=etag, + last_modified=last_modified, conditional=True + ) + response.headers["Cache-Control"] = cache_control + response.headers["Content-Encoding"] = "gzip" + response.vary.add("Accept-Encoding") + return response + + with open(path, "rb") as src: + data = gzip.compress( + src.read(), compresslevel=ON_THE_FLY_LEVEL, mtime=0 + ) + response = Response(data, mimetype=mimetype) + response.headers["Content-Encoding"] = "gzip" + return _finish(response, etag, last_modified, cache_control, vary) + + +def serve_bytes(data, mimetype, cache_control, etag_seed=None): + """Serve in-memory data (e.g. UGC lxfml from the database). + + ``etag_seed`` should be cheap, stable bytes identifying the content (for + example the stored compressed blob); defaults to ``data``. ``data`` may + be a callable producing the body, so a 304 can skip building it (this + requires ``etag_seed``). + """ + if callable(data): + if etag_seed is None: + data = data() + if isinstance(data, str): + data = data.encode() + base_etag = hashlib.sha1( + etag_seed if etag_seed is not None else data + ).hexdigest() + vary = _compression_enabled() + gz = _client_accepts_gzip() + etag = base_etag + ("-gz" if gz else "") + + early = _not_modified(etag, None, cache_control, vary, mimetype) + if early is not None: + return early + + if callable(data): + data = data() + if isinstance(data, str): + data = data.encode() + if gz: + data = gzip.compress(data, compresslevel=ON_THE_FLY_LEVEL, mtime=0) + response = Response(data, mimetype=mimetype) + if gz: + response.headers["Content-Encoding"] = "gzip" + return _finish(response, etag, None, cache_control, vary) diff --git a/app/luclient.py b/app/luclient.py index aa108ae..726f2b4 100644 --- a/app/luclient.py +++ b/app/luclient.py @@ -10,6 +10,7 @@ from flask import ( ) from flask_user import login_required from app.models import CharacterInfo +from app.compressed_files import serve_file import glob import os from wand import image @@ -164,7 +165,12 @@ def dir_listing(req_path): # Check if path is a file and serve if rel_path.is_file(): - return send_file(rel_path) + # Thousands of small, effectively immutable brick primitives are + # fetched + # per viewer load; a modest max-age avoids a 304 round trip for each. + return serve_file( + rel_path, None, "private, max-age=86400", user_independent=True + ) else: return abort(404) diff --git a/app/properties.py b/app/properties.py index 9809b30..836d04b 100644 --- a/app/properties.py +++ b/app/properties.py @@ -16,9 +16,12 @@ import time from app.models import Property, db, UGC, CharacterInfo, PropertyContent, Account, Mail from app import gm_level, log_audit from app.luclient import query_cdclient +from app.compressed_files import serve_bytes, serve_file from app.forms import RejectPropertyForm import zlib +import json +import hashlib import app.pylddlib as ldd import pathlib @@ -305,10 +308,30 @@ def view_model(id, lod): return render_template( 'ldd/ldd.html.j2', content=formatted_data, - lod=lod + lod=lod, + **viewer_redraw_options() ) +def viewer_redraw_options(): + """Redraw-while-loading settings for the model viewer template. + + ?redraw=none|interval and ?redraw_ms=N override the config for testing. + """ + config = current_app.config + mode = request.args.get( + "redraw", config.get("PROPERTY_VIEWER_LOAD_REDRAW", "none") + ) + if mode not in ("none", "interval"): + mode = "none" + interval = config.get("PROPERTY_VIEWER_REDRAW_INTERVAL_MS", 500) + try: + interval = max(int(request.args.get("redraw_ms", interval)), 16) + except ValueError: + pass + return {"load_redraw": mode, "redraw_interval_ms": interval} + + property_center = { 1150: "(-17, 432, -60)", 1151: "(0, 455, -110)", @@ -368,7 +391,9 @@ def view_models(id, lod): property_data=property_data, content=consolidated_list, center=property_center[property_data.zone_id], - lod=lod + lod=lod, + ugc_batch_url=url_for('properties.get_ugc_models', id=id), + **viewer_redraw_options() ) @@ -378,13 +403,53 @@ def get_model(id, file_format, lod): content = PropertyContent.query.filter(PropertyContent.id == id).first() if not(0 <= int(lod) <= 2): abort(404) + # Large model files: let the browser keep a copy but always revalidate it + # (cheap 304 via the ETag) instead of storing long-lived entries. + cache_control = "private, no-cache" if content.lot == 14: # ugc model - response = ugc(content)[0] + ugc_data = UGC.query.filter(UGC.id == content.ugc_id).first() + # decompress lazily so a matching If-None-Match skips the work + return serve_bytes( + lambda: decompress(ugc_data.lxfml), 'text/xml', cache_control, + etag_seed=ugc_data.lxfml + ) else: # prebuilt model - response = prebuilt(content, file_format, lod)[0] + path = prebuilt_path(content, file_format, lod) + if not isinstance(path, pathlib.Path): + return path + return serve_file(path, 'text/xml', cache_control) - response.headers.set('Content-Type', 'text/xml') - return response + +@property_blueprint.route('/get_ugc_models/', methods=['GET']) +@login_required +def get_ugc_models(id): + """All UGC (brick-built) LXFML of a property in one JSON response. + + Returns ``{"": "", ...}`` so the viewer does + not need one request per model (a property can have hundreds). + """ + rows = db.session.query(PropertyContent.id, UGC.lxfml).join( + UGC, UGC.id == PropertyContent.ugc_id + ).filter( + PropertyContent.property_id == id, PropertyContent.lot == 14 + ).order_by(PropertyContent.id).all() + + # ETag over the whole set: content ids plus the stored (compressed) blobs + seed = hashlib.sha1() + for content_id, lxfml in rows: + seed.update(f"{content_id}:{len(lxfml)}:".encode()) + seed.update(lxfml) + + def body(): + return json.dumps({ + str(content_id): decompress(lxfml).decode("utf-8", "replace") + for content_id, lxfml in rows + }) + + return serve_bytes( + body, 'application/json', "private, no-cache", + etag_seed=seed.digest() + ) @property_blueprint.route('/download_model/', methods=['GET']) @@ -434,7 +499,7 @@ def render_asset_basename(render_asset): return name.split(".")[0].lower() -def prebuilt(content, file_format, lod): +def prebuilt_path(content, file_format, lod): # translate LOT to component id # we need to get a type of 2 for the render component to find the filename render_component_id = query_cdclient( @@ -458,11 +523,7 @@ def prebuilt(content, file_format, lod): # if we just want the lxfml, fine t and return it lxfml = pathlib.Path(f'{current_app.config["CLIENT_LOCATION"]}res/BrickModels/{filename.split(".")[0]}.lxfml') if file_format == "lxfml": - - with open(lxfml, 'r') as file: - lxfml_data = file.read() - response = make_response(lxfml_data) - + return lxfml # else we handle getting the files for lddviewer elif file_format in ["obj", "mtl"]: # check to see if the file exists @@ -474,13 +535,16 @@ def prebuilt(content, file_format, lod): ldd.main(str(lxfml.as_posix()), str(cache.with_suffix("").as_posix()), lod) # convert to OBJ except Exception as e: current_app.logger.error(f"ERROR on {cache}:\n {e}") - # then just read it - with open(str(cache.as_posix()), 'r') as file: - cache_data = file.read() - # and serve it - response = make_response(cache_data) - + return cache else: raise(Exception("INVALID FILE FORMAT")) - return response, f"{filename}.{file_format}" + +def prebuilt(content, file_format, lod): + path = prebuilt_path(content, file_format, lod) + if not isinstance(path, pathlib.Path): + return path + with open(path, 'r') as file: + response = make_response(file.read()) + + return response, path.name diff --git a/app/settings_example.py b/app/settings_example.py index a449e51..2a966d0 100644 --- a/app/settings_example.py +++ b/app/settings_example.py @@ -71,3 +71,23 @@ RECAPTCHA_PRIVATE_KEY = '' # RECAPTCHA_API_SERVER = '' # RECAPTCHA_PARAMETERS = '' RECAPTCHA_DATA_ATTRS = {'theme': 'white', 'size': 'invisible'} + +# Property model viewer response compression +# Gzip model (OBJ/MTL/LXFML) and LDD brick database responses for clients that +# accept it. Set to False to always send uncompressed files. +COMPRESS_MODEL_RESPONSES = True +# When True, a gzip copy is written to disk next to each cached file the first +# time it is served (regenerated if the source file is newer) and served +# directly: uses more disk space, but no CPU per request. +# When False, responses are compressed in memory on every request at a fast +# compression level and nothing extra is stored. +CACHE_COMPRESSED_MODELS = False + +# Property model viewer redraw while models are loading +# 'none': do not redraw as individual models finish loading; draw once when +# all loads have completed (or failed), then only on camera move/resize. +# 'interval': redraw at most every PROPERTY_VIEWER_REDRAW_INTERVAL_MS while +# loading so models appear progressively, then a final draw. +# Can be overridden per page for testing with ?redraw=none|interval&redraw_ms=N +PROPERTY_VIEWER_LOAD_REDRAW = 'none' +PROPERTY_VIEWER_REDRAW_INTERVAL_MS = 500 diff --git a/app/templates/ldd/ldd.html.j2 b/app/templates/ldd/ldd.html.j2 index 7b184b3..74cb646 100644 --- a/app/templates/ldd/ldd.html.j2 +++ b/app/templates/ldd/ldd.html.j2 @@ -279,7 +279,11 @@ AddScene(file){ let lxfmlfile = new DBURLFile(file,file) - this.xmldata = lxfmlfile.read() + this.AddSceneData(lxfmlfile.read()) + } + + AddSceneData(xmldata){ + this.xmldata = xmldata if (this.xmldata === ""){ alert(this.xmldata) @@ -521,6 +525,8 @@ constructor(){ this.allMaterials = '' + // brick geometry is shared by every UGC model converted by this converter + this.geometryCache = {} this.scene = new Scene() } @@ -546,14 +552,18 @@ // using the position/rotation stored for the model (properties_contents). Export(filename, positions = []){ let modelGroup = new THREE.Group() - let allMaterials = new Materials(this.database.filelist['Materials.xml'].read()) + // reuse the Materials.xml parsed once in LoadDBURL + if (!this.allMaterials) { + this.allMaterials = new Materials(this.database.filelist['Materials.xml'].read()) + } + let allMaterials = this.allMaterials let invert = new Matrix3D() //invert.n33 = -1 //uncomment to invert the Z-Axis let indexOffset = 1 let textOffset = 1 let usedmaterials = [] - let geometriecache = {} + let geometriecache = this.geometryCache let total = this.scene.Bricks.length let current = 0 @@ -955,7 +965,25 @@ this.name = name } + // Fetch the file asynchronously (once) and keep it for read(). + load() { + if (!this.loading) { + this.loading = fetch(this.urlHandle).then(function (response) { + if (!response.ok) throw new Error('HTTP ' + response.status + ' for ' + response.url) + return response.arrayBuffer() + }).then((buffer) => { + this.content = bytesToUserDefinedString(buffer) + }).catch((error) => { + console.log('Error loading', this.urlHandle, error) + this.content = '' + }) + } + return this.loading + } + read() { + // Prefetched by load(): no request needed + if (this.content !== undefined) return this.content let fileContent let self = this; let xhr = new XMLHttpRequest(); @@ -987,6 +1015,72 @@ } + // Same string the synchronous XHR with the 'x-user-defined' charset hack + // produced: ASCII bytes as-is, bytes >= 0x80 mapped to U+F780..U+F7FF. + function bytesToUserDefinedString(buffer) { + const bytes = new Uint8Array(buffer) + const chunk = 8192 + let out = '' + for (let i = 0; i < bytes.length; i += chunk) { + const part = bytes.subarray(i, i + chunk) + const codes = new Array(part.length) + for (let j = 0; j < part.length; j++) { + const b = part[j] + codes[j] = b < 0x80 ? b : 0xF700 + b + } + out += String.fromCharCode.apply(null, codes) + } + return out + } + + // Run fn over items with at most `limit` in flight at a time. + async function runLimited(items, limit, fn) { + let next = 0 + async function worker() { + while (next < items.length) { + const item = items[next++] + await fn(item) + } + } + const workers = [] + for (let i = 0; i < Math.min(limit, items.length); i++) workers.push(worker()) + await Promise.all(workers) + } + + const FETCH_CONCURRENCY = 8 + + // Asynchronous version of DBURLReader: lists the brick database and + // prefetches info.xml / Materials.xml without blocking the main thread. + async function loadDBURLReader(dburl) { + const reader = Object.create(DBURLReader.prototype) + reader.filelist = {} + reader.initok = false + reader.location = dburl + reader.dbinfo = '' + async function parse(url) { + const response = await fetch(url) + if (!response.ok) { + console.log('HTTP error', response.status, response.statusText) + return + } + const data = await response.json() + const subdirs = [] + for (const obj of data) { + if (obj.type == 'directory') subdirs.push(url + obj.name + '/') + else if (obj.type == 'file') reader.filelist[obj.name] = new DBURLFile(url + obj.name, obj.name) + else console.log('Strange object parsed: ' + obj.type) + } + await Promise.all(subdirs.map(parse)) + } + await parse(dburl) + if (reader.fileexist('Materials.xml') && reader.fileexist('info.xml')) { + await Promise.all([reader.filelist['Materials.xml'].load(), reader.filelist['info.xml'].load()]) + reader.dbinfo = new DBinfo(reader.filelist['info.xml'].read()) + reader.initok = true + } + return reader + } + class DBURLReader { constructor(dburl) { this.filelist = {}; @@ -1056,27 +1150,126 @@ {% for model in content %} {% if model.lot == 14 %} { + "id": {{ model.id|string|tojson }}, "url": "{{url_for('properties.get_model', id=model.id, file_format='lxfml', lod=lod)}}", "pos": {{ model.pos|tojson }} }, {% endif %} {% endfor %} ] - if (lxfml_file_list.length > 0) { - let ldddburl = FindDBURL() + // All UGC LXFML of a property in one request (view_models); otherwise per model + const ugcBatchUrl = {{ ugc_batch_url|default(none)|tojson }} - if (ldddburl) { - let converter = new Converter() - converter.LoadDBURL(ldddburl) - for (let i = 0; i < lxfml_file_list.length; i++) { - converter.LoadScene(lxfml_file_list[i].url) - converter.Export('test.webgl', lxfml_file_list[i].pos) + // Load the brick-built (UGC) models without blocking the page: fetch the + // LXFML (batched), then each needed brick primitive once with bounded + // concurrency, then build the meshes in small slices so the progress badge + // keeps updating. + async function loadUGCModels() { + if (lxfml_file_list.length === 0) return + let built = 0 + try { + setProgressDetail('brick database') + const database = await loadDBURLReader('/luclient/ldddb/') + if (!database.initok) { + alert("LDD database not available. Please look for LEGO-Digital-Designer database.") + throw new Error('LDD database not available') + } + const converter = new Converter() + converter.database = database + converter.allMaterials = new Materials(database.filelist['Materials.xml'].read()) + + setProgressDetail('model files') + let texts + if (ugcBatchUrl) { + const response = await fetch(ugcBatchUrl) + if (!response.ok) throw new Error('HTTP ' + response.status + ' for ' + ugcBatchUrl) + const data = await response.json() + texts = lxfml_file_list.map(function (m) { return data[m.id] }) + } else { + texts = new Array(lxfml_file_list.length) + await runLimited(lxfml_file_list.map(function (m, i) { return i }), FETCH_CONCURRENCY, async function (i) { + try { + const response = await fetch(lxfml_file_list[i].url) + if (response.ok) texts[i] = bytesToUserDefinedString(await response.arrayBuffer()) + } catch (error) { console.log(error) } + }) } + let lastYield = performance.now() + async function maybeYield() { + if (performance.now() - lastYield > 30) { + await new Promise(function (resolve) { setTimeout(resolve) }) + lastYield = performance.now() + } + } + + // parse the scenes and collect the bricks they use + const scenes = [] + for (let i = 0; i < texts.length; i++) { + let parsed = null + if (texts[i]) { + try { parsed = new Scene(); parsed.AddSceneData(texts[i]) } catch (error) { console.log(error); parsed = null } + } + scenes.push(parsed) + await maybeYield() + } + const files = new Set() + const lod = {{ lod }} + for (const parsed of scenes) { + if (!parsed) continue + for (const brick of parsed.Bricks) { + for (const part of brick.Parts) { + let n = 0 + let location = `brickprimitives/lod${lod}/${part.designID}.g` + while (location in database.filelist) { + files.add(database.filelist[location]) + n += 1 + location = `brickprimitives/lod${lod}/${part.designID}.g${n}` + } + const primitive = database.filelist[`Primitives/${part.designID}.xml`] + if (primitive) files.add(primitive) + } + } + } + // each primitive is fetched once (Set + DBURLFile.load() caching) + const fileList = Array.from(files) + let fetched = 0 + setProgressDetail('bricks 0/' + fileList.length) + await runLimited(fileList, FETCH_CONCURRENCY, async function (file) { + await file.load() + fetched += 1 + setProgressDetail('bricks ' + fetched + '/' + fileList.length) + }) + setProgressDetail('') + + // build the meshes + for (let i = 0; i < scenes.length; i++) { + built += 1 + if (!scenes[i]) { modelDone(true); continue } + try { + converter.scene = scenes[i] + converter.Export('test.webgl', lxfml_file_list[i].pos) + modelDone(false) + } catch (error) { + console.log(error) + modelDone(true) + } + if (performance.now() - lastYield > 30) { + loadProgressRender() + await maybeYield() + } + } + } catch (error) { + console.log(error) + // whatever was not built counts as failed + for (; built < lxfml_file_list.length; built++) modelDone(true) } - else { - alert("LDD database not available. Please look for LEGO-Digital-Designer database.") - } + setProgressDetail('') + {% if not center and not content|length > 1 %} + // the camera follows the brick-built model, now that its position is known + camera.position.set( brick_pos.x + 200, brick_pos.y + 300, brick_pos.z + 200 ); + controls.update() + {% endif %} } const onProgress = function ( xhr ) { @@ -1092,8 +1285,120 @@ const onError = function (error) { console.log(error) + loadProgressRender() }; + // Render on demand: only draw when something changed (a load finished, + // the camera moved or the window resized), at most once per animation frame. + let renderPending = false + function requestRender() { + if (renderPending) return + renderPending = true + requestAnimationFrame(function () { + renderPending = false + renderer.render(scene, camera) + }) + } + + // Redraw behaviour while models are still loading (PROPERTY_VIEWER_LOAD_REDRAW): + // 'none' - do not redraw for finished loads; draw once when everything is done + // 'interval' - redraw at most every PROPERTY_VIEWER_REDRAW_INTERVAL_MS while loading + // Camera moves and resizes always redraw. The final draw also happens when loads fail. + const loadRedraw = {{ load_redraw|default('none')|tojson }} + const redrawIntervalMs = {{ redraw_interval_ms|default(500)|int }} + let modelsLoading = true + let lastLoadRender = 0 + let loadRenderTimer = null + // Progress indicator (a DOM element, so updating it never redraws the canvas). + // Counts models: brick-built (lxfml) models count once their meshes are built, + // pre-built models when their OBJ loads or their MTL/OBJ fails. + const loadingIndicator = document.createElement('div') + loadingIndicator.id = 'loading-progress' + loadingIndicator.style.cssText = 'position:fixed;top:8px;left:8px;padding:4px 8px;background:rgba(20,20,20,0.93);color:#fff;font:bold 14px sans-serif;border:1px solid rgba(255,255,255,0.35);border-radius:4px;text-shadow:0 1px 2px rgba(0,0,0,0.8);box-shadow:0 1px 4px rgba(0,0,0,0.5);z-index:10;pointer-events:none;transition:opacity 1s' + document.body.appendChild(loadingIndicator) + const spinnerStyle = document.createElement('style') + spinnerStyle.textContent = '@keyframes loading-spin{to{transform:rotate(360deg)}}#loading-progress .loading-spinner{display:inline-block;width:11px;height:11px;margin-right:7px;vertical-align:-1px;border:2px solid rgba(255,255,255,0.3);border-top-color:#fff;border-radius:50%;animation:loading-spin 0.8s linear infinite;will-change:transform}' + document.head.appendChild(spinnerStyle) + let finalRendering = false + let modelsTotal = 0, modelsDone = 0, modelsFailed = 0 + let progressPending = false + let progressDetail = '' + let prebuiltDetail = '' + function progressText() { + let pct = modelsTotal ? Math.floor(modelsDone / modelsTotal * 100) : 100 + let failed = modelsFailed ? ' (' + modelsFailed + ' failed)' : '' + let parts = [prebuiltDetail, progressDetail].filter(Boolean).join(', ') + let detail = (modelsLoading && parts) ? ' \u2013 ' + parts : '' + return (modelsLoading ? 'Loading models: ' : 'Loaded ') + pct + '% (' + modelsDone + '/' + modelsTotal + ')' + failed + detail + } + function setProgressDetail(text) { + progressDetail = text + updateProgress() + } + function updateProgress() { + // Batch DOM updates to one per animation frame + if (progressPending) return + progressPending = true + requestAnimationFrame(function () { progressPending = false; if (!finalRendering) loadingIndicator.textContent = progressText() }) + } + function modelDone(failed) { + modelsDone++ + if (failed) modelsFailed++ + updateProgress() + } + + function loadProgressRender() { + if (!modelsLoading) return requestRender() + if (loadRedraw !== 'interval' || loadRenderTimer) return + let wait = Math.max(0, lastLoadRender + redrawIntervalMs - performance.now()) + loadRenderTimer = setTimeout(function () { + loadRenderTimer = null + lastLoadRender = performance.now() + requestRender() + }, wait) + } + + // Done once both the loading manager (pre-built models, ground texture) and + // the brick-built models have finished. + let managerDone = false + let ugcDone = lxfml_file_list.length === 0 + function partFinished() { + if (managerDone && ugcDone && modelsLoading) loadsFinished() + } + + function loadsFinished() { + modelsLoading = false + if (loadRenderTimer) { clearTimeout(loadRenderTimer); loadRenderTimer = null } + // The final full render can block the main thread for a while on large + // properties. Show a 'Rendering' state with a CSS spinner (a transform + // animation, which the compositor keeps running while the main thread is + // busy), let it paint, then render and show the result. + let parts = 0 + scene.traverse(function (o) { if (o.isMesh) parts++ }) + finalRendering = true + loadingIndicator.textContent = '' + const spinner = document.createElement('span') + spinner.className = 'loading-spinner' + loadingIndicator.appendChild(spinner) + loadingIndicator.appendChild(document.createTextNode(parts ? 'Rendering ' + parts + ' parts\u2026' : 'Rendering\u2026')) + requestAnimationFrame(function () { + requestAnimationFrame(function () { + renderer.render(scene, camera) + finalRendering = false + loadingIndicator.textContent = progressText() + loadingIndicator.style.background = modelsFailed ? 'rgba(170,70,0,0.94)' : 'rgba(0,105,40,0.94)' + // Keep the completion state visible briefly, then fade to a faint badge + setTimeout(function () { loadingIndicator.style.opacity = '0.7' }, 3000) + }) + }) + } + + // Shared by the model/texture loaders; onLoad fires once all queued items + // (successful or failed) have finished. + let loadsStarted = false + const loadingManager = new THREE.LoadingManager(function () { managerDone = true; partFinished() }, loadProgressRender, loadProgressRender) + loadingManager.onStart = function () { loadsStarted = true; managerDone = false; loadingIndicator.style.opacity = '1' } + // Load in pre-built models let obj_file_list = [ {% for model in content %} @@ -1103,12 +1408,25 @@ {% endfor %} ] - for (let i = 0; i < obj_file_list.length; i++) { - let mtlLoader = new MTLLoader(); - mtlLoader.load( obj_file_list[i].mtl, function( materials ) { + modelsTotal = obj_file_list.length + lxfml_file_list.length + modelsDone = 0 + loadingIndicator.textContent = progressText() + // Pre-built stage on the badge: materials arrive before any OBJ can load + let materialsLoaded = 0 + function materialLoaded() { + materialsLoaded++ + prebuiltDetail = materialsLoaded < obj_file_list.length ? 'materials ' + materialsLoaded + '/' + obj_file_list.length : '' + updateProgress() + } + if (obj_file_list.length) { prebuiltDetail = 'materials 0/' + obj_file_list.length; updateProgress() } + + for (let i = 0; i < obj_file_list.length; i++) { + let mtlLoader = new MTLLoader(loadingManager); + mtlLoader.load( obj_file_list[i].mtl, function( materials ) { + materialLoaded() materials.preload(); - let objLoader = new OBJLoader(); + let objLoader = new OBJLoader(loadingManager); objLoader.setMaterials( materials ); objLoader.load( obj_file_list[i].obj, function ( object ) { @@ -1130,8 +1448,10 @@ scene.add( newModel ); }; - }, onProgress, onError ); - }); + modelDone(false) + loadProgressRender() + }, onProgress, function (error) { modelDone(true); onError(error) } ); + }, undefined, function (error) { materialLoaded(); modelDone(true); onError(error) }); } // Three.JS stuff @@ -1149,7 +1469,7 @@ {% if center %} let center = new THREE.Vector3{{ center }}; - let groundTexture = new THREE.TextureLoader().load( "{{url_for('luclient.get_dds_as_png', filename='env_nim_ag_grass.dds')}}"); + let groundTexture = new THREE.TextureLoader(loadingManager).load( "{{url_for('luclient.get_dds_as_png', filename='env_nim_ag_grass.dds')}}"); groundTexture.wrapS = groundTexture.wrapT = THREE.RepeatWrapping; groundTexture.repeat.set( 10, 10 ); groundTexture.anisotropy = 16; @@ -1192,13 +1512,20 @@ controls.update() // if ( vnh ) vnh.update(); - let render = function () { - requestAnimationFrame(render); - // camera.rotation.z += 0.01; - renderer.render(scene, camera); - }; + controls.addEventListener('change', requestRender) - render(); + window.addEventListener('resize', function () { + camera.aspect = window.innerWidth / window.innerHeight + camera.updateProjectionMatrix() + renderer.setSize(window.innerWidth, window.innerHeight) + requestRender() + }) + + // Nothing queued on the loading manager (e.g. only brick-built models) + if (!loadsStarted) managerDone = true + loadUGCModels().then(function () { ugcDone = true; partFinished() }) + partFinished() + requestRender();