From 8646c40943f83d730a33897e93c3fec82a345110 Mon Sep 17 00:00:00 2001 From: Aaron Kimbre Date: Thu, 10 Feb 2022 12:01:51 -0600 Subject: [PATCH 1/3] lod differentiation pre-gen commands --- app/__init__.py | 4 +++- app/commands.py | 56 +++++++++++++++++++++++++++++++++++++++++++++-- app/properties.py | 14 ++++-------- app/pylddlib.py | 23 ++++++++++--------- 4 files changed, 74 insertions(+), 23 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index e1879cc..ed5c7a2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -12,7 +12,7 @@ from flask_wtf.csrf import CSRFProtect from flask_apscheduler import APScheduler from app.luclient import query_cdclient, register_luclient_jinja_helpers -from app.commands import init_db, init_accounts, load_property +from app.commands import init_db, init_accounts, load_property, gen_image_cache, gen_model_cache from app.models import Account, AccountInvitation # Instantiate Flask extensions @@ -65,6 +65,8 @@ def create_app(): app.cli.add_command(init_db) app.cli.add_command(init_accounts) app.cli.add_command(load_property) + app.cli.add_command(gen_image_cache) + app.cli.add_command(gen_model_cache) register_settings(app) register_extensions(app) diff --git a/app/commands.py b/app/commands.py index a9925a5..01cfc24 100644 --- a/app/commands.py +++ b/app/commands.py @@ -7,6 +7,12 @@ from app import db from app.models import Account, PlayKey, CharacterInfo, Property, PropertyContent, UGC import pathlib import zlib +import os +from wand import image +from wand.exceptions import BlobError as BE +import app.pylddlib as ldd +from multiprocessing import Pool +from functools import partial @click.command("init_db") @click.argument('drop_tables', nargs=1) @@ -89,6 +95,54 @@ def load_property(zone, player): ) new_prop_content.save() +@click.command("gen_image_cache") +def gen_image_cache(): + luclient = pathlib.Path('app/luclient/res') + files = [path for path in luclient.rglob("*.dds") if path.is_file()] + + for file in files: + cache = get_cache_file(file).with_suffix(".png") + if not cache.exists(): + try: + print(f"Convert {file.as_posix()} to {cache}") + cache.parent.mkdir(parents=True, exist_ok=True) + with image.Image(filename=str(file.as_posix())) as img: + img.compression = "no" + img.save(filename=str(cache.as_posix())) + except BE: + return print(f"Error on {file}") + +@click.command("gen_model_cache") +def gen_model_cache(): + luclient = pathlib.Path('app/luclient/res') + files = [path for path in luclient.rglob("*.lxfml") if path.is_file()] + pool = Pool(processes=4) + pool.map(partial(convert_lxfml_to_obj, lod=0), files) + pool.map(partial(convert_lxfml_to_obj, lod=1), files) + pool.map(partial(convert_lxfml_to_obj, lod=2), files) + +def convert_lxfml_to_obj(file, lod): + mtl = get_cache_file(file).with_suffix(f".lod{lod}.mtl") + if not mtl.exists(): + mtl.parent.mkdir(parents=True, exist_ok=True) + print(f"Convert LXFML {file.as_posix()} to obj and mtl @ {mtl}") + try: + ldd.main(str(file.as_posix()), str(mtl.with_suffix("").as_posix()), lod) # convert to OBJ + except Exception as e: + print(f"ERROR on {file}:\n {e}") + else: + # print(f"Already Exists: {file} with LOD {lod}") + return + +def get_cache_file(path): + # convert to list so that we can change elements + parts = list(path.parts) + + # replace part that matches src with dst + parts[parts.index("luclient")] = "cache" + del parts[parts.index("res")] + + return pathlib.Path(*parts) def find_or_create_account(name, email, password, gm_level=9): """ Find existing account or create new account """ @@ -119,5 +173,3 @@ def find_or_create_account(name, email, password, gm_level=9): db.session.add(play_key) db.session.commit() return # account - - diff --git a/app/properties.py b/app/properties.py index c90c5b3..ef4b936 100644 --- a/app/properties.py +++ b/app/properties.py @@ -367,19 +367,13 @@ def prebuilt(content, file_format): cache = f"app/cache/{filename}.{file_format}" - if os.path.exists(cache): - with open(cache, 'r') as file: - cache_data = file.read() - response = make_response(cache_data) - - else: + if not os.path.exists(cache): lxfml = f'app/luclient/res/BrickModels/{filename.split(".")[0]}.lxfml' ldd.main(lxfml, cache.split('.')[0]) # convert to OBJ - if os.path.exists(cache): - with open(cache, 'r') as file: - cache_data = file.read() - response = make_response(cache_data) + with open(cache, 'r') as file: + cache_data = file.read() + response = make_response(cache_data) else: raise(Exception("INVALID FILE FORMAT")) diff --git a/app/pylddlib.py b/app/pylddlib.py index 1054cb1..f482c8d 100644 --- a/app/pylddlib.py +++ b/app/pylddlib.py @@ -30,7 +30,7 @@ if sys.version_info < (3, 0): sys.setdefaultencoding('utf-8') PRIMITIVEPATH = '/Primitives/' -GEOMETRIEPATH = PRIMITIVEPATH + 'LOD0/' +GEOMETRIEPATH = PRIMITIVEPATH DECORATIONPATH = '/Decorations/' MATERIALNAMESPATH = '/MaterialNames/' @@ -238,10 +238,10 @@ class Scene: for childnode in node.childNodes: if childnode.nodeName == 'BrickSet': self.Version = str(childnode.getAttribute('version')) - elif node.nodeName == 'Cameras': - for childnode in node.childNodes: - if childnode.nodeName == 'Camera': - self.Scenecamera.append(SceneCamera(node=childnode)) + # elif node.nodeName == 'Cameras': + # for childnode in node.childNodes: + # if childnode.nodeName == 'Camera': + # self.Scenecamera.append(SceneCamera(node=childnode)) elif node.nodeName == 'Bricks': for childnode in node.childNodes: if childnode.nodeName == 'Brick': @@ -720,8 +720,10 @@ class Converter: start_time = time.time() out = open(filename + ".obj.tmp", "w+") + out.truncate(0) out.write("mtllib " + filename + ".mtl" + '\n\n') outtext = open(filename + ".mtl.tmp", "w+") + outtext.truncate(0) total = len(self.scene.Bricks) current = 0 @@ -823,13 +825,13 @@ class Converter: sys.stdout.write('%s\r' % (' ')) # print("--- %s seconds ---" % (time.time() - start_time)) -def setDBFolderVars(dbfolderlocation): +def setDBFolderVars(dbfolderlocation, lod): global PRIMITIVEPATH global GEOMETRIEPATH global DECORATIONPATH global MATERIALNAMESPATH PRIMITIVEPATH = os.path.join(dbfolderlocation, 'Primitives', '') - GEOMETRIEPATH = os.path.join(dbfolderlocation, 'brickprimitives', 'lod0', '') + GEOMETRIEPATH = os.path.join(dbfolderlocation, 'brickprimitives', f'lod{lod}', '') DECORATIONPATH = os.path.join(dbfolderlocation, 'Decorations', '') MATERIALNAMESPATH = os.path.join(dbfolderlocation, 'MaterialNames', '') # print(MATERIALNAMESPATH) @@ -880,7 +882,7 @@ def progress(count, total, status='', suffix = ''): sys.stdout.write('Progress: [%s] %s%s %s %s\r' % (bar, percents, '%', suffix, status)) sys.stdout.flush() -def main(lxf_filename, obj_filename): +def main(lxf_filename, obj_filename, lod="2"): # print("- - - pylddlib - - -") # print(" _ ") # print(" [_]") @@ -890,10 +892,11 @@ def main(lxf_filename, obj_filename): # print(" [=|=]") # print("") # print("- - - - - - - - - - - -") - + global GEOMETRIEPATH + GEOMETRIEPATH = GEOMETRIEPATH + f"LOD{lod}/" converter = Converter() # print("Found DB folder. Will use this instead of db.lif!") - setDBFolderVars(dbfolderlocation = "app/luclient/res/") + setDBFolderVars(dbfolderlocation = "app/luclient/res/", lod=lod) converter.LoadDBFolder(dbfolderlocation = "app/luclient/res/") converter.LoadScene(filename=lxf_filename) converter.Export(filename=obj_filename) From 869bbdf7e6b7f208ac7b08b09a49250c93c568fe Mon Sep 17 00:00:00 2001 From: Aaron Kimbre Date: Fri, 11 Feb 2022 17:37:43 -0600 Subject: [PATCH 2/3] Lod picker --- app/properties.py | 38 ++++++++++--------- app/templates/ldd/ldd.html.j2 | 20 ++++++---- app/templates/partials/_property.html.j2 | 12 +++++- .../partials/_property_content.html.j2 | 33 +++++++++++----- 4 files changed, 67 insertions(+), 36 deletions(-) diff --git a/app/properties.py b/app/properties.py index ef4b936..dd78bd4 100644 --- a/app/properties.py +++ b/app/properties.py @@ -23,6 +23,7 @@ import zlib import xmltodict import os import app.pylddlib as ldd +import pathlib property_blueprint = Blueprint('properties', __name__) @@ -206,16 +207,16 @@ def get(status="all"): return data -@property_blueprint.route('/view_model/', methods=['GET']) +@property_blueprint.route('/view_model//', methods=['GET']) @login_required -def view_model(id): +def view_model(id, lod): property_content_data = PropertyContent.query.filter(PropertyContent.id==id).all() # TODO: Restrict somehow formatted_data = [ { - "obj": url_for('properties.get_model', id=property_content_data[0].id, file_format='obj'), - "mtl": url_for('properties.get_model', id=property_content_data[0].id, file_format='mtl'), + "obj": url_for('properties.get_model', id=property_content_data[0].id, file_format='obj', lod=lod), + "mtl": url_for('properties.get_model', id=property_content_data[0].id, file_format='mtl', lod=lod), "lot": property_content_data[0].lot, "id": property_content_data[0].id, "pos": [{ @@ -232,7 +233,8 @@ def view_model(id): return render_template( 'ldd/ldd.html.j2', - content=formatted_data + content=formatted_data, + lod=lod ) property_center = { @@ -245,9 +247,9 @@ property_center = { } -@property_blueprint.route('/view_models/', methods=['GET']) +@property_blueprint.route('/view_models//', methods=['GET']) @login_required -def view_models(id): +def view_models(id, lod): property_content_data = PropertyContent.query.filter( PropertyContent.property_id==id ).order_by(PropertyContent.lot).all() @@ -273,8 +275,8 @@ def view_models(id): # add new lot consolidated_list.append( { - "obj": url_for('properties.get_model', id=property_content_data[item].id, file_format='obj'), - "mtl": url_for('properties.get_model', id=property_content_data[item].id, file_format='mtl'), + "obj": url_for('properties.get_model', id=property_content_data[item].id, file_format='obj', lod=lod), + "mtl": url_for('properties.get_model', id=property_content_data[item].id, file_format='mtl', lod=lod), "lot": property_content_data[item].lot, "id": property_content_data[item].id, "pos": [{ @@ -293,18 +295,20 @@ def view_models(id): 'ldd/ldd.html.j2', property_data=property_data, content=consolidated_list, - center=property_center[property_data.zone_id] + center=property_center[property_data.zone_id], + lod=lod ) -@property_blueprint.route('/get_model//', methods=['GET']) +@property_blueprint.route('/get_model///', methods=['GET']) @login_required -def get_model(id, file_format): +def get_model(id, file_format, lod): content = PropertyContent.query.filter(PropertyContent.id==id).first() - + if not(0 <= int(lod) <= 2): + abort(404) if content.lot == 14: # ugc model response = ugc(content)[0] else: # prebuild model - response = prebuilt(content, file_format)[0] + response = prebuilt(content, file_format, lod)[0] response.headers.set('Content-Type', 'text/xml') return response @@ -337,7 +341,7 @@ def ugc(content): return response, ugc_data.filename -def prebuilt(content, file_format): +def prebuilt(content, file_format, lod): # translate LOT to component id # we need to get a type of 2 because reasons render_component_id = query_cdclient( @@ -357,7 +361,7 @@ def prebuilt(content, file_format): return f"No filename for LOT {content.lot}" if file_format == "lxfml": - lxfml = f'app/luclient/res/BrickModels/{filename.split(".")[0]}.lxfml' + lxfml = pathilob.Path(f'app/luclient/res/BrickModels/{filename.split(".")[0]}.lxfml') with open(lxfml, 'r') as file: lxfml_data = file.read() # print(lxfml_data) @@ -365,7 +369,7 @@ def prebuilt(content, file_format): elif file_format in ["obj", "mtl"]: - cache = f"app/cache/{filename}.{file_format}" + cache = f"app/cache/BrickModels/{filename}.lod{lod}.{file_format}" if not os.path.exists(cache): lxfml = f'app/luclient/res/BrickModels/{filename.split(".")[0]}.lxfml' diff --git a/app/templates/ldd/ldd.html.j2 b/app/templates/ldd/ldd.html.j2 index 7dd506d..7fde421 100644 --- a/app/templates/ldd/ldd.html.j2 +++ b/app/templates/ldd/ldd.html.j2 @@ -467,16 +467,16 @@ constructor(designID, database){ this.designID = designID this.Parts = [] - this.studsFields2D = [] - let GeometryLocation = `${designID}.g` + let lod = {{ lod }} + let GeometryLocation = `lod${lod}/${designID}.g` let PrimitiveLocation = `${designID}.xml` let GeometryCount = 0 while (GeometryLocation in database.filelist) { this.Parts[GeometryCount] = new GeometryReader(database.filelist[GeometryLocation].read()) GeometryCount = GeometryCount + 1 - GeometryLocation = `${designID}.g${GeometryCount}` + GeometryLocation = `lod${lod}/${designID}.g${GeometryCount}` } let primitive = new Primitive(database.filelist[PrimitiveLocation].read()) this.Partname = primitive.Designname @@ -1019,7 +1019,7 @@ return self.filelist[filename]; } - parse(dburl) { + parse(dburl, folder="") { let self = this; let xhr = new XMLHttpRequest(); xhr.open('GET', dburl, false); @@ -1038,10 +1038,16 @@ let obj = data[i]; if (obj.type == 'directory'){ // parse subdirs - self.parse(dburl + obj.name + '/') + self.parse(dburl + obj.name + '/', obj.name) } else if (obj.type == 'file'){ - self.filelist[obj.name] = new DBURLFile(dburl + obj.name, obj.name) + + if (folder.includes("lod")){ + + self.filelist[`${folder}/${obj.name}`] = new DBURLFile(dburl + obj.name, obj.name) + } else { + self.filelist[obj.name] = new DBURLFile(dburl + obj.name, obj.name) + } } else { console.log('Strange object parsed: ' + obj.type) @@ -1063,7 +1069,7 @@ let lxfml_file_list = [ {% for model in content %} {% if model.lot == 14 %} - "{{url_for('properties.get_model', id=model.id, file_format='lxfml')}}"{{ ", " if not loop.last else "" }} + "{{url_for('properties.get_model', id=model.id, file_format='lxfml', lod=lod)}}"{{ ", " if not loop.last else "" }} {% endif %} {% endfor %} ] diff --git a/app/templates/partials/_property.html.j2 b/app/templates/partials/_property.html.j2 index 3b301ec..7ba689a 100644 --- a/app/templates/partials/_property.html.j2 +++ b/app/templates/partials/_property.html.j2 @@ -84,8 +84,16 @@
- Render Property + href='{{ url_for('properties.view_models', id=property.id, lod=0) }}'> + Render: High + + + Render: Medium + + + Render: Low
diff --git a/app/templates/partials/_property_content.html.j2 b/app/templates/partials/_property_content.html.j2 index 2830d4e..859a189 100644 --- a/app/templates/partials/_property_content.html.j2 +++ b/app/templates/partials/_property_content.html.j2 @@ -44,21 +44,12 @@
+ href='{{ url_for('properties.get_model', id=item.id, file_format="lxfml", lod=0) }}'> View Model XML

-
-
- - Render Model - -
-
-
+
+
Render
+
+
+ + 0 + +
+
+ + 1 + +
+
+ + 2 + +
+
From f0df3572582cde2ee45108bf8e066c91ecaa9a1f Mon Sep 17 00:00:00 2001 From: Aaron Kimbre Date: Fri, 11 Feb 2022 17:51:07 -0600 Subject: [PATCH 3/3] Fix layout a bit --- app/templates/partials/_property.html.j2 | 11 ++++++++--- app/templates/partials/_property_content.html.j2 | 14 +++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/app/templates/partials/_property.html.j2 b/app/templates/partials/_property.html.j2 index 7ba689a..14012a9 100644 --- a/app/templates/partials/_property.html.j2 +++ b/app/templates/partials/_property.html.j2 @@ -81,19 +81,24 @@ {% else %}
+
Render Quality
- Render: High + High (0) +
+
- Render: Medium + Med (1) +
+
- Render: Low + Low (2)
diff --git a/app/templates/partials/_property_content.html.j2 b/app/templates/partials/_property_content.html.j2 index 859a189..1712ab5 100644 --- a/app/templates/partials/_property_content.html.j2 +++ b/app/templates/partials/_property_content.html.j2 @@ -59,24 +59,24 @@
-
Render
+
Render Quality
- - 0 + High (0)
- - 1 + Med (1)
- - 2 + Low (2)