diff --git a/dDashboardServer/routes/ClientAssets.cpp b/dDashboardServer/routes/ClientAssets.cpp index f9400a381..b6b2391bb 100644 --- a/dDashboardServer/routes/ClientAssets.cpp +++ b/dDashboardServer/routes/ClientAssets.cpp @@ -8,6 +8,7 @@ #include #include "OnceCache.h" +#include "UgcBricks.h" #include "RouteUtils.h" #include "CDClientDatabase.h" #include "Game.h" @@ -451,6 +452,28 @@ namespace { } void RegisterClientAssetRoutes() { + Route(eHTTPMethod::GET, "/api/bricks/materials.js", 0, + "The brick colours (MatID -> [r, g, b, a]) from Materials.xml in the client's res/brickdb.zip, as a script setting " + "window.LDD_MATERIALS for the 3D viewers. Read once; an empty table when the client's brick database can't be read", + [](HTTPReply& reply, const HTTPContext&) { + // Read on first use (thread-safe static init); the client only changes with a restart + static const std::string script = [] { + nlohmann::json colours = nlohmann::json::object(); + const auto zip = ClientAssets::ReadResFile("brickdb.zip"); + const auto xml = zip ? UgcBricks::ReadZipEntry(*zip, "Materials.xml") : std::nullopt; + if (xml) { + for (const auto& [id, m] : UgcBricks::ParseMaterials(*xml)) colours[std::to_string(id)] = { m.r, m.g, m.b, m.a }; + } else { + LOG("Couldn't read Materials.xml from the client's brickdb.zip; the 3D viewers' bricks will be grey"); + } + return "window.LDD_MATERIALS = " + colours.dump() + ";\n"; + }(); + reply.status = eHTTPStatusCode::OK; + reply.message = script; + reply.contentType = eContentType::TEXT_JAVASCRIPT; + reply.headers.push_back("Cache-Control: private, max-age=3600"); + }); + Route(eHTTPMethod::GET, "/api/icon/:lot", 0, "PNG icon for an item LOT (requires client_location and ImageMagick)", [](HTTPReply& reply, const HTTPContext& context) { const auto lot = PathId(context.path, 2); diff --git a/dDashboardServer/static/js/ldd-materials.js b/dDashboardServer/static/js/ldd-materials.js deleted file mode 100644 index b3b6be413..000000000 --- a/dDashboardServer/static/js/ldd-materials.js +++ /dev/null @@ -1,171 +0,0 @@ -/** - * LDD material colours (MatID -> [r, g, b, a]) from Materials.xml in the LEGO Universe client's res/brickdb.zip. - * Static game data; regenerate only if the client's brick database changes. - */ -window.LDD_MATERIALS = { - 1: [244, 244, 244, 255], - 2: [138, 146, 141, 255], - 3: [255, 214, 127, 255], - 4: [242, 112, 94, 255], - 5: [217, 187, 123, 255], - 6: [173, 217, 168, 255], - 7: [255, 133, 0, 255], - 8: [140, 0, 255, 255], - 9: [246, 169, 187, 255], - 10: [255, 255, 189, 150], - 11: [171, 217, 255, 255], - 12: [216, 109, 44, 255], - 13: [255, 128, 20, 255], - 14: [120, 252, 120, 255], - 15: [255, 242, 48, 255], - 16: [255, 135, 156, 255], - 17: [255, 148, 148, 255], - 18: [214, 114, 64, 255], - 19: [207, 138, 71, 255], - 20: [233, 233, 233, 150], - 21: [222, 0, 13, 255], - 22: [208, 80, 152, 255], - 23: [0, 87, 168, 255], - 24: [254, 196, 0, 255], - 25: [84, 51, 36, 255], - 26: [50, 50, 50, 255], - 27: [84, 89, 85, 255], - 28: [0, 123, 40, 255], - 29: [127, 196, 117, 255], - 36: [253, 195, 131, 255], - 37: [0, 150, 36, 255], - 38: [168, 61, 21, 255], - 39: [175, 190, 214, 255], - 40: [238, 238, 238, 150], - 41: [224, 42, 41, 150], - 42: [182, 224, 239, 150], - 43: [80, 177, 232, 150], - 44: [249, 239, 105, 150], - 45: [151, 203, 217, 255], - 47: [231, 102, 72, 150], - 48: [99, 178, 110, 150], - 49: [250, 237, 91, 150], - 50: [229, 223, 211, 255], - 100: [249, 183, 165, 255], - 101: [240, 109, 97, 255], - 102: [71, 140, 198, 255], - 103: [188, 180, 165, 255], - 104: [103, 31, 129, 255], - 105: [245, 134, 36, 255], - 106: [231, 99, 24, 255], - 107: [6, 157, 159, 255], - 108: [86, 71, 47, 255], - 109: [0, 20, 20, 255], - 110: [38, 70, 154, 255], - 111: [189, 173, 162, 150], - 112: [72, 97, 172, 255], - 113: [238, 157, 195, 150], - 114: [254, 0, 254, 150], - 115: [183, 212, 37, 255], - 116: [0, 170, 164, 255], - 117: [247, 247, 247, 150], - 118: [156, 214, 204, 255], - 119: [149, 185, 11, 255], - 120: [216, 228, 141, 255], - 121: [248, 154, 57, 255], - 122: [254, 203, 152, 255], - 123: [238, 84, 52, 255], - 124: [156, 0, 107, 255], - 125: [249, 167, 119, 255], - 126: [156, 149, 199, 150], - 127: [222, 172, 102, 255], - 128: [173, 97, 64, 255], - 129: [67, 84, 147, 255], - 131: [141, 148, 150, 255], - 133: [239, 88, 40, 255], - 134: [205, 221, 52, 255], - 135: [94, 116, 140, 255], - 136: [117, 101, 125, 255], - 137: [244, 129, 71, 255], - 138: [141, 116, 82, 255], - 139: [116, 73, 48, 255], - 140: [0, 37, 65, 255], - 141: [0, 52, 22, 255], - 143: [206, 227, 246, 150], - 145: [91, 117, 144, 255], - 146: [129, 117, 144, 255], - 147: [131, 114, 79, 255], - 148: [73, 63, 59, 255], - 149: [10, 19, 39, 255], - 150: [152, 155, 153, 255], - 151: [95, 130, 101, 255], - 153: [136, 96, 94, 255], - 154: [128, 8, 27, 255], - 157: [255, 246, 92, 150], - 158: [241, 142, 187, 150], - 168: [96, 86, 76, 255], - 176: [148, 81, 72, 255], - 178: [171, 103, 58, 255], - 179: [115, 114, 113, 255], - 180: [221, 152, 46, 255], - 182: [236, 118, 14, 150], - 183: [246, 242, 223, 255], - 184: [214, 0, 38, 255], - 185: [0, 89, 163, 255], - 186: [0, 142, 60, 255], - 187: [87, 57, 44, 255], - 188: [0, 158, 206, 255], - 189: [172, 130, 71, 255], - 190: [255, 207, 11, 255], - 191: [244, 155, 0, 255], - 192: [91, 28, 12, 255], - 193: [236, 68, 29, 255], - 194: [156, 146, 145, 255], - 195: [28, 88, 167, 255], - 196: [14, 62, 154, 255], - 197: [49, 43, 135, 255], - 198: [138, 18, 168, 255], - 199: [76, 81, 86, 255], - 200: [106, 121, 68, 255], - 208: [228, 228, 218, 255], - 209: [164, 118, 36, 255], - 210: [70, 138, 95, 255], - 211: [63, 182, 169, 255], - 212: [135, 192, 234, 255], - 213: [71, 111, 182, 255], - 216: [135, 43, 23, 255], - 217: [123, 93, 65, 255], - 218: [142, 85, 151, 255], - 219: [86, 78, 157, 255], - 220: [145, 149, 202, 255], - 221: [222, 55, 139, 255], - 222: [238, 157, 195, 255], - 223: [241, 120, 128, 255], - 224: [243, 201, 136, 255], - 225: [250, 169, 100, 255], - 226: [255, 228, 104, 255], - 227: [201, 231, 136, 150], - 228: [85, 165, 175, 150], - 229: [172, 212, 222, 150], - 230: [236, 163, 201, 150], - 231: [252, 183, 109, 150], - 232: [119, 201, 216, 255], - 233: [96, 186, 118, 255], - 234: [251, 232, 144, 150], - 236: [141, 115, 179, 150], - 268: [44, 21, 119, 255], - 269: [62, 149, 182, 255], - 283: [245, 193, 137, 255], - 284: [224, 208, 229, 150], - 285: [228, 214, 218, 150], - 294: [254, 252, 213, 150], - 296: [173, 173, 173, 255], - 297: [170, 127, 46, 255], - 298: [118, 118, 118, 255], - 308: [53, 33, 0, 255], - 311: [175, 210, 70, 150], - 312: [170, 125, 85, 255], - 315: [140, 140, 140, 255], - 316: [62, 60, 57, 255], - 321: [70, 155, 195, 255], - 322: [104, 195, 226, 255], - 323: [211, 242, 234, 255], - 324: [160, 110, 185, 255], - 325: [205, 164, 222, 255], - 326: [226, 249, 154, 255] -}; diff --git a/dDashboardServer/templates/cdclient.jinja2 b/dDashboardServer/templates/cdclient.jinja2 index 75c853c1c..34ce56810 100644 --- a/dDashboardServer/templates/cdclient.jinja2 +++ b/dDashboardServer/templates/cdclient.jinja2 @@ -55,6 +55,6 @@ } } - + {% endblock %} diff --git a/dDashboardServer/templates/property-3d.jinja2 b/dDashboardServer/templates/property-3d.jinja2 index aaf94bf88..541e7db1c 100644 --- a/dDashboardServer/templates/property-3d.jinja2 +++ b/dDashboardServer/templates/property-3d.jinja2 @@ -118,7 +118,7 @@ } } - + - + diff --git a/dUgcServer/UgcBricks.cpp b/dUgcServer/UgcBricks.cpp index d2d27ccc4..3cc1e3609 100644 --- a/dUgcServer/UgcBricks.cpp +++ b/dUgcServer/UgcBricks.cpp @@ -161,11 +161,19 @@ namespace UgcBricks { return contents.str(); } - BrickLibrary::BrickLibrary(std::filesystem::path res, uint32_t lod) : m_Res(std::move(res)), m_Lod(std::min(lod, 2)) {} + BrickLibrary::BrickLibrary(std::filesystem::path res, uint32_t lod, FileReader reader) + : m_Res(std::move(res)), m_Lod(std::min(lod, 2)), m_Reader(std::move(reader)) {} + + std::optional BrickLibrary::Read(const std::string& relative) const { + if (m_Reader) { + if (auto data = m_Reader(relative)) return data; + } + const auto path = ResolvePath(m_Res, relative); + return path ? ReadFile(*path) : std::nullopt; + } bool BrickLibrary::LoadMaterials() { - const auto zipPath = ResolvePath(m_Res, "brickdb.zip"); - const auto zip = zipPath ? ReadFile(*zipPath) : std::nullopt; + const auto zip = Read("brickdb.zip"); const auto xml = zip ? ReadZipEntry(*zip, "Materials.xml") : std::nullopt; if (!xml) return false; auto materials = ParseMaterials(*xml); @@ -185,6 +193,11 @@ namespace UgcBricks { return it != m_Materials.end() ? it->second : Material{}; } + bool BrickLibrary::HasMaterial(uint32_t id) const { + // Written once before the workers start, only read after + return m_Materials.contains(id); + } + std::shared_ptr> BrickLibrary::GetDesign(uint32_t design, std::optional lodLevel) { const uint32_t lod = std::min(lodLevel.value_or(m_Lod), 2); const uint64_t key = (static_cast(lod) << 32) | design; @@ -197,8 +210,7 @@ namespace UgcBricks { const auto folder = "brickprimitives/lod" + std::to_string(lod) + "/"; for (uint32_t index = 0; index < MAX_GEOMETRY_PARTS; index++) { const auto name = std::to_string(design) + ".g" + (index == 0 ? "" : std::to_string(index)); - const auto path = ResolvePath(m_Res, folder + name); - const auto data = path ? ReadFile(*path) : std::nullopt; + const auto data = Read(folder + name); if (!data) break; auto geometry = ParseGeometry(*data); if (!geometry) break; diff --git a/dUgcServer/UgcBricks.h b/dUgcServer/UgcBricks.h index 7d4347709..1bb82fbfd 100644 --- a/dUgcServer/UgcBricks.h +++ b/dUgcServer/UgcBricks.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -14,7 +15,8 @@ /** * The client's brick data the UGC server builds models from: LDD geometry (res/brickprimitives/lod/.g, .g1, * ...) and the material colors (Materials.xml in res/brickdb.zip). The parsers are pure; BrickLibrary loads and caches - * files and may be used from any thread. + * files and may be used from any thread. Files are read through a FileReader (the server's reads packed and unpacked + * clients alike), falling back to loose files under the res folder. */ namespace UgcBricks { // One LDD .g file: a triangle mesh with a normal per vertex @@ -49,10 +51,15 @@ namespace UgcBricks { // Reads a whole file; nullopt when it can't std::optional ReadFile(const std::filesystem::path& path); + // Reads a client file by its path under res/ ("brickdb.zip", "brickprimitives/lod0/3001.g"); nullopt when there is + // none. Called from any thread. + using FileReader = std::function(const std::string& relative)>; + class BrickLibrary { public: - // `res` is the client's res folder; `lod` the brickprimitives level used (0 is the most detailed) - BrickLibrary(std::filesystem::path res, uint32_t lod); + // `res` is the client's res folder; `lod` the brickprimitives level used (0 is the most detailed); `reader` + // the files' source, tried before loose files under `res` + BrickLibrary(std::filesystem::path res, uint32_t lod, FileReader reader = {}); // Loads the material colors; false when brickdb.zip or its Materials.xml can't be read bool LoadMaterials(); @@ -63,6 +70,9 @@ namespace UgcBricks { // A material's color (a grey when the id is unknown) Material GetMaterial(uint32_t id) const; + // Whether the client's Materials.xml has the id + bool HasMaterial(uint32_t id) const; + // Every geometry file of a design, in order (.g, .g1, ...); empty when the design has none. Loaded once. // `lod`: the brickprimitives level, the library's own when omitted; a design without that level uses the // next more detailed one. @@ -74,8 +84,12 @@ namespace UgcBricks { const std::filesystem::path& GetResPath() const { return m_Res; } private: + // A file from the reader, else loose under m_Res + std::optional Read(const std::string& relative) const; + std::filesystem::path m_Res; uint32_t m_Lod; + FileReader m_Reader; std::map m_Materials; mutable std::mutex m_Mutex; std::map>> m_Designs; // lod << 32 | design diff --git a/dUgcServer/UgcModel.cpp b/dUgcServer/UgcModel.cpp index f52a6804f..84e39e1e0 100644 --- a/dUgcServer/UgcModel.cpp +++ b/dUgcServer/UgcModel.cpp @@ -198,17 +198,22 @@ namespace UgcModel { continue; } model.bricks++; - const auto materialOf = [&part](size_t index) { + const auto materialOf = [&part, &library](size_t index) { auto id = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); - // Unknown colors are black in LU Toolbox (its name included, so black's variation too) - if (id == 0 || !UgcPalette::Linear(id)) id = UgcPalette::FALLBACK_ID; + // Unknown colors are black in LU Toolbox (its name included, so black's variation too). A color LU + // Toolbox doesn't know but the client's Materials.xml has (one added to the brick database) keeps its id + // and takes its color from there. + if (id == 0 || (!UgcPalette::Linear(id) && !library.HasMaterial(id))) id = UgcPalette::FALLBACK_ID; return id; }; + // Whether LU Toolbox's own palette colors a material (else it's one only the client's Materials.xml has) + const auto inToolbox = [](uint32_t id) { return UgcPalette::Linear(id).has_value(); }; // A brick is transparent only when all of its materials are (LU Toolbox's IS_TRANSPARENT) bool transparent = true; for (size_t index = 0; index < design->size(); index++) { const auto id = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); - transparent = transparent && (luToolbox ? UgcPalette::IsTransparent(materialOf(index)) : library.GetMaterial(id).Transparent()); + const auto toolboxId = materialOf(index); + transparent = transparent && (luToolbox && inToolbox(toolboxId) ? UgcPalette::IsTransparent(toolboxId) : library.GetMaterial(luToolbox ? toolboxId : id).Transparent()); } auto& mesh = transparent ? model.transparent : model.opaque; if (transparent) model.transparentBricks.push_back(mesh.indices.size()); @@ -219,7 +224,13 @@ namespace UgcModel { float alpha = 1.0f; glm::vec3 glow(0.0f); uint32_t colorId{}; - if (luToolbox) { + if (luToolbox && !inToolbox(materialOf(index))) { + // A color only the client's Materials.xml has + colorId = materialOf(index); + const auto material = library.GetMaterial(colorId); + linear = UgcPalette::SrgbToLinear(glm::vec3(material.r, material.g, material.b) / 255.0f); + if (transparent) alpha = material.a / 255.0f; + } else if (luToolbox) { colorId = materialOf(index); linear = *UgcPalette::Linear(colorId, options.icon); if (transparent) alpha = std::clamp(options.transparentOpacity / 100.0f, 0.0f, 1.0f); diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 6d65888f4..5c032f0db 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -12,6 +12,7 @@ #include #include +#include "AssetManager.h" #include "BinaryPathFinder.h" #include "CDClientDatabase.h" #include "ConfigSync.h" @@ -389,11 +390,37 @@ namespace { return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res"; } + // Brick files through the client's assets, packed or unpacked (loose files first, as the game reads them). Made on + // the main thread; the AssetManager is only read after that, so the workers may call it at once. + UgcBricks::FileReader ClientReader() { + const auto client = Game::config->GetValue("client_location"); + if (client.empty()) return {}; + std::shared_ptr assets; + try { + assets = std::make_shared(client); + } catch (const std::exception& e) { + LOG("Couldn't open the client's assets at %s (%s); reading loose files only", client.c_str(), e.what()); + return {}; + } + return [assets](const std::string& relative) -> std::optional { + char* data = nullptr; + uint32_t length = 0; + try { + if (!assets->GetFile(relative, &data, &length)) return std::nullopt; + } catch (const std::exception&) { + return std::nullopt; // a pack the index names but that isn't there + } + std::string out(data, length); + free(data); + return out; + }; + } + // Command line tools: make one model's or modular build's files into a folder, without a database int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) { const auto res = ResPath(); const auto settings = ReadSettings(); - UgcBricks::BrickLibrary library(res, 0); + UgcBricks::BrickLibrary library(res, 0, ClientReader()); if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n"; UgcJobs::Outcome outcome; const auto start = std::chrono::steady_clock::now(); @@ -491,7 +518,7 @@ int main(int argc, char** argv) { ServiceType::UGC, Game::config, &Game::lastSignal, masterPassword); Game::server = g_Server; - UgcBricks::BrickLibrary library(res, 0); + UgcBricks::BrickLibrary library(res, 0, ClientReader()); if (!library.LoadMaterials()) LOG("Couldn't read Materials.xml from %s; bricks will be grey", (res / "brickdb.zip").string().c_str()); auto outputDir = std::filesystem::path(Game::config->GetValue("ugc_output_dir").empty() ? "ugc" : Game::config->GetValue("ugc_output_dir")); diff --git a/docs/UgcServer.md b/docs/UgcServer.md index c38f0e2c9..270b5b2c0 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -67,9 +67,11 @@ defaults. The table below goes through it step by step. 1. The LXFML is read from `ugc.lxfml` (an sd0 stream). Parts come from `Bricks/Brick/Part` (LXFML 5: row-major rotation and translation per bone) or `Scene/Model/Group/Part` (LXFML 4: axis angle). 2. Each level of detail in `lods` (default `0,2`, as LU Toolbox imports) is built from the client's LDD primitives, - `res/brickprimitives/lod/.g`, `.g1`, ... (sub-part `i` uses the part's `i`-th material, material 0 meaning + `res/brickprimitives/lod/.g`, `.g1`, ... (read like the game does: loose files first, then the client's + packs, so packed clients and bricks added to them work) (sub-part `i` uses the part's `i`-th material, material 0 meaning the part's first). Colors come from LU Toolbox's palette (`color_palette=lu_toolbox`; `brickdb` uses the client's - `Materials.xml`): LU's colors, the LDD colors LU doesn't have mapped onto the nearest LU one, unknown ones black. A + `Materials.xml`): LU's colors, the LDD colors LU doesn't have mapped onto the nearest LU one, colors LU Toolbox doesn't know but the + client's `Materials.xml` has (colors added to the brick database) from `Materials.xml`, unknown ones black. A brick is transparent only when all of its materials are; transparent bricks get `transparent_opacity` (58.82%). 3. Color variation: each material of each brick has its brightness shifted like LU Toolbox's "Apply Color Variation": the color's HSV value is taken to a 1/2.224 gamma, moved by a random amount of up to `color_variation`/200 (5%: diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 4fce61622..fcd6d3049 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -161,6 +161,33 @@ TEST(UgcModel, BuildsOpaqueAndTransparentMeshes) { EXPECT_NEAR(model.opaque.positions[0].x, 10.0f, 1e-5f); } +// A color LU Toolbox's palette doesn't have but the client's Materials.xml does (one added to the brick database) is +// drawn in its Materials.xml color; one neither knows is LU Toolbox's black +TEST(UgcModel, ColorsOnlyInMaterialsXmlAreNotBlack) { + UgcBricks::BrickLibrary library(MakeRes(), 0); + constexpr uint32_t ADDED = 50001; + ASSERT_FALSE(UgcPalette::Linear(ADDED)); + library.SetMaterials({ { ADDED, { 0, 200, 100, 255 } } }); + std::string error; + UgcModel::BuildOptions options; + options.palette = UgcModel::ePalette::LU_TOOLBOX; + options.colorVariation = 0.0f; + const auto brick = [&error](uint32_t material) { + return UgcModel::ParseLxfml("", error); + }; + const auto added = UgcModel::Build(brick(ADDED), library, options); + ASSERT_FALSE(added.opaque.colors.empty()); + EXPECT_NEAR(added.opaque.colors[0].g, 200.0f / 255.0f, 1e-3f); + EXPECT_NEAR(added.opaque.colors[0].r, 0.0f, 1e-3f); + ASSERT_FALSE(UgcPalette::Linear(50002)); + const auto unknown = UgcModel::Build(brick(50002), library, options); + ASSERT_FALSE(unknown.opaque.colors.empty()); + const auto black = UgcModel::Build(brick(UgcPalette::FALLBACK_ID), library, options); + ASSERT_FALSE(black.opaque.colors.empty()); + EXPECT_EQ(unknown.opaque.colors[0], black.opaque.colors[0]); +} + TEST(UgcModel, SplitsBigMeshes) { UgcModel::Mesh mesh; for (uint32_t i = 0; i < 30; i++) {