mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-14 16:44:09 +00:00
feat(ugc): answer the client's UGC manifest requests without 3D services
With UGCUSE3DSERVICES=7:0 (the client's default) the client asks its world for a blueprint file's MD5 and size (REQUEST_UGC_MANIFEST_INFO, world 27) and then downloads BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0. Layouts checked in the 1.10.64 client: the request is a u64 blueprint id and a u8 resource type; the answer (UGC_MANIFEST_RESPONSE, client 60) repeats them and adds a u8 valid, the u32 size and the 16 byte MD5 of the inflated file, 37 bytes after the 0x53 exactly or the client drops it. - dNet: WorldPackets::RequestUgcManifestInfo, ClientPackets::UgcManifestResponse (eUgcResourceType), with byte tests against the client's layouts. - Database: ugc_file_checksums (per model or module combination and file) and ugc_modular_build.combination_id (migrations 89 / 72), GetUgcFileChecksum looks a blueprint up as a model, else as a build through its combination. - UGC server: every download is also written as .sd0 (Sd0::Compress); workers hand the checksums back and the main thread stores them; old items get their sd0 icon and checksum, and builds their combination id, once at start-up, a few per tick; serves <dir>/BrickModels/UserMade/<bucket>/<id>.<ext>.sd0 under client_path, /<folder>/UserBrickModels and the root (.hkx 404). - World: UgcManifest answers on the main thread with one indexed query per request; files not made yet are answered when they are (looked at again every 5 seconds), and a model in its quiet period is made right away. No worker threads, HTTP or file reads in the world. Off by default (ugc_manifest=0): checked in game, the client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says and logs the player out when it can't connect, so icons need the UGC server on port 80 of each player's machine. docs/UgcServer.md has the details. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
15
dCommon/dEnums/eUgcResourceType.h
Normal file
15
dCommon/dEnums/eUgcResourceType.h
Normal file
@@ -0,0 +1,15 @@
|
||||
#ifndef EUGCRESOURCETYPE_H
|
||||
#define EUGCRESOURCETYPE_H
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
// The kinds of file the client keeps for a blueprint (a player's model or a car or rocket), as it numbers them in
|
||||
// REQUEST_UGC_MANIFEST_INFO and UGC_MANIFEST_RESPONSE and in its download paths
|
||||
enum class eUgcResourceType : uint8_t {
|
||||
LXFML = 0,
|
||||
NIF,
|
||||
HKX,
|
||||
DDS
|
||||
};
|
||||
|
||||
#endif //!EUGCRESOURCETYPE_H
|
||||
@@ -417,6 +417,7 @@ namespace {
|
||||
c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST));
|
||||
c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012"));
|
||||
c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true));
|
||||
c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models. The client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and one that can't connect there is logged out: only turn on when the UGC server answers there for every player.", false));
|
||||
c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL));
|
||||
c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL));
|
||||
c.AddSection("UGC processing", "When models are made and how much of the machine the workers may use.", eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } });
|
||||
|
||||
@@ -107,5 +107,26 @@ public:
|
||||
|
||||
// How many models are in each state
|
||||
virtual std::vector<std::pair<eProcessState, uint64_t>> GetUgcProcessCounts() = 0;
|
||||
|
||||
// ---- Checksums of the files the UGC server made (ugc_file_checksums), for the client's manifest requests ----
|
||||
|
||||
// What the files are stored under: a player model's ugc id, or a combination of car or rocket modules
|
||||
enum class eFileOwner : int32_t {
|
||||
MODEL = 0,
|
||||
COMBINATION = 1
|
||||
};
|
||||
|
||||
// A made file as the client has it after inflating the download: its MD5 (32 lowercase hex digits) and size
|
||||
struct FileChecksum {
|
||||
std::string md5;
|
||||
uint32_t size{};
|
||||
};
|
||||
|
||||
// Records (or replaces) the checksum of a made file, e.g. "icon.dds" or "model.nif"
|
||||
virtual void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) = 0;
|
||||
|
||||
// The checksum of a blueprint's file: a player model's (the blueprint is its ugc id), else a car or rocket build's
|
||||
// (the blueprint is its ugc_modular_build id; the file is its combination's)
|
||||
virtual std::optional<FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) = 0;
|
||||
};
|
||||
#endif //!__IUGC__H__
|
||||
|
||||
@@ -29,6 +29,12 @@ public:
|
||||
virtual std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) = 0;
|
||||
virtual std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() = 0;
|
||||
|
||||
// The combination of modules a build was made as (UgcModularKey::StorageId of its modules), whose files it shares
|
||||
virtual void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) = 0;
|
||||
|
||||
// Up to `limit` builds whose combination isn't recorded yet (combination_id 0): their ids and modules
|
||||
virtual std::vector<PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) = 0;
|
||||
|
||||
// How many builds there are of each ldf_config as stored (the same modules may be written differently)
|
||||
virtual std::vector<std::pair<std::string, uint64_t>> GetModularBuildConfigCounts() = 0;
|
||||
|
||||
|
||||
@@ -429,6 +429,10 @@ public:
|
||||
void DeleteUgcIconSettings(const std::string_view target) override;
|
||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
||||
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override;
|
||||
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
|
||||
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
|
||||
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
|
||||
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override;
|
||||
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
|
||||
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
|
||||
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override;
|
||||
|
||||
@@ -181,3 +181,16 @@ void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) {
|
||||
void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) {
|
||||
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id);
|
||||
}
|
||||
|
||||
void MySQLDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
|
||||
ExecuteInsert("INSERT INTO ugc_file_checksums (kind, storage_id, file, md5, size) VALUES (?, ?, ?, ?, ?) "
|
||||
"ON DUPLICATE KEY UPDATE md5 = VALUES(md5), size = VALUES(size);", static_cast<int32_t>(owner), storageId, file, md5, size);
|
||||
}
|
||||
|
||||
std::optional<IUgc::FileChecksum> MySQLDatabase::GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) {
|
||||
auto result = ExecuteSelect("SELECT md5, size FROM ugc_file_checksums WHERE kind = 0 AND storage_id = ? AND file = ? "
|
||||
"UNION ALL SELECT f.md5, f.size FROM ugc_modular_build b JOIN ugc_file_checksums f ON f.kind = 1 AND f.storage_id = b.combination_id AND f.file = ? "
|
||||
"WHERE b.ugc_id = ? AND b.combination_id != 0 LIMIT 1;", blueprintId, file, file, blueprintId);
|
||||
if (!result->next()) return std::nullopt;
|
||||
return IUgc::FileChecksum{ std::string(result->getString("md5").c_str()), static_cast<uint32_t>(result->getUInt("size")) };
|
||||
}
|
||||
|
||||
@@ -106,3 +106,16 @@ void MySQLDatabase::SetUgcIconSettings(const std::string_view target, const std:
|
||||
void MySQLDatabase::DeleteUgcIconSettings(const std::string_view target) {
|
||||
ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target);
|
||||
}
|
||||
|
||||
void MySQLDatabase::SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) {
|
||||
ExecuteUpdate("UPDATE ugc_modular_build SET combination_id = ? WHERE ugc_id = ?;", combinationId, id);
|
||||
}
|
||||
|
||||
std::vector<IUgcModularBuild::PendingBuild> MySQLDatabase::GetModularBuildsWithoutCombination(const uint32_t limit) {
|
||||
auto result = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE combination_id = 0 LIMIT ?;", limit);
|
||||
std::vector<PendingBuild> builds;
|
||||
while (result->next()) {
|
||||
builds.push_back({ result->getInt64("ugc_id"), std::string(result->getString("ldf_config").c_str()), static_cast<uint32_t>(result->getInt("process_attempts")) });
|
||||
}
|
||||
return builds;
|
||||
}
|
||||
|
||||
@@ -413,6 +413,10 @@ public:
|
||||
void DeleteUgcIconSettings(const std::string_view target) override;
|
||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
||||
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override;
|
||||
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
|
||||
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
|
||||
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
|
||||
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override;
|
||||
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
|
||||
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
|
||||
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override;
|
||||
|
||||
@@ -184,3 +184,16 @@ void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) {
|
||||
void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) {
|
||||
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id);
|
||||
}
|
||||
|
||||
void SQLiteDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
|
||||
ExecuteInsert("INSERT INTO ugc_file_checksums (kind, storage_id, file, md5, size) VALUES (?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(kind, storage_id, file) DO UPDATE SET md5 = excluded.md5, size = excluded.size;", static_cast<int32_t>(owner), storageId, file, md5, size);
|
||||
}
|
||||
|
||||
std::optional<IUgc::FileChecksum> SQLiteDatabase::GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) {
|
||||
auto [_, result] = ExecuteSelect("SELECT md5, size FROM ugc_file_checksums WHERE kind = 0 AND storage_id = ? AND file = ? "
|
||||
"UNION ALL SELECT f.md5, f.size FROM ugc_modular_build b JOIN ugc_file_checksums f ON f.kind = 1 AND f.storage_id = b.combination_id AND f.file = ? "
|
||||
"WHERE b.ugc_id = ? AND b.combination_id != 0 LIMIT 1;", blueprintId, file, file, blueprintId);
|
||||
if (result.eof()) return std::nullopt;
|
||||
return IUgc::FileChecksum{ std::string(result.getStringField("md5", "")), static_cast<uint32_t>(result.getInt64Field("size")) };
|
||||
}
|
||||
|
||||
@@ -112,3 +112,17 @@ void SQLiteDatabase::SetUgcIconSettings(const std::string_view target, const std
|
||||
void SQLiteDatabase::DeleteUgcIconSettings(const std::string_view target) {
|
||||
ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target);
|
||||
}
|
||||
|
||||
void SQLiteDatabase::SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) {
|
||||
ExecuteUpdate("UPDATE ugc_modular_build SET combination_id = ? WHERE ugc_id = ?;", combinationId, id);
|
||||
}
|
||||
|
||||
std::vector<IUgcModularBuild::PendingBuild> SQLiteDatabase::GetModularBuildsWithoutCombination(const uint32_t limit) {
|
||||
auto [_, result] = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE combination_id = 0 LIMIT ?;", limit);
|
||||
std::vector<PendingBuild> builds;
|
||||
while (!result.eof()) {
|
||||
builds.push_back({ result.getInt64Field("ugc_id"), result.getStringField("ldf_config", ""), static_cast<uint32_t>(result.getIntField("process_attempts")) });
|
||||
result.nextRow();
|
||||
}
|
||||
return builds;
|
||||
}
|
||||
|
||||
@@ -411,6 +411,10 @@ class TestSQLDatabase : public GameDatabase {
|
||||
void DeleteUgcIconSettings(const std::string_view target) override {}
|
||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {}
|
||||
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override {}
|
||||
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {}
|
||||
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; }
|
||||
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {}
|
||||
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override { return {}; }
|
||||
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override { return {}; }
|
||||
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
|
||||
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override { return {}; }
|
||||
|
||||
@@ -17,6 +17,7 @@ set(DGAME_DUTILITIES_SOURCES "BrickDatabase.cpp"
|
||||
"PlayerReports.cpp"
|
||||
"Preconditions.cpp"
|
||||
"SlashCommandHandler.cpp"
|
||||
"UgcManifest.cpp"
|
||||
"VanityUtilities.cpp"
|
||||
"WorldMigration.cpp")
|
||||
|
||||
|
||||
102
dGame/dUtilities/UgcManifest.cpp
Normal file
102
dGame/dUtilities/UgcManifest.cpp
Normal file
@@ -0,0 +1,102 @@
|
||||
#include "UgcManifest.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <optional>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "ClientPackets.h"
|
||||
#include "Database.h"
|
||||
#include "dConfig.h"
|
||||
#include "Game.h"
|
||||
#include "Logger.h"
|
||||
|
||||
namespace {
|
||||
// How often the waiting requests are looked up again, how long they wait at most and how many there can be
|
||||
constexpr auto RETRY_INTERVAL = std::chrono::seconds(5);
|
||||
constexpr auto MAX_WAIT = std::chrono::minutes(15);
|
||||
constexpr size_t MAX_WAITING = 512;
|
||||
|
||||
struct Waiting {
|
||||
SystemAddress sysAddr;
|
||||
LWOOBJID blueprintId{};
|
||||
eUgcResourceType resourceType{};
|
||||
std::chrono::steady_clock::time_point since;
|
||||
};
|
||||
|
||||
std::vector<Waiting> g_Waiting;
|
||||
std::chrono::steady_clock::time_point g_NextRetry{};
|
||||
|
||||
// The UGC server's name for a blueprint's file of this type; nothing for the ones it doesn't make
|
||||
std::optional<std::string_view> FileName(eUgcResourceType type) {
|
||||
switch (type) {
|
||||
case eUgcResourceType::DDS: return "icon.dds";
|
||||
case eUgcResourceType::NIF: return "model.nif";
|
||||
default: return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
bool Enabled() {
|
||||
// Off unless set: the 1.10.64 client downloads from http://127.0.0.1:80 whatever its boot.cfg says, and one that
|
||||
// can't connect there is logged out (docs/UgcServer.md)
|
||||
return Game::config->GetValue("ugc_manifest") == "1";
|
||||
}
|
||||
|
||||
int HexDigit(char c) {
|
||||
if (c >= '0' && c <= '9') return c - '0';
|
||||
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
|
||||
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Answers when the file's checksum is known; false when it isn't (yet)
|
||||
bool TryAnswer(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
|
||||
const auto file = FileName(resourceType);
|
||||
if (!file) return false;
|
||||
const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, *file);
|
||||
if (!checksum || checksum->md5.size() != 32) return false;
|
||||
|
||||
ClientPackets::UgcManifestResponse response;
|
||||
response.blueprintId = blueprintId;
|
||||
response.resourceType = resourceType;
|
||||
response.fileSize = checksum->size;
|
||||
for (size_t i = 0; i < response.md5.size(); i++) {
|
||||
const auto high = HexDigit(checksum->md5[i * 2]);
|
||||
const auto low = HexDigit(checksum->md5[i * 2 + 1]);
|
||||
if (high < 0 || low < 0) return false;
|
||||
response.md5[i] = static_cast<uint8_t>(high << 4 | low);
|
||||
}
|
||||
response.valid = true;
|
||||
response.Send(sysAddr);
|
||||
LOG_DEBUG("Sent the UGC manifest of %llu (type %i): %s, %u bytes", static_cast<unsigned long long>(blueprintId),
|
||||
static_cast<int>(resourceType), checksum->md5.c_str(), checksum->size);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void UgcManifest::OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
|
||||
if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Enabled() || !FileName(resourceType)) return;
|
||||
if (TryAnswer(sysAddr, blueprintId, resourceType)) return;
|
||||
// Not made yet: a client wants it, so a model still in its quiet period after a save is made now (as when a client
|
||||
// asks the UGC server for it), and the request is answered when it's made
|
||||
Database::Get()->ExpediteUgcModel(blueprintId);
|
||||
for (const auto& waiting : g_Waiting) {
|
||||
if (waiting.sysAddr == sysAddr && waiting.blueprintId == blueprintId && waiting.resourceType == resourceType) return;
|
||||
}
|
||||
if (g_Waiting.size() >= MAX_WAITING) g_Waiting.erase(g_Waiting.begin());
|
||||
g_Waiting.push_back({ sysAddr, blueprintId, resourceType, std::chrono::steady_clock::now() });
|
||||
}
|
||||
|
||||
void UgcManifest::Update() {
|
||||
if (g_Waiting.empty()) return;
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
if (now < g_NextRetry) return;
|
||||
g_NextRetry = now + RETRY_INTERVAL;
|
||||
std::erase_if(g_Waiting, [now](const Waiting& waiting) {
|
||||
return now - waiting.since > MAX_WAIT || TryAnswer(waiting.sysAddr, waiting.blueprintId, waiting.resourceType);
|
||||
});
|
||||
}
|
||||
|
||||
void UgcManifest::OnDisconnect(const SystemAddress& sysAddr) {
|
||||
std::erase_if(g_Waiting, [&sysAddr](const Waiting& waiting) { return waiting.sysAddr == sysAddr; });
|
||||
}
|
||||
29
dGame/dUtilities/UgcManifest.h
Normal file
29
dGame/dUtilities/UgcManifest.h
Normal file
@@ -0,0 +1,29 @@
|
||||
#ifndef UGCMANIFEST_H
|
||||
#define UGCMANIFEST_H
|
||||
|
||||
#include "dCommonVars.h"
|
||||
#include "eUgcResourceType.h"
|
||||
#include "RakNetTypes.h"
|
||||
|
||||
/**
|
||||
* Answers the client's REQUEST_UGC_MANIFEST_INFO (sent when its UGCUSE3DSERVICES is off, the default): the MD5 and
|
||||
* size of a blueprint's file, which the client needs before it downloads the file from the UGC server
|
||||
* (BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0). The checksums are the ones the UGC server stored when it made the
|
||||
* files (ugc_file_checksums); a file it hasn't made yet is answered once it has (Update looks again every few seconds
|
||||
* for a while). Nothing is answered for files it never makes (LXFML, HKX), or unless `ugc_manifest` is 1: the client
|
||||
* then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and is logged out when
|
||||
* it can't connect there.
|
||||
*
|
||||
* Main thread only: each lookup is one indexed query and the answers are sent from here. See docs/UgcServer.md.
|
||||
*/
|
||||
namespace UgcManifest {
|
||||
void OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType);
|
||||
|
||||
// Answers the waiting requests whose files have been made since, and forgets the ones waited on too long
|
||||
void Update();
|
||||
|
||||
// A client left: its waiting requests are dropped
|
||||
void OnDisconnect(const SystemAddress& sysAddr);
|
||||
}
|
||||
|
||||
#endif //!UGCMANIFEST_H
|
||||
@@ -484,6 +484,25 @@ bool ClientPackets::BlueprintLoadItemResponse::Deserialize(RakNet::BitStream& bi
|
||||
return true;
|
||||
}
|
||||
|
||||
void ClientPackets::UgcManifestResponse::Serialize(RakNet::BitStream& bitStream) const {
|
||||
bitStream.Write(blueprintId);
|
||||
bitStream.Write(resourceType);
|
||||
bitStream.Write<uint8_t>(valid ? 1 : 0);
|
||||
bitStream.Write(fileSize);
|
||||
for (const auto byte : md5) bitStream.Write(byte);
|
||||
}
|
||||
|
||||
bool ClientPackets::UgcManifestResponse::Deserialize(RakNet::BitStream& bitStream) {
|
||||
VALIDATE_READ(bitStream.Read(blueprintId));
|
||||
VALIDATE_READ(bitStream.Read(resourceType));
|
||||
uint8_t validByte{};
|
||||
VALIDATE_READ(bitStream.Read(validByte));
|
||||
valid = validByte != 0;
|
||||
VALIDATE_READ(bitStream.Read(fileSize));
|
||||
for (auto& byte : md5) VALIDATE_READ(bitStream.Read(byte));
|
||||
return true;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// A zone ID as three fields: map (u16), instance (u16), clone (u32)
|
||||
void WriteZone(RakNet::BitStream& bitStream, const LWOZONEID& zoneID) {
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#ifndef CLIENTPACKETS_H
|
||||
#define CLIENTPACKETS_H
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <ctime>
|
||||
#include <set>
|
||||
@@ -18,6 +19,7 @@
|
||||
#include "eAddFriendResponseType.h"
|
||||
#include "eAddIgnoreResponse.h"
|
||||
#include "eBlueprintSaveResponseType.h"
|
||||
#include "eUgcResourceType.h"
|
||||
#include "MessageType/Client.h"
|
||||
#include "MessageType/Game.h"
|
||||
#include "NiPoint3.h"
|
||||
@@ -264,6 +266,23 @@ namespace ClientPackets {
|
||||
LWOOBJID itemId{};
|
||||
LWOOBJID destItemId{};
|
||||
};
|
||||
|
||||
// Server -> client. The answer to REQUEST_UGC_MANIFEST_INFO: the MD5 and size of one of a blueprint's files as the
|
||||
// client has it after downloading and inflating it (BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0 from the UGC
|
||||
// server). The client checks a file it has against it and downloads the file when it differs; with valid 0 it uses
|
||||
// a file it has as it is and downloads it only when it has none. The client ignores the answer unless it is exactly
|
||||
// this long (37 bytes after the 0x53).
|
||||
struct UgcManifestResponse : public LUBitStream {
|
||||
UgcManifestResponse() : LUBitStream(ServiceType::CLIENT, MessageType::Client::UGC_MANIFEST_RESPONSE) {}
|
||||
void Serialize(RakNet::BitStream& bitStream) const override;
|
||||
bool Deserialize(RakNet::BitStream& bitStream) override;
|
||||
|
||||
LWOOBJID blueprintId{};
|
||||
eUgcResourceType resourceType{};
|
||||
bool valid{}; // A byte
|
||||
uint32_t fileSize{}; // Of the inflated file
|
||||
std::array<uint8_t, 16> md5{}; // Of the inflated file, as bytes
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -340,6 +340,17 @@ namespace WorldPackets {
|
||||
return true;
|
||||
}
|
||||
|
||||
void RequestUgcManifestInfo::Serialize(RakNet::BitStream& bitStream) const {
|
||||
bitStream.Write(blueprintId);
|
||||
bitStream.Write(resourceType);
|
||||
}
|
||||
|
||||
bool RequestUgcManifestInfo::Deserialize(RakNet::BitStream& bitStream) {
|
||||
VALIDATE_READ(bitStream.Read(blueprintId));
|
||||
VALIDATE_READ(bitStream.Read(resourceType));
|
||||
return true;
|
||||
}
|
||||
|
||||
void UIHelpTop5::Serialize(RakNet::BitStream& bitStream) const {
|
||||
bitStream.Write(language);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "ChatPackets.h"
|
||||
#include "dCommonVars.h"
|
||||
#include "eFunnessTypes.h"
|
||||
#include "eUgcResourceType.h"
|
||||
#include "MessageType/Game.h"
|
||||
#include "MessageType/World.h"
|
||||
#include "PositionUpdate.h"
|
||||
@@ -201,6 +202,17 @@ namespace WorldPackets {
|
||||
bool Deserialize(RakNet::BitStream& bitStream) override;
|
||||
};
|
||||
|
||||
// Client -> server, when its UGCUSE3DSERVICES is off (the default): the MD5 and size of one of a blueprint's files
|
||||
// (a player's model, or a car or rocket's build), answered with ClientPackets::UgcManifestResponse.
|
||||
struct RequestUgcManifestInfo : public WorldLUBitStream {
|
||||
LWOOBJID blueprintId{};
|
||||
eUgcResourceType resourceType{};
|
||||
|
||||
RequestUgcManifestInfo() : WorldLUBitStream(MessageType::World::REQUEST_UGC_MANIFEST_INFO) {}
|
||||
void Serialize(RakNet::BitStream& bitStream) const override;
|
||||
bool Deserialize(RakNet::BitStream& bitStream) override;
|
||||
};
|
||||
|
||||
// Client -> server. The help screen wants its top 5 issues.
|
||||
struct UIHelpTop5 : public WorldLUBitStream {
|
||||
int32_t language{}; // 0: en_US, 1: pl_US, 2: de_DE, 3: en_GB
|
||||
|
||||
@@ -372,4 +372,26 @@ namespace UgcFormats {
|
||||
std::string ChecksumXml(std::string_view data) {
|
||||
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Checksum><MD5>" + Md5Hex(data) + "</MD5><Filesize>" + std::to_string(data.size()) + "</Filesize></Checksum>\n";
|
||||
}
|
||||
|
||||
bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size) {
|
||||
const auto between = [xml](std::string_view open, std::string_view close) -> std::string_view {
|
||||
const auto start = xml.find(open);
|
||||
if (start == std::string_view::npos) return {};
|
||||
const auto end = xml.find(close, start + open.size());
|
||||
if (end == std::string_view::npos) return {};
|
||||
return xml.substr(start + open.size(), end - start - open.size());
|
||||
};
|
||||
const auto hash = between("<MD5>", "</MD5>");
|
||||
const auto length = between("<Filesize>", "</Filesize>");
|
||||
if (hash.size() != 32 || length.empty()) return false;
|
||||
uint64_t parsed = 0;
|
||||
for (const char c : length) {
|
||||
if (c < '0' || c > '9') return false;
|
||||
parsed = parsed * 10 + static_cast<uint64_t>(c - '0');
|
||||
if (parsed > std::numeric_limits<uint32_t>::max()) return false;
|
||||
}
|
||||
md5 = hash;
|
||||
size = static_cast<uint32_t>(parsed);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
@@ -57,4 +58,7 @@ namespace UgcFormats {
|
||||
|
||||
// What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it
|
||||
std::string ChecksumXml(std::string_view data);
|
||||
|
||||
// The MD5 (lowercase hex) and size a .checksum (ChecksumXml) holds; false when it doesn't hold both
|
||||
bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@ namespace UgcJobs {
|
||||
void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) {
|
||||
files[name + ".gz"] = ZCompression::Gzip(data);
|
||||
files[name + ".checksum"] = UgcFormats::ChecksumXml(data);
|
||||
// Without 3D services the client downloads it as sd0 (after asking its world for the checksum)
|
||||
auto sd0 = Sd0::Compress(data);
|
||||
if (!sd0.empty()) files[name + ".sd0"] = std::move(sd0);
|
||||
}
|
||||
|
||||
std::string LxfmlFromBlob(const std::string& blob) {
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
#include "Logger.h"
|
||||
#include "UgcBricks.h"
|
||||
#include "UgcCdClient.h"
|
||||
#include "Sd0.h"
|
||||
#include "UgcFormats.h"
|
||||
#include "UgcKeys.h"
|
||||
#include "ZCompression.h"
|
||||
#include "UgcThrottle.h"
|
||||
#include "json.hpp"
|
||||
|
||||
@@ -26,6 +29,12 @@ namespace {
|
||||
constexpr size_t LOG_LENGTH = 50;
|
||||
constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5);
|
||||
constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
|
||||
// In the storage folder once every item stored has its sd0 icon and checksums (Backfill)
|
||||
constexpr auto BACKFILL_MARKER = ".checksums-stored";
|
||||
constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16;
|
||||
constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200;
|
||||
// Combination id of a build whose modules can't be told (so it isn't looked at again)
|
||||
constexpr LWOOBJID NO_COMBINATION = -1;
|
||||
|
||||
const char* KindName(UgcStorage::Kind kind) {
|
||||
return kind == UgcStorage::Kind::MODEL ? "model" : "modular";
|
||||
@@ -95,7 +104,12 @@ UgcProcessor::~UgcProcessor() {
|
||||
}
|
||||
|
||||
void UgcProcessor::Start() {
|
||||
for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
|
||||
const auto stored = m_Storage.List();
|
||||
for (const auto& entry : stored) m_StoredBytes += entry.bytes;
|
||||
// Items made before the checksums were stored get their sd0 icon and checksums once (Backfill)
|
||||
std::error_code error;
|
||||
m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error);
|
||||
if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end());
|
||||
m_NextEviction = std::chrono::steady_clock::now();
|
||||
m_Stopping = false;
|
||||
for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
|
||||
@@ -233,6 +247,14 @@ void UgcProcessor::Worker() {
|
||||
const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error);
|
||||
if (bytes) {
|
||||
done.bytes = *bytes;
|
||||
// The checksums of what the client downloads as sd0, for the main thread to store (the worlds answer
|
||||
// the clients' manifest requests with them)
|
||||
for (const std::string name : { "icon.dds", "model.nif" }) {
|
||||
const auto checksum = done.outcome.files.find(name + ".checksum");
|
||||
if (checksum == done.outcome.files.end() || !done.outcome.files.contains(name + ".sd0")) continue;
|
||||
Checksum parsed{ name };
|
||||
if (UgcFormats::ReadChecksumXml(checksum->second, parsed.md5, parsed.size)) done.checksums.push_back(std::move(parsed));
|
||||
}
|
||||
} else {
|
||||
done.outcome.ok = false;
|
||||
done.outcome.error = error;
|
||||
@@ -355,6 +377,8 @@ void UgcProcessor::Record(const Done& done) {
|
||||
}
|
||||
} else {
|
||||
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
|
||||
// Which combination's files it shares, for the worlds' manifest answers
|
||||
if (const auto combo = m_ComboOf.find(done.id); combo != m_ComboOf.end()) Database::Get()->SetModularBuildCombination(done.id, combo->second);
|
||||
}
|
||||
m_Recent.erase({ done.kind, done.id });
|
||||
|
||||
@@ -381,6 +405,8 @@ void UgcProcessor::Collect() {
|
||||
finished.swap(m_Done);
|
||||
}
|
||||
for (const auto& done : finished) {
|
||||
// done.id is the model, or the combination
|
||||
if (done.outcome.ok) StoreChecksums(done.kind, done.id, done.checksums);
|
||||
if (done.kind == Kind::MODEL) {
|
||||
m_InFlight.erase({ done.kind, done.id });
|
||||
if (!done.iconOnly) {
|
||||
@@ -413,8 +439,58 @@ void UgcProcessor::Collect() {
|
||||
if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
|
||||
}
|
||||
|
||||
void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums) {
|
||||
const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION;
|
||||
for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size);
|
||||
}
|
||||
|
||||
void UgcProcessor::Backfill() {
|
||||
if (!m_BackfillBuildsDone) {
|
||||
const auto builds = Database::Get()->GetModularBuildsWithoutCombination(BACKFILL_BUILDS_PER_UPDATE);
|
||||
for (const auto& build : builds) {
|
||||
const auto key = UgcModularKey::Normalize(build.modules);
|
||||
const auto combo = key.empty() ? NO_COMBINATION : UgcModularKey::StorageId(key);
|
||||
m_ComboOf[build.id] = combo;
|
||||
Database::Get()->SetModularBuildCombination(build.id, combo);
|
||||
}
|
||||
if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true;
|
||||
}
|
||||
if (m_BackfillItemsDone) return;
|
||||
const auto read = [](const std::filesystem::path& path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
return std::string(std::istreambuf_iterator<char>(in), {});
|
||||
};
|
||||
for (size_t i = 0; i < BACKFILL_ITEMS_PER_UPDATE && !m_BackfillItems.empty(); i++) {
|
||||
const auto entry = m_BackfillItems.front();
|
||||
m_BackfillItems.pop_front();
|
||||
if (m_InFlight.contains({ entry.kind, entry.id }) || (entry.kind == Kind::MODULAR && m_ComboJobs.contains(entry.id))) continue;
|
||||
// Only the icon: it is small (a model's .nif isn't asked for without 3D services and gets its sd0 when made again)
|
||||
const auto checksumFile = m_Storage.File(entry.kind, entry.id, "icon.dds.checksum");
|
||||
if (!checksumFile) continue;
|
||||
Checksum checksum{ "icon.dds" };
|
||||
if (!UgcFormats::ReadChecksumXml(read(*checksumFile), checksum.md5, checksum.size)) continue;
|
||||
if (!m_Storage.File(entry.kind, entry.id, "icon.dds.sd0")) {
|
||||
const auto packed = m_Storage.File(entry.kind, entry.id, "icon.dds.gz");
|
||||
const auto icon = packed ? ZCompression::Gunzip(read(*packed)) : std::nullopt;
|
||||
if (!icon || UgcFormats::Md5Hex(*icon) != checksum.md5) continue;
|
||||
std::string error;
|
||||
if (!m_Storage.Update(entry.kind, entry.id, { { "icon.dds.sd0", Sd0::Compress(*icon) } }, error)) {
|
||||
LOG("Couldn't write the sd0 icon of %llu: %s", static_cast<unsigned long long>(entry.id), error.c_str());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
StoreChecksums(entry.kind, entry.id, { checksum });
|
||||
}
|
||||
if (m_BackfillItems.empty()) {
|
||||
m_BackfillItemsDone = true;
|
||||
std::ofstream(m_Storage.GetRoot() / BACKFILL_MARKER) << "1\n";
|
||||
LOG("Stored the checksums of the icons made before");
|
||||
}
|
||||
}
|
||||
|
||||
void UgcProcessor::Update() {
|
||||
Collect();
|
||||
Backfill();
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage();
|
||||
if (now >= m_NextPoll) {
|
||||
|
||||
@@ -155,6 +155,13 @@ private:
|
||||
UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams)
|
||||
};
|
||||
|
||||
// A written file's MD5 and size as the client has it after inflating it (from its .checksum)
|
||||
struct Checksum {
|
||||
std::string file; // icon.dds, model.nif
|
||||
std::string md5;
|
||||
uint32_t size{};
|
||||
};
|
||||
|
||||
struct Done {
|
||||
Kind kind{};
|
||||
LWOOBJID id{};
|
||||
@@ -163,8 +170,19 @@ private:
|
||||
uint64_t bytes{};
|
||||
double milliseconds{};
|
||||
bool iconOnly{};
|
||||
std::vector<Checksum> checksums; // of the files written that the client downloads as sd0
|
||||
};
|
||||
|
||||
// Main thread: records the checksums of an item's files (ugc_file_checksums)
|
||||
void StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums);
|
||||
|
||||
// Main thread: fills in what was made before the worlds answered manifest requests, a little per Update: the
|
||||
// icons' sd0 files and checksums of the items stored, and the builds' combination ids
|
||||
void Backfill();
|
||||
std::deque<UgcStorage::Entry> m_BackfillItems;
|
||||
bool m_BackfillItemsDone{ true };
|
||||
bool m_BackfillBuildsDone{ false };
|
||||
|
||||
void Poll();
|
||||
// Main thread: the icon values for a kind and an item (the kind's preset, then the item's override)
|
||||
UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget);
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
#include "UgcCdClient.h"
|
||||
#include "UgcIconParams.h"
|
||||
#include "UgcJobs.h"
|
||||
#include "Sd0.h"
|
||||
#include "UgcFormats.h"
|
||||
#include "ZCompression.h"
|
||||
#include "UgcModel.h"
|
||||
@@ -228,6 +229,49 @@ namespace {
|
||||
// .hkx: no physics is made
|
||||
}
|
||||
|
||||
/**
|
||||
* A game client download without 3D services (UGCUSE3DSERVICES=7:0, the client's default):
|
||||
* BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><.lxfml|.nif|.hkx|.dds>.sd0 under its UGCSERVERDIR. The
|
||||
* client asked its world for the file's checksum first (UGC_MANIFEST_RESPONSE) and checks the inflated download
|
||||
* against it.
|
||||
*/
|
||||
void ServeClientSd0(HTTPReply& reply, const std::string& bucket, const std::string& name) {
|
||||
NotFound(reply);
|
||||
auto file = Lower(name);
|
||||
if (!file.ends_with(".sd0")) return;
|
||||
file.resize(file.size() - 4);
|
||||
const auto dot = file.find('.');
|
||||
if (dot == std::string::npos) return;
|
||||
const auto extension = file.substr(dot);
|
||||
const auto number = file.substr(0, dot);
|
||||
const auto id = GeneralUtils::TryParse<uint64_t>(number);
|
||||
if (!id || number.empty() || GeneralUtils::TryParse<uint64_t>(bucket) != *id % 1000) return;
|
||||
const auto blueprint = static_cast<LWOOBJID>(*id);
|
||||
if (extension == ".lxfml") {
|
||||
// Straight from the ugc row, like the 3D services download; the last few are kept
|
||||
static std::map<LWOOBJID, std::string> cache; // main thread only
|
||||
auto it = cache.find(blueprint);
|
||||
if (it == cache.end()) {
|
||||
const auto model = Database::Get()->GetUgcModel(blueprint);
|
||||
const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string();
|
||||
if (lxfml.empty()) return;
|
||||
if (cache.size() >= 64) cache.erase(cache.begin());
|
||||
it = cache.emplace(blueprint, Sd0::Compress(lxfml)).first;
|
||||
}
|
||||
reply.status = eHTTPStatusCode::OK;
|
||||
reply.contentType = eContentType::APPLICATION_OCTET_STREAM;
|
||||
reply.message = it->second;
|
||||
reply.headers.push_back("Cache-Control: public, max-age=60");
|
||||
} else if (extension == ".nif") {
|
||||
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
|
||||
} else if (extension == ".dds") {
|
||||
// A player model's icon, else a car or rocket's (its combination's)
|
||||
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
|
||||
if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
|
||||
}
|
||||
// .hkx: no physics is made, the client makes its own
|
||||
}
|
||||
|
||||
void RegisterRoutes() {
|
||||
// The configured path, and the one the 1.10.64 client uses whatever its boot.cfg says (its built-in patch server
|
||||
// folder, lwoclient/UserBrickModels; see docs/UgcServer.md)
|
||||
@@ -253,6 +297,25 @@ namespace {
|
||||
LOG("Serving the client's downloads under %s/UGCC<datacenter>/", base.c_str());
|
||||
}
|
||||
|
||||
// Without 3D services: <UGCSERVERDIR>/BrickModels/UserMade/<bucket>/<file>.sd0, where UGCSERVERDIR is client_path
|
||||
// when boot.cfg sets it, else the client's default <PATCHSERVERDIR>/UserBrickModels (any patch folder), or empty
|
||||
std::set<std::string> sd0Bases = { "/:patchdir/userbrickmodels", "" };
|
||||
{
|
||||
std::string base;
|
||||
for (const auto& segment : Segments(Lower(Game::config->GetValue("client_path").empty() ? std::string("/ugc") : Game::config->GetValue("client_path")))) base += "/" + segment;
|
||||
sd0Bases.insert(base);
|
||||
}
|
||||
for (const auto& base : sd0Bases) {
|
||||
Game::web.RegisterHTTPRoute({ .path = base + "/brickmodels/usermade/:bucket/:file", .method = eHTTPMethod::GET, .middleware = {},
|
||||
.handle = [](HTTPReply& reply, const HTTPContext& context) {
|
||||
const auto segments = Segments(context.originalPath);
|
||||
if (segments.size() < 2) return NotFound(reply);
|
||||
ServeClientSd0(reply, segments[segments.size() - 2], segments.back());
|
||||
LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast<int>(reply.status), reply.file.empty() ? "" : " (file)");
|
||||
} });
|
||||
LOG("Serving the client's sd0 downloads under %s/BrickModels/UserMade/", base.c_str());
|
||||
}
|
||||
|
||||
// Previews for the dashboard
|
||||
Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {},
|
||||
.handle = [](HTTPReply& reply, const HTTPContext& context) {
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
|
||||
namespace {
|
||||
constexpr std::array KNOWN_FILES = {
|
||||
"model.nif", "model.nif.gz", "model.nif.checksum",
|
||||
"model.nif", "model.nif.gz", "model.nif.checksum", "model.nif.sd0",
|
||||
"model.lxfml.gz", "model.lxfml.checksum",
|
||||
"icon.dds.gz", "icon.dds.checksum", "icon.png",
|
||||
"icon.dds.gz", "icon.dds.checksum", "icon.dds.sd0", "icon.png",
|
||||
"model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json",
|
||||
"previous.icon.png", "previous.model.nif.gz", "previous.stats.json",
|
||||
};
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "PropertyRent.h"
|
||||
#include "PropertyReputation.h"
|
||||
#include "LiveEvents.h"
|
||||
#include "UgcManifest.h"
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <ctime>
|
||||
@@ -474,6 +475,7 @@ int main(int argc, char** argv) {
|
||||
DashboardNotify::SendPlayerPositions(g_InstanceID);
|
||||
MessageInspector::Update();
|
||||
LiveEvents::Update();
|
||||
UgcManifest::Update();
|
||||
|
||||
Metrics::StartMeasurement(MetricVariable::UpdateSpawners);
|
||||
Game::zoneManager->Update(deltaTime);
|
||||
@@ -1104,6 +1106,7 @@ void CleanupDisconnectedUser(const SystemAddress& sysAddr) {
|
||||
// other instance has saved since
|
||||
const bool savedByMigration = WorldMigration::IsLeaving(sysAddr);
|
||||
WorldMigration::OnDisconnected(sysAddr);
|
||||
UgcManifest::OnDisconnect(sysAddr);
|
||||
|
||||
auto user = UserManager::Instance()->GetUser(sysAddr);
|
||||
if (!user) return;
|
||||
@@ -1604,6 +1607,13 @@ namespace {
|
||||
}
|
||||
};
|
||||
|
||||
struct RequestUgcManifestInfoPacket final : public WorldPackets::RequestUgcManifestInfo {
|
||||
void Handle() override {
|
||||
if (!UserManager::Instance()->GetUser(sysAddr)) return;
|
||||
UgcManifest::OnRequest(sysAddr, blueprintId, resourceType);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
std::unique_ptr<WorldPackets::WorldLUBitStream> Create() { return std::make_unique<T>(); }
|
||||
|
||||
@@ -1623,6 +1633,7 @@ namespace {
|
||||
{ MessageType::World::GENERAL_CHAT_MESSAGE, Create<GeneralChatMessagePacket> },
|
||||
{ MessageType::World::HANDLE_FUNNESS, Create<HandleFunnessPacket> },
|
||||
{ MessageType::World::UI_HELP_TOP_5, Create<UIHelpTop5Packet> },
|
||||
{ MessageType::World::REQUEST_UGC_MANIFEST_INFO, Create<RequestUgcManifestInfoPacket> },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -21,9 +21,34 @@ Resource types are 0 LXFML, 1 NIF, 2 HKX, 3 DDS.
|
||||
* the same name with `.checksum` instead of `.gz`: `<Checksum><MD5>32 hex digits</MD5><Filesize>n</Filesize></Checksum>`,
|
||||
the MD5 and size of the uncompressed file. The client checks the download against it.
|
||||
* HTTP 408 makes the client try again later; other errors are logged as failed downloads.
|
||||
* Without 3D services the client asks the world server for each file's MD5 (`REQUEST_UGC_MANIFEST_INFO`, answered
|
||||
with `UGC_MANIFEST_RESPONSE`) and downloads `BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><ext>.sd0`.
|
||||
That needs the world to answer the manifest requests, which it doesn't yet, so it is not supported.
|
||||
* `UGCUSE3DSERVICES=7:0` (the client's default, without 3D services). The client asks its world for each file's MD5
|
||||
and size (`REQUEST_UGC_MANIFEST_INFO`, answered with `UGC_MANIFEST_RESPONSE`, below) and downloads
|
||||
`UGCSERVERDIR/BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><.lxfml|.nif|.hkx|.dds>.sd0`: the file as
|
||||
an sd0 stream (the bytes `s` `d` `0` 0x01 0xff, then chunks of a u32 size and zlib data, each inflating to at most
|
||||
256 KiB). It inflates it, saves it as `res/BrickModels/UserMade/<id % 1000>/<id><ext>` and checks its MD5 against
|
||||
the world's answer; a file it has already is only downloaded again when its MD5 differs. HTTP 408 makes it try
|
||||
again later. The worlds answer from checksums the UGC server stores (`ugc_file_checksums`), and the UGC server serves
|
||||
the `.sd0` files; see "Without 3D services" below for what keeps this off by default.
|
||||
|
||||
### The manifest packets (checked in the client)
|
||||
|
||||
Both are plain packets (not game messages); offsets are after the 0x53 byte and the header (connection type, message
|
||||
id, a padding byte).
|
||||
|
||||
| Packet | Layout |
|
||||
| --- | --- |
|
||||
| `REQUEST_UGC_MANIFEST_INFO` (world message 27), client to world | u64 blueprint id, u8 resource type (16 bytes after the 0x53). Sent by `SendRequestUGCManifestInfoPacket` when the client needs a blueprint's file and 3D services are off. |
|
||||
| `UGC_MANIFEST_RESPONSE` (client message 60), world to client | u64 blueprint id, u8 resource type, then 21 bytes of manifest info: u8 valid, u32 size of the inflated file, 16 bytes MD5 of it. The client ignores an answer that isn't exactly 37 bytes after the 0x53 (`PacketHandler_MSG_CLIENT_UGC_MANIFEST_RESPONSE`). |
|
||||
|
||||
The answer is cached per blueprint and type (the same message the 3D services `.checksum` download produces). With
|
||||
it, the client uses a file it has when its MD5 matches and downloads it otherwise; with valid 0 it uses a file it has as
|
||||
it is and downloads it only when it has none. Structs: `WorldPackets::RequestUgcManifestInfo`,
|
||||
`ClientPackets::UgcManifestResponse` (`eUgcResourceType`).
|
||||
|
||||
The blueprint id of an inventory item's icon (`LWOInventoryComponent_Client::LoadBlueprintIcon`) is the item's
|
||||
`blueprintid` config when it has one (Brick-by-Brick models, LOT 6662), else its subkey (cars and rockets: their
|
||||
`ugc_modular_build` id, which DLU gives them as subkey when they are built). Cars and rockets from before builds were
|
||||
stored have subkey 0 and no build row, so the client never asks for their icons.
|
||||
|
||||
A model's render component uses the downloaded NIF when the model's spawn data has `renderUserGen=1` with its
|
||||
`blueprintid`; `NotifyClientUGCModelReady` (game message 909) makes the client fetch the blueprint's NIF and HKX
|
||||
@@ -50,8 +75,18 @@ DATACENTERID=1:150,
|
||||
then don't show. So 3D services only work when the UGC server answers on port 80 of that address; it serves
|
||||
`/lwoclient/UserBrickModels/...` as well as `client_path` for that.
|
||||
* With `UGCUSE3DSERVICES=7:0` (the client's own default) the client builds the models itself from the LXFML the world
|
||||
sends, and property models load. It also asks the world for manifests (`REQUEST_UGC_MANIFEST_INFO`), which the world
|
||||
doesn't answer; that doesn't stop the models from loading. Use this until the UGC server can be reached as above.
|
||||
sends, and property models load. Opening the backpack's models asks the world for each car's, rocket's and model's
|
||||
icon manifest. With `ugc_manifest=1` the world answers (checked: the answers are read and the client goes on to
|
||||
download each icon), and the client then downloads from the same built-in address as with 3D services, whatever its
|
||||
`boot.cfg` says (`UGCSERVERPORT=1:14108`, `UGCSERVERDIR=0:/ugc`, `PATCHSERVERDIR=0:luclient` in the client tested):
|
||||
its log shows `Download Info: Host 'http://127.0.0.1/'`, `UGC site Info: Sharing main download connection` and
|
||||
requests for `lwoclient/UserBrickModels/BrickModels/UserMade/<bucket>/<id>.dds.sd0`. In the client the download
|
||||
thread reads `PATCHSERVERIP`/`PORT`/`DIR` and `UGCSERVERIP`/`PORT`/`DIR` through the config interface
|
||||
(`LWOResMgr2Interface::DownloadThread_Run`, defaults 127.0.0.1, 80 and `lwoclient`, the UGC ones defaulting to the
|
||||
patch ones), but the values in `boot.cfg` don't reach it. A download that can't connect (HTTP status 0) counts as a
|
||||
UGC connection failure, and the client logs the player out for it ("connection failed downloading UGC assets",
|
||||
`MainThread_LogoutDueToConnectionFailures`) unless a client mod stops that. So, as with 3D services, the icons only
|
||||
arrive when the UGC server answers on port 80 of the player's own machine; see "Without 3D services".
|
||||
* A NIF the UGC server makes, put in place of one of the game's own models and spawned, renders in the client with its
|
||||
colors: the NIFs are written like the game's `res/BrickModels/ndmade` files (nif.xml's 20.3.0.9, user version 0;
|
||||
every shape with a material, alpha blending by the vertex alpha, specular off and vertex colors, in that order).
|
||||
@@ -183,11 +218,20 @@ shared files. `combo.json` in the combination's folder says its LOTs and build t
|
||||
Files live under `ugc_output_dir` (default `ugc` next to the server binaries):
|
||||
|
||||
```
|
||||
ugc/models/<id % 1000>/<id>/model.nif.gz, model.nif.checksum, icon.dds.gz, icon.dds.checksum, icon.png,
|
||||
model.noao.nif.gz, stats.json, previous.icon.png, previous.model.nif.gz, previous.stats.json
|
||||
ugc/modular/<combination id % 1000>/<combination id>/icon.dds.gz, icon.dds.checksum, icon.png, combo.json
|
||||
ugc/models/<id % 1000>/<id>/model.nif.gz, model.nif.checksum, model.nif.sd0, icon.dds.gz, icon.dds.checksum,
|
||||
icon.dds.sd0, icon.png, model.noao.nif.gz, stats.json, previous.icon.png,
|
||||
previous.model.nif.gz, previous.stats.json
|
||||
ugc/modular/<combination id % 1000>/<combination id>/icon.dds.gz, icon.dds.checksum, icon.dds.sd0, icon.png, combo.json
|
||||
ugc/.checksums-stored
|
||||
```
|
||||
|
||||
Every file the client downloads is written three ways: `.gz` and `.checksum` for 3D services and `.sd0` (dCommon's
|
||||
`Sd0::Compress`) without them. When a worker writes an item, the checksums of its `.sd0` files (MD5 and size of the
|
||||
inflated file, from the `.checksum`) go back to the main thread, which stores them in `ugc_file_checksums` for the
|
||||
worlds. Items made before this get their `icon.dds.sd0` (from `icon.dds.gz`) and their icon's checksum once, a few per
|
||||
tick on the main thread when the server starts; `.checksums-stored` marks that done. Their `.nif` gets its `.sd0` when
|
||||
the model is made again (the client doesn't ask for it without 3D services, see "Not done yet").
|
||||
|
||||
A model's files are written to a temporary folder and renamed into place, so a half written model is never served.
|
||||
Meshes are only stored compressed (a model took about 18 MB when the .nif was kept uncompressed beside its .gz, so the
|
||||
2 GB cap held about 110 models and the server kept evicting and remaking them); the dashboard's copies are inflated
|
||||
@@ -228,6 +272,39 @@ Migrations `dlu/mysql/86_ugc_debounce_icon_settings.sql` and `dlu/sqlite/69_ugc_
|
||||
`ugc.process_after` and the table `ugc_icon_settings` (`target`: `kind:<kind>`, `model:<id>` or `combo:<key>`; `params`
|
||||
JSON; `updated_at`).
|
||||
|
||||
Migrations `dlu/mysql/89_ugc_file_checksums.sql` and `dlu/sqlite/72_ugc_file_checksums.sql`: the table
|
||||
`ugc_file_checksums` (`kind` 0 a player model, 1 a combination of car or rocket modules; `storage_id` the model's ugc
|
||||
id or the combination's id; `file` `icon.dds` or `model.nif`; `md5`, 32 lowercase hex digits, and `size` of the inflated
|
||||
file), written by the UGC server whenever it writes a file, and `ugc_modular_build.combination_id`, the combination a
|
||||
build shares its files with (0 until the UGC server has seen the build: it fills them in for existing builds when it
|
||||
starts, -1 when the modules can't be told). `IUgc::GetUgcFileChecksum(blueprint, file)` looks a blueprint up as a model
|
||||
first, then as a build through its combination.
|
||||
|
||||
## Without 3D services (`UGCUSE3DSERVICES=7:0`)
|
||||
|
||||
`UgcManifest` (dGame/dUtilities) answers `REQUEST_UGC_MANIFEST_INFO` when `ugc_manifest=1` (`sharedconfig.ini`,
|
||||
default 0):
|
||||
|
||||
* The world's main thread handles the packet (`g_WorldHandlers`), looks the checksum up with one indexed query and sends
|
||||
`UGC_MANIFEST_RESPONSE` to that client in the same tick. Only DDS (`icon.dds`) and NIF (`model.nif`) are answered;
|
||||
LXFML and HKX aren't (the client builds those itself, as today).
|
||||
* A file that isn't made yet isn't answered: the request waits (at most 512, for 15 minutes) and `UgcManifest::Update`
|
||||
(world tick) looks again every 5 seconds, answering once the UGC server has stored the checksum. A model still in its
|
||||
quiet period after a save is made right away (`ExpediteUgcModel`), as when a client asks the UGC server directly.
|
||||
Waiting requests are dropped when the client disconnects.
|
||||
* No worker threads, HTTP or file reads in the world: the UGC server precomputes everything into the database.
|
||||
|
||||
The UGC server serves `BrickModels/UserMade/<bucket>/<id>.<ext>.sd0` under `client_path`, under
|
||||
`/<any folder>/UserBrickModels` (the client's default `UGCSERVERDIR`, `lwoclient/UserBrickModels` with its built-in
|
||||
patch folder) and at the root; `.dds` is a model's icon or a car or rocket build's combination icon, `.nif` a model's
|
||||
mesh, `.lxfml` the model's LXFML from the database, `.hkx` 404.
|
||||
|
||||
It is off by default because the 1.10.64 client downloads from `http://127.0.0.1:80/lwoclient/UserBrickModels/`
|
||||
whatever its `boot.cfg` says (checked in game, see "What the 1.10.64 client does in practice"), and logs the player
|
||||
out when it can't connect there. Turning it on only helps when the UGC server answers on port 80 of every player's own
|
||||
machine, which normal players don't have; until a way is found to point the client at the server, the icons don't show
|
||||
in 7:0.
|
||||
|
||||
### Waiting while the owner is still building
|
||||
|
||||
A saved model isn't made right away: `InsertNewUgcModel` stores `process_after` = now + `ugc_debounce_seconds`
|
||||
@@ -255,8 +332,9 @@ No master messages are needed for any of it.
|
||||
The main thread owns RakNet (the master link), mongoose (HTTP) and the database. `worker_threads` (default: half the
|
||||
CPUs) workers do only pure work: parse LXFML, build and write meshes and icons, compress files. They get their input
|
||||
(LXFML text, module data) from the main thread and hand results back through a queue the main thread drains, which
|
||||
then updates the database. Brick geometry and materials are loaded once and shared read-only (the cache has its own
|
||||
lock).
|
||||
then updates the database (including the files' checksums, parsed by the worker from the `.checksum` it wrote).
|
||||
Brick geometry and materials are loaded once and shared read-only (the cache has its own lock). The start-up backfill
|
||||
of old items' `.sd0` icons and checksums and of builds' combination ids runs on the main thread, a few items per tick.
|
||||
|
||||
## CPU and memory
|
||||
|
||||
@@ -297,6 +375,9 @@ files are what every player on a property sees anyway):
|
||||
the client's `UGCSERVERDIR`), and the same under `/lwoclient/UserBrickModels` (the path the client really uses, see
|
||||
above). HKX answers 404. The LXFML comes from the database. A model that exists but isn't made yet (or was evicted) is moved to
|
||||
the front of the queue and answers 408 so the client asks again.
|
||||
* `<client_path>/BrickModels/UserMade/<id % 1000>/<id, 20 digits>.<lxfml|nif|hkx|dds>.sd0`, the same under
|
||||
`/<folder>/UserBrickModels` and at the root, for clients without 3D services (see "Without 3D services"); 408 and 404
|
||||
as above.
|
||||
* `/files/model/<id>/<file>` and `/files/modular/<id>/<file>` for the dashboard (`icon.png`, `model.nif` and
|
||||
`model.noao.nif` (inflated), `stats.json`, `combo.json` and `previous.` versions), not cached by browsers.
|
||||
* `/admin/preview`, `/admin/assembly`, `/admin/regenerate-icons`, `/admin/delete` (POST, JSON) for the dashboard only:
|
||||
@@ -424,4 +505,10 @@ pages show the item's own icon.
|
||||
keeps building its own meshes; switching them to the served NIFs (and sending `NotifyClientUGCModelReady` when a
|
||||
model is made) is the next step, and needs checking in game.
|
||||
* HKX (physics) is not generated.
|
||||
* The non-3D-services download path (world manifest packets) is not answered.
|
||||
* Icons in game: both of the client's modes download from its built-in `http://127.0.0.1:80/lwoclient/...` whatever
|
||||
its `boot.cfg` says, so neither reaches the UGC server for normal players. Worlds answer the 7:0 manifest requests
|
||||
(`ugc_manifest=1`) and the UGC server serves the `.sd0` files, but it stays off until the client can be pointed at
|
||||
the server (why the client ignores its `UGCSERVER*`/`PATCHSERVER*` lines is not found yet).
|
||||
* Cars and rockets built before builds were stored (subkey 0, no `ugc_modular_build` row) never get an icon: the client
|
||||
has no blueprint id to ask for.
|
||||
* A model's `.nif` made before `.sd0` files were written has none until the model is made again.
|
||||
|
||||
19
migrations/dlu/mysql/89_ugc_file_checksums.sql
Normal file
19
migrations/dlu/mysql/89_ugc_file_checksums.sql
Normal file
@@ -0,0 +1,19 @@
|
||||
/* UGC server: ugc_file_checksums holds the MD5 (lowercase hex) and size of each file it made, as the game client has
|
||||
it after inflating the download (kind 0: a player model, storage_id its ugc id; kind 1: a combination of car or
|
||||
rocket modules, storage_id the combination's id). ugc_modular_build.combination_id is the combination a build was
|
||||
made as (0 until the UGC server has seen it). Worlds read both to answer the client's REQUEST_UGC_MANIFEST_INFO.
|
||||
See docs/UgcServer.md. */
|
||||
CREATE TABLE IF NOT EXISTS ugc_file_checksums (
|
||||
kind INT NOT NULL,
|
||||
storage_id BIGINT NOT NULL,
|
||||
file VARCHAR(32) NOT NULL,
|
||||
md5 CHAR(32) NOT NULL,
|
||||
size INT UNSIGNED NOT NULL,
|
||||
PRIMARY KEY (kind, storage_id, file)
|
||||
);
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN combination_id BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'combination_id');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
11
migrations/dlu/sqlite/72_ugc_file_checksums.sql
Normal file
11
migrations/dlu/sqlite/72_ugc_file_checksums.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
/* UGC server: the checksums of the files it made and each car or rocket build's combination. See the MySQL migration. */
|
||||
CREATE TABLE IF NOT EXISTS ugc_file_checksums (
|
||||
kind INTEGER NOT NULL,
|
||||
storage_id BIGINT NOT NULL,
|
||||
file TEXT NOT NULL,
|
||||
md5 TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
PRIMARY KEY (kind, storage_id, file)
|
||||
);
|
||||
|
||||
ALTER TABLE ugc_modular_build ADD COLUMN combination_id BIGINT NOT NULL DEFAULT 0;
|
||||
@@ -93,3 +93,9 @@ mute_auto_reject_names=1
|
||||
# again for all their waiting models), or as soon as a game client asks for it or the owner leaves the world. 0: right
|
||||
# away. See docs/UgcServer.md.
|
||||
ugc_debounce_seconds=120
|
||||
|
||||
# 0 or 1: worlds answer game clients without 3D services (UGCUSE3DSERVICES=7:0, the default) with the checksums of the
|
||||
# files the UGC server made, so they download the icons of cars, rockets and models. The 1.10.64 client then downloads
|
||||
# from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and a client that can't connect there
|
||||
# is logged out, so only set 1 when the UGC server answers there for every player. See docs/UgcServer.md.
|
||||
ugc_manifest=0
|
||||
|
||||
@@ -774,6 +774,21 @@ TEST_F(ParitySeeded, UgcModel) {
|
||||
out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").value_or(""));
|
||||
db.DeleteUgcIconSettings("combo:4713-4714-4715");
|
||||
out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value());
|
||||
// The files' checksums the worlds answer manifest requests with: a build's are its combination's
|
||||
const auto checksumJson = [](const std::optional<IUgc::FileChecksum>& checksum) {
|
||||
return checksum ? json{ checksum->md5, checksum->size } : json(nullptr);
|
||||
};
|
||||
for (const auto& build : db.GetModularBuildsWithoutCombination(10)) out.push_back(build.id);
|
||||
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "icon.dds")));
|
||||
db.SetModularBuildCombination(1152921510000500002LL, 77);
|
||||
db.SetUgcFileChecksum(IUgc::eFileOwner::COMBINATION, 77, "icon.dds", "900150983cd24fb0d6963f7d28e17f72", 3);
|
||||
db.SetUgcFileChecksum(IUgc::eFileOwner::COMBINATION, 77, "icon.dds", "0123456789abcdef0123456789abcdef", 65664);
|
||||
out.push_back(db.GetModularBuildsWithoutCombination(10).size());
|
||||
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "icon.dds")));
|
||||
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "model.nif")));
|
||||
db.SetUgcFileChecksum(IUgc::eFileOwner::MODEL, 1152921510000500009LL, "icon.dds", "11111111111111111111111111111111", 10);
|
||||
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500009LL, "icon.dds")));
|
||||
out.push_back(checksumJson(db.GetUgcFileChecksum(77, "icon.dds")));
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -717,3 +717,35 @@ TEST_F(WorldPacketsTests, SmallRequestsMatchLegacy) {
|
||||
ASSERT_TRUE(mailRead.Deserialize(mailBytes));
|
||||
EXPECT_PACKET_EQ(FromBitStream(mail.data), FromBitStream(mailRead.data));
|
||||
}
|
||||
|
||||
// The layouts the 1.10.64 client uses without 3D services (UGCUSE3DSERVICES=7:0): it sends 16 bytes after the 0x53
|
||||
// (SendRequestUGCManifestInfoPacket) and only reads an answer that is exactly 37 bytes after it
|
||||
// (PacketHandler_MSG_CLIENT_UGC_MANIFEST_RESPONSE: 21 bytes of manifest info after the blueprint and type).
|
||||
TEST_F(WorldPacketsTests, UgcManifestPacketsMatchTheClient) {
|
||||
WorldPackets::RequestUgcManifestInfo request;
|
||||
request.blueprintId = 0x0102030405060708;
|
||||
request.resourceType = eUgcResourceType::DDS;
|
||||
RakNet::BitStream requestBytes;
|
||||
request.WritePacket(requestBytes);
|
||||
EXPECT_PACKET_EQ(FromHex("53 04 00 1b 00 00 00 00 08 07 06 05 04 03 02 01 03"), FromBitStream(requestBytes));
|
||||
EXPECT_EQ(requestBytes.GetNumberOfBytesUsed() - 1, 16);
|
||||
EXPECT_EQ(RoundTrip(request).blueprintId, request.blueprintId);
|
||||
ExpectTruncatedFails(request);
|
||||
|
||||
ClientPackets::UgcManifestResponse response;
|
||||
response.blueprintId = 0x0102030405060708;
|
||||
response.resourceType = eUgcResourceType::DDS;
|
||||
response.valid = true;
|
||||
response.fileSize = 65664;
|
||||
for (size_t i = 0; i < response.md5.size(); i++) response.md5[i] = static_cast<uint8_t>(0xa0 + i);
|
||||
RakNet::BitStream responseBytes;
|
||||
response.WritePacket(responseBytes);
|
||||
EXPECT_PACKET_EQ(FromHex("53 05 00 3c 00 00 00 00 08 07 06 05 04 03 02 01 03 01 80 00 01 00 "
|
||||
"a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af"), FromBitStream(responseBytes));
|
||||
EXPECT_EQ(responseBytes.GetNumberOfBytesUsed() - 1, 37);
|
||||
const auto read = RoundTrip(response);
|
||||
EXPECT_TRUE(read.valid);
|
||||
EXPECT_EQ(read.fileSize, 65664u);
|
||||
EXPECT_EQ(read.md5, response.md5);
|
||||
ExpectTruncatedFails(response);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <glm/gtc/matrix_transform.hpp>
|
||||
@@ -22,6 +23,7 @@
|
||||
#include "UgcRender.h"
|
||||
#include "UgcStorage.h"
|
||||
#include "UgcThrottle.h"
|
||||
#include "Sd0.h"
|
||||
#include "ZCompression.h"
|
||||
#include "json.hpp"
|
||||
|
||||
@@ -269,6 +271,28 @@ TEST(UgcFormats, ImagesAndChecksums) {
|
||||
EXPECT_EQ(dds[128 + 2], 10); // stored BGRA
|
||||
EXPECT_EQ(UgcFormats::Md5Hex("abc"), "900150983cd24fb0d6963f7d28e17f72");
|
||||
EXPECT_NE(UgcFormats::ChecksumXml("abc").find("<Checksum><MD5>900150983cd24fb0d6963f7d28e17f72</MD5><Filesize>3</Filesize></Checksum>"), std::string::npos);
|
||||
std::string md5;
|
||||
uint32_t size{};
|
||||
ASSERT_TRUE(UgcFormats::ReadChecksumXml(UgcFormats::ChecksumXml("abc"), md5, size));
|
||||
EXPECT_EQ(md5, "900150983cd24fb0d6963f7d28e17f72");
|
||||
EXPECT_EQ(size, 3u);
|
||||
EXPECT_FALSE(UgcFormats::ReadChecksumXml("<Checksum><MD5>abc</MD5><Filesize>3</Filesize></Checksum>", md5, size));
|
||||
EXPECT_FALSE(UgcFormats::ReadChecksumXml("<Checksum><MD5>900150983cd24fb0d6963f7d28e17f72</MD5><Filesize>x</Filesize></Checksum>", md5, size));
|
||||
}
|
||||
|
||||
// A download is written for both of the client's modes: .gz and .checksum (3D services) and .sd0 (without), all
|
||||
// holding the same file
|
||||
TEST(UgcJobs, AddsTheDownloadForBothClientModes) {
|
||||
UgcStorage::Files files;
|
||||
UgcJobs::AddDownload(files, "icon.dds", "abc");
|
||||
EXPECT_EQ(ZCompression::Gunzip(files.at("icon.dds.gz")).value_or(""), "abc");
|
||||
std::istringstream sd0(files.at("icon.dds.sd0"));
|
||||
EXPECT_EQ(Sd0(sd0).GetAsStringUncompressed(), "abc");
|
||||
std::string md5;
|
||||
uint32_t size{};
|
||||
ASSERT_TRUE(UgcFormats::ReadChecksumXml(files.at("icon.dds.checksum"), md5, size));
|
||||
EXPECT_EQ(md5, "900150983cd24fb0d6963f7d28e17f72");
|
||||
EXPECT_EQ(size, 3u);
|
||||
}
|
||||
|
||||
TEST(UgcModular, ParsesTheCdClientData) {
|
||||
|
||||
Reference in New Issue
Block a user