Merge remote-tracking branch 'origin/main' into raw-parsing-for-scene-data

# Conflicts:
#	dNavigation/dTerrain/RawChunk.cpp
This commit is contained in:
Aaron Kimbrell
2026-06-21 01:13:34 -05:00
274 changed files with 4280 additions and 2608 deletions

View File

@@ -3,6 +3,7 @@
#include <stdexcept>
#include "Amf3.h"
#include "StringifiedEnum.h"
/**
* AMF3 Reference document https://rtmp.veriskope.com/pdf/amf3-file-format-spec.pdf
@@ -53,7 +54,7 @@ std::unique_ptr<AMFBaseValue> AMFDeserialize::Read(RakNet::BitStream& inStream)
case eAmf::VectorObject:
[[fallthrough]];
case eAmf::Dictionary:
throw marker;
throw std::invalid_argument(StringifiedEnum::ToString(marker).data());
default:
throw std::invalid_argument("Invalid AMF3 marker" + std::to_string(static_cast<int32_t>(marker)));
}
@@ -88,6 +89,11 @@ const std::string AMFDeserialize::ReadString(RakNet::BitStream& inStream) {
// Right shift by 1 bit to get index if reference or size of next string if value
length = length >> 1;
if (isReference) {
constexpr int32_t maxStringSize = 1024 * 1024;
if (length > maxStringSize) {
LOG("1MB string attempted to be allocated in AMF deserialize, possible spoof, aborting deserialize.");
throw std::invalid_argument("1MB string attempted to be allocated in AMF deserialize, possible spoof, aborting deserialize.");
}
std::string value(length, 0);
inStream.Read(&value[0], length);
// Empty strings are never sent by reference
@@ -117,6 +123,12 @@ std::unique_ptr<AMFArrayValue> AMFDeserialize::ReadAmfArray(RakNet::BitStream& i
if (key.size() == 0) break;
arrayValue->Insert(key, Read(inStream));
}
constexpr int32_t maxArraySize = 10'000;
if (sizeOfDenseArray > maxArraySize) {
LOG("Someone sent 10,000 dense array entries, probably a bad packet.");
throw std::invalid_argument("Someone sent 10,000 dense array entries, probably a bad packet.");
}
// Finally read dense portion
for (uint32_t i = 0; i < sizeOfDenseArray; i++) {
arrayValue->Insert(i, Read(inStream));

View File

@@ -369,11 +369,39 @@ public:
template<typename AmfType = AMFArrayValue>
AmfType& PushDebug(const std::string_view name) {
size_t i = 0;
for (; i < m_Dense.size(); i++) {
const auto& cast = dynamic_cast<AMFArrayValue*>(m_Dense[i].get());
if (!cast) continue;
const auto& nameValue = cast->Get<std::string>("name");
if (!nameValue || nameValue->GetValue() != name) continue;
// found a duplicate, return this instead
auto valueCast = dynamic_cast<AmfType*>(cast->Get("value"));
if (valueCast) return *valueCast;
}
auto* value = PushArray();
value->Insert("name", name.data());
value->Insert<std::string>("name", name.data());
return value->Insert<AmfType>("value", std::make_unique<AmfType>());
}
AMFArrayValue& PushDebug(const NiPoint3& point) {
PushDebug<AMFDoubleValue>("X") = point.x;
PushDebug<AMFDoubleValue>("Y") = point.y;
PushDebug<AMFDoubleValue>("Z") = point.z;
return *this;
}
AMFArrayValue& PushDebug(const NiQuaternion& rot) {
PushDebug<AMFDoubleValue>("W") = rot.w;
PushDebug<AMFDoubleValue>("X") = rot.x;
PushDebug<AMFDoubleValue>("Y") = rot.y;
PushDebug<AMFDoubleValue>("Z") = rot.z;
return *this;
}
private:
/**
* The associative portion. These values are key'd with strings to an AMFValue.

View File

@@ -52,8 +52,7 @@ uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
if (actualUncompressedSize != -1) {
uint32_t previousSize = completeUncompressedModel.size();
completeUncompressedModel.append(reinterpret_cast<char*>(uncompressedChunk.get()));
completeUncompressedModel.resize(previousSize + actualUncompressedSize);
completeUncompressedModel.append(reinterpret_cast<char*>(uncompressedChunk.get()), actualUncompressedSize);
} else {
LOG("Failed to inflate chunk %i for model %llu. Error: %i", chunkCount, model.id, err);
break;

View File

@@ -49,11 +49,10 @@ if (UNIX)
elseif (WIN32)
include(FetchContent)
# TODO Keep an eye on the zlib repository for an update to disable testing. Don't forget to update CMakePresets
FetchContent_Declare(
zlib
URL https://github.com/madler/zlib/archive/refs/tags/v1.2.11.zip
URL_HASH MD5=9d6a627693163bbbf3f26403a3a0b0b1
URL https://github.com/madler/zlib/archive/refs/tags/v1.3.2.zip
URL_HASH MD5=adbba6eef8960c3412818b2e241f46dc
GIT_PROGRESS TRUE
GIT_SHALLOW 1
)
@@ -62,12 +61,12 @@ elseif (WIN32)
set(CMAKE_POLICY_DEFAULT_CMP0048 NEW)
# Disable warning about the minimum version of cmake used for bcrypt being deprecated in the future
set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE)
# Disable zlib tests
set(ZLIB_BUILD_TESTING OFF CACHE BOOL "" FORCE)
FetchContent_MakeAvailable(zlib)
set(ZLIB_INCLUDE_DIRS ${zlib_SOURCE_DIR} ${zlib_BINARY_DIR})
set_target_properties(zlib PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_INCLUDE_DIRS}")
add_library(ZLIB::ZLIB ALIAS zlib)
else ()
message(
FATAL_ERROR

View File

@@ -308,8 +308,9 @@ std::vector<std::string> GeneralUtils::GetSqlFileNamesFromFolder(const std::stri
for (const auto& t : std::filesystem::directory_iterator(folder)) {
if (t.is_directory() || t.is_symlink()) continue;
auto filename = t.path().filename().string();
const auto index = std::stoi(GeneralUtils::SplitString(filename, '_').at(0));
filenames.emplace(index, std::move(filename));
// Ensure the file has a name in the format of xxxxxxxx_anything_goes_here.sql
const auto migrationNumber = TryParse<uint32_t>(GeneralUtils::SplitString(filename, '_').at(0));
if (migrationNumber.has_value()) filenames.emplace(migrationNumber.value(), std::move(filename));
}
// Now sort the map by the oldest migration.

View File

@@ -205,6 +205,12 @@ namespace GeneralUtils {
return isParsed ? static_cast<T>(result) : std::optional<T>{};
}
// A version of TryParse that will return `errorVal` if `str` failed to parse.
template <Numeric T>
[[nodiscard]] T TryParse(std::string_view str, const T errorVal) {
return TryParse<T>(str).value_or(errorVal);
}
template<typename T>
requires(!Numeric<T>)
[[nodiscard]] std::optional<T> TryParse(std::string_view str);
@@ -237,6 +243,12 @@ namespace GeneralUtils {
return std::nullopt;
}
// A version of TryParse that will return `errorVal` if `str` failed to parse.
template <std::floating_point T>
[[nodiscard]] T TryParse(std::string_view str, const T errorVal) {
return TryParse<T>(str).value_or(errorVal);
}
#endif
/**
@@ -258,6 +270,11 @@ namespace GeneralUtils {
return z ? std::make_optional<T>(x.value(), y.value(), z.value()) : std::nullopt;
}
// Alternative overload of TryParse with a default value
[[nodiscard]] inline NiPoint3 TryParse(const std::string_view strX, const std::string_view strY, const std::string_view strZ, const NiPoint3 errorVal) {
return TryParse<NiPoint3>(strX, strY, strZ).value_or(errorVal);
}
/**
* The TryParse overload for handling NiPoint3 by passing a span of three strings
* @param str The string vector representing the X, Y, and Z coordinates
@@ -268,6 +285,11 @@ namespace GeneralUtils {
return (str.size() == 3) ? TryParse<T>(str[0], str[1], str[2]) : std::nullopt;
}
// Alternative overload of TryParse with a default value
[[nodiscard]] inline NiPoint3 TryParse(const std::span<const std::string> str, const NiPoint3 errorVal) {
return TryParse<NiPoint3>(str).value_or(errorVal);
}
template <typename T>
std::u16string to_u16string(const T value) {
return GeneralUtils::ASCIIToUTF16(std::to_string(value));

View File

@@ -10,163 +10,151 @@
#include <string_view>
#include <vector>
using LDFKey = std::string_view;
using LDFTypeAndValue = std::string_view;
using LDFType = std::string_view;
using LDFValue = std::string_view;
//! Returns a pointer to a LDFData value based on string format
LDFBaseData* LDFBaseData::DataFromString(const std::string_view& format) {
std::unique_ptr<LDFBaseData> LDFBaseData::DataFromString(const std::string_view& format) {
std::unique_ptr<LDFBaseData> toReturn;
// A valid LDF must be at least 3 characters long (=0:) is the shortest valid LDF (empty UTF-16 key with no initial value)
if (format.empty() || format.length() <= 2) return nullptr;
auto equalsPosition = format.find('=');
// You can have an empty key, just make sure the type and value might exist
if (equalsPosition == std::string::npos || equalsPosition == (format.size() - 1)) return nullptr;
if (!format.empty() && format.length() > 2) {
auto equalsPosition = format.find('=');
// You can have an empty key, just make sure the type and value might exist
if (equalsPosition != std::string::npos && equalsPosition != (format.size() - 1)) {
std::pair<LDFKey, LDFTypeAndValue> keyValue;
keyValue.first = format.substr(0, equalsPosition);
keyValue.second = format.substr(equalsPosition + 1, format.size());
const std::string_view keyValue = format.substr(0, equalsPosition);
const std::string_view typeAndValue = format.substr(equalsPosition + 1, format.size());
std::u16string key = GeneralUtils::ASCIIToUTF16(keyValue.first);
const auto key = GeneralUtils::ASCIIToUTF16(keyValue);
auto colonPosition = keyValue.second.find(':');
const auto colonPosition = typeAndValue.find(':');
// If : is the first thing after an =, then this is an invalid LDF since
// we dont have a type to use.
if (colonPosition == std::string::npos || colonPosition == 0) return nullptr;
// If : is the first thing after an =, then this is an invalid LDF since
// we dont have a type to use.
if (colonPosition != std::string::npos && colonPosition != 0) {
const std::string_view ldfType = typeAndValue.substr(0, colonPosition);
const std::string_view ldfValue = typeAndValue.substr(colonPosition + 1, typeAndValue.size());
std::pair<LDFType, LDFValue> ldfTypeAndValue;
ldfTypeAndValue.first = keyValue.second.substr(0, colonPosition);
ldfTypeAndValue.second = keyValue.second.substr(colonPosition + 1, keyValue.second.size());
// Only allow empty values for string values.
if (!ldfValue.empty() || (ldfType == "0" /* UTF-16 */ || ldfType == "13" /* UTF-8 */)) {
const eLDFType type = GeneralUtils::TryParse<eLDFType>(ldfType, LDF_TYPE_UNKNOWN);
switch (type) {
case LDF_TYPE_UTF_16: {
std::u16string data = GeneralUtils::UTF8ToUTF16(ldfValue);
toReturn.reset(new LDFData<std::u16string>(key, data));
break;
}
// Only allow empty values for string values.
if (ldfTypeAndValue.second.size() == 0 && !(ldfTypeAndValue.first == "0" || ldfTypeAndValue.first == "13")) return nullptr;
case LDF_TYPE_S32: {
const auto data = GeneralUtils::TryParse<int32_t>(ldfValue);
if (data) {
toReturn.reset(new LDFData<int32_t>(key, data.value()));
} else {
LOG("Warning: Attempted to process invalid int32 value (%s) from string (%s)", ldfValue.data(), format.data());
}
eLDFType type;
char* storage;
try {
type = static_cast<eLDFType>(strtol(ldfTypeAndValue.first.data(), &storage, 10));
} catch (std::exception) {
LOG("Attempted to process invalid ldf type (%s) from string (%s)", ldfTypeAndValue.first.data(), format.data());
return nullptr;
}
break;
}
LDFBaseData* returnValue = nullptr;
switch (type) {
case LDF_TYPE_UTF_16: {
std::u16string data = GeneralUtils::UTF8ToUTF16(ldfTypeAndValue.second);
returnValue = new LDFData<std::u16string>(key, data);
break;
}
case LDF_TYPE_FLOAT: {
const auto data = GeneralUtils::TryParse<float>(ldfValue);
if (data) {
toReturn.reset(new LDFData<float>(key, data.value()));
} else {
LOG("Warning: Attempted to process invalid float value (%s) from string (%s)", ldfValue.data(), format.data());
}
break;
}
case LDF_TYPE_S32: {
const auto data = GeneralUtils::TryParse<int32_t>(ldfTypeAndValue.second);
if (!data) {
LOG("Warning: Attempted to process invalid int32 value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
}
returnValue = new LDFData<int32_t>(key, data.value());
case LDF_TYPE_DOUBLE: {
const auto data = GeneralUtils::TryParse<double>(ldfValue);
if (data) {
toReturn.reset(new LDFData<double>(key, data.value()));
} else {
LOG("Warning: Attempted to process invalid double value (%s) from string (%s)", ldfValue.data(), format.data());
}
break;
}
break;
}
case LDF_TYPE_U32:
{
uint32_t data;
bool parsed = true;
// Have to do this really weird parsing to allow for copy ellision
if (ldfValue == "true") {
data = 1;
} else if (ldfValue == "false") {
data = 0;
} else {
const auto dataOptional = GeneralUtils::TryParse<uint32_t>(ldfValue);
if (!dataOptional) {
LOG("Warning: Attempted to process invalid uint32 value (%s) from string (%s)", ldfValue.data(), format.data());
parsed = false;
} else {
data = dataOptional.value();
}
}
case LDF_TYPE_FLOAT: {
const auto data = GeneralUtils::TryParse<float>(ldfTypeAndValue.second);
if (!data) {
LOG("Warning: Attempted to process invalid float value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
}
returnValue = new LDFData<float>(key, data.value());
break;
}
if (parsed) toReturn.reset(new LDFData<uint32_t>(key, data));
break;
}
case LDF_TYPE_DOUBLE: {
const auto data = GeneralUtils::TryParse<double>(ldfTypeAndValue.second);
if (!data) {
LOG("Warning: Attempted to process invalid double value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
}
returnValue = new LDFData<double>(key, data.value());
break;
}
case LDF_TYPE_BOOLEAN: {
bool data;
bool parsed = true;
// Have to do this really weird parsing to allow for copy ellision
if (ldfValue == "true") {
data = true;
} else if (ldfValue == "false") {
data = false;
} else {
const auto dataOptional = GeneralUtils::TryParse<bool>(ldfValue);
if (!dataOptional) {
LOG("Warning: Attempted to process invalid bool value (%s) from string (%s)", ldfValue.data(), format.data());
parsed = false;
} else {
data = dataOptional.value();
}
}
case LDF_TYPE_U32:
{
uint32_t data;
if (parsed) toReturn.reset(new LDFData<bool>(key, data));
break;
}
if (ldfTypeAndValue.second == "true") {
data = 1;
} else if (ldfTypeAndValue.second == "false") {
data = 0;
} else {
const auto dataOptional = GeneralUtils::TryParse<uint32_t>(ldfTypeAndValue.second);
if (!dataOptional) {
LOG("Warning: Attempted to process invalid uint32 value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
case LDF_TYPE_U64: {
const auto data = GeneralUtils::TryParse<uint64_t>(ldfValue);
if (data) {
toReturn.reset(new LDFData<uint64_t>(key, data.value()));
} else {
LOG("Warning: Attempted to process invalid uint64 value (%s) from string (%s)", ldfValue.data(), format.data());
}
break;
}
case LDF_TYPE_OBJID: {
const auto data = GeneralUtils::TryParse<LWOOBJID>(ldfValue);
if (data) {
toReturn.reset(new LDFData<LWOOBJID>(key, data.value()));
} else {
LOG("Warning: Attempted to process invalid LWOOBJID value (%s) from string (%s)", ldfValue.data(), format.data());
}
break;
}
case LDF_TYPE_UTF_8: {
toReturn.reset(new LDFData<std::string>(key, ldfValue.data()));
break;
}
case LDF_TYPE_UNKNOWN:
[[fallthrough]];
default: {
LOG("Warning: Attempted to process invalid unknown value (%s) from string (%s)", ldfValue.data(), format.data());
break;
}
}
}
}
data = dataOptional.value();
}
returnValue = new LDFData<uint32_t>(key, data);
break;
}
case LDF_TYPE_BOOLEAN: {
bool data;
if (ldfTypeAndValue.second == "true") {
data = true;
} else if (ldfTypeAndValue.second == "false") {
data = false;
} else {
const auto dataOptional = GeneralUtils::TryParse<bool>(ldfTypeAndValue.second);
if (!dataOptional) {
LOG("Warning: Attempted to process invalid bool value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
}
data = dataOptional.value();
}
returnValue = new LDFData<bool>(key, data);
break;
}
case LDF_TYPE_U64: {
const auto data = GeneralUtils::TryParse<uint64_t>(ldfTypeAndValue.second);
if (!data) {
LOG("Warning: Attempted to process invalid uint64 value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
}
returnValue = new LDFData<uint64_t>(key, data.value());
break;
}
case LDF_TYPE_OBJID: {
const auto data = GeneralUtils::TryParse<LWOOBJID>(ldfTypeAndValue.second);
if (!data) {
LOG("Warning: Attempted to process invalid LWOOBJID value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
return nullptr;
}
returnValue = new LDFData<LWOOBJID>(key, data.value());
break;
}
case LDF_TYPE_UTF_8: {
std::string data = ldfTypeAndValue.second.data();
returnValue = new LDFData<std::string>(key, data);
break;
}
case LDF_TYPE_UNKNOWN: {
LOG("Warning: Attempted to process invalid unknown value (%s) from string (%s)", ldfTypeAndValue.second.data(), format.data());
break;
}
default: {
LOG("Warning: Attempted to process invalid LDF type (%d) from string (%s)", type, format.data());
break;
}
}
return returnValue;
return toReturn;
}

View File

@@ -1,11 +1,12 @@
#ifndef __LDFFORMAT__H__
#define __LDFFORMAT__H__
#ifndef LDFFORMAT_H
#define LDFFORMAT_H
// Custom Classes
#include "dCommonVars.h"
#include "GeneralUtils.h"
// C++
#include <map>
#include <string>
#include <string_view>
#include <sstream>
@@ -46,17 +47,17 @@ public:
virtual std::string GetValueAsString() const = 0;
virtual LDFBaseData* Copy() const = 0;
virtual std::unique_ptr<LDFBaseData> Copy() const = 0;
/**
* Given an input string, return the data as a LDF key.
*/
static LDFBaseData* DataFromString(const std::string_view& format);
static std::unique_ptr<LDFBaseData> DataFromString(const std::string_view& format);
};
template<typename T>
class LDFData: public LDFBaseData {
class LDFData : public LDFBaseData {
private:
std::u16string key;
T value;
@@ -164,8 +165,8 @@ public:
return this->GetValueString();
}
LDFBaseData* Copy() const override {
return new LDFData<T>(key, value);
std::unique_ptr<LDFBaseData> Copy() const override {
return std::make_unique<LDFData<T>>(key, value);
}
inline static const T Default = {};
@@ -226,4 +227,89 @@ template<> inline std::string LDFData<LWOOBJID>::GetValueString() const { return
template<> inline std::string LDFData<std::string>::GetValueString() const { return this->value; }
#endif //!__LDFFORMAT__H__
struct LwoNameValue {
using LDFPtr = std::unique_ptr<LDFBaseData>;
using ValueType = std::map<std::u16string, LDFPtr>;
LwoNameValue& operator=(const LwoNameValue& other) {
this->values = other.Copy();
return *this;
}
template<typename T>
void Insert(const std::u16string& key, const T& value) {
this->values.insert_or_assign(key, std::unique_ptr(std::make_unique<LDFData<T>>(key, value)));
}
void Insert(const std::u16string& key, const char* value) {
this->Insert<std::string>(key, value);
}
void Insert(const std::u16string& key, const char16_t* value) {
this->Insert<std::u16string>(key, value);
}
template<typename T>
void Insert(const std::string& key, const T& value) {
this->Insert<T>(GeneralUtils::UTF8ToUTF16(key), value);
}
void Insert(const std::string& key, const char* value) {
this->Insert<std::string>(GeneralUtils::UTF8ToUTF16(key), value);
}
void Insert(const std::string& key, const char16_t* value) {
this->Insert<std::u16string>(GeneralUtils::UTF8ToUTF16(key), value);
}
const LDFPtr& ParseInsert(const std::string& data) {
LDFPtr toInsert(LDFBaseData::DataFromString(data));
return toInsert ?
this->values.insert_or_assign(toInsert->GetKey(), std::move(toInsert)).first->second :
this->values.insert_or_assign(u"FAILED_TO_PARSE_" + GeneralUtils::UTF8ToUTF16(data), std::make_unique<LDFData<std::string>>("", "")).first->second;
}
const LDFPtr& ParseInsert(const std::u16string& data) {
return this->ParseInsert(GeneralUtils::UTF16ToWTF8(data));
}
ValueType::const_iterator begin() const {
return this->values.cbegin();
}
ValueType::const_iterator end() const {
return this->values.cend();
}
void Erase(const std::u16string& key) {
this->values.erase(key);
}
void Erase(const std::string& key) {
this->Erase(GeneralUtils::ASCIIToUTF16(key));
}
ValueType::iterator find(const ValueType::key_type& key) {
return this->values.find(key);
}
ValueType::const_iterator find(const ValueType::key_type& key) const {
return this->values.find(key);
}
LwoNameValue() = default;
LwoNameValue(const LwoNameValue& other) {
this->values = other.Copy();
}
ValueType values;
private:
ValueType Copy() const {
ValueType copy;
for (const auto& [key, value] : this->values) copy.insert_or_assign(key, value->Copy());
return copy;
}
};
#endif //!LDFFORMAT_H

View File

@@ -96,3 +96,17 @@ bool Logger::GetLogToConsole() const {
}
return toReturn;
}
FuncEntry::FuncEntry(const char* funcName, const char* fileName, const uint32_t line) {
m_FuncName = funcName;
if (!m_FuncName) m_FuncName = "Unknown";
m_Line = line;
m_FileName = fileName;
LOG("--> %s::%s:%i", m_FileName, m_FuncName, m_Line);
}
FuncEntry::~FuncEntry() {
if (!m_FuncName || !m_FileName) return;
LOG("<-- %s::%s:%i", m_FileName, m_FuncName, m_Line);
}

View File

@@ -32,6 +32,19 @@ constexpr const char* GetFileNameFromAbsolutePath(const char* path) {
#define LOG(message, ...) do { auto str_ = FILENAME_AND_LINE; Game::logger->Log(str_, message, ##__VA_ARGS__); } while(0)
#define LOG_DEBUG(message, ...) do { auto str_ = FILENAME_AND_LINE; Game::logger->LogDebug(str_, message, ##__VA_ARGS__); } while(0)
// Place this right at the start of a function. Will log a message when called and then once you leave the function.
#define LOG_ENTRY auto str_ = GetFileNameFromAbsolutePath(__FILE__); FuncEntry funcEntry_(__FUNCTION__, str_, __LINE__)
class FuncEntry {
public:
FuncEntry(const char* funcName, const char* fileName, const uint32_t line);
~FuncEntry();
private:
const char* m_FuncName = nullptr;
const char* m_FileName = nullptr;
uint32_t m_Line = 0;
};
// Writer class for writing data to files.
class Writer {
public:

View File

@@ -53,16 +53,21 @@ Lxfml::Result Lxfml::NormalizePositionOnlyFirstPart(const std::string_view data)
continue;
}
auto x = GeneralUtils::TryParse<float>(split[9]).value();
auto y = GeneralUtils::TryParse<float>(split[10]).value();
auto z = GeneralUtils::TryParse<float>(split[11]).value();
if (x < lowest.x) lowest.x = x;
if (y < lowest.y) lowest.y = y;
if (z < lowest.z) lowest.z = z;
try {
auto x = GeneralUtils::TryParse<float>(split[9]).value();
auto y = GeneralUtils::TryParse<float>(split[10]).value();
auto z = GeneralUtils::TryParse<float>(split[11]).value();
if (x < lowest.x) lowest.x = x;
if (y < lowest.y) lowest.y = y;
if (z < lowest.z) lowest.z = z;
if (highest.x < x) highest.x = x;
if (highest.y < y) highest.y = y;
if (highest.z < z) highest.z = z;
if (highest.x < x) highest.x = x;
if (highest.y < y) highest.y = y;
if (highest.z < z) highest.z = z;
} catch (std::exception& e) {
LOG("Failed to parse a split value of either (%s), (%s), or (%s).", split[9].c_str(), split[10].c_str(), split[11].c_str());
return toReturn; // Early return since we failed to parse this lxfml.
}
}
auto delta = (highest - lowest) / 2.0f;

View File

@@ -1,105 +1,77 @@
#include "Metrics.hpp"
#include "Metrics.h"
#include "StringifiedEnum.h"
#include <chrono>
std::unordered_map<MetricVariable, Metric*> Metrics::m_Metrics = {};
std::vector<MetricVariable> Metrics::m_Variables = {
MetricVariable::GameLoop,
MetricVariable::PacketHandling,
MetricVariable::UpdateEntities,
MetricVariable::UpdateSpawners,
MetricVariable::Physics,
MetricVariable::UpdateReplica,
MetricVariable::Ghosting,
MetricVariable::CPUTime,
MetricVariable::Sleep,
MetricVariable::Frame,
};
namespace {
std::unordered_map<MetricVariable, Metric> g_Metrics = {};
std::vector<MetricVariable> g_Variables = {
MetricVariable::GameLoop,
MetricVariable::PacketHandling,
MetricVariable::UpdateEntities,
MetricVariable::UpdateSpawners,
MetricVariable::Physics,
MetricVariable::UpdateReplica,
MetricVariable::Ghosting,
MetricVariable::CPUTime,
MetricVariable::Sleep,
MetricVariable::Frame,
};
}
void Metrics::AddMeasurement(MetricVariable variable, int64_t value) {
const auto& iter = m_Metrics.find(variable);
Metric* metric;
if (iter == m_Metrics.end()) {
metric = new Metric();
m_Metrics[variable] = metric;
} else {
metric = iter->second;
}
auto& metric = g_Metrics[variable];
AddMeasurement(metric, value);
}
void Metrics::AddMeasurement(Metric* metric, int64_t value) {
const auto index = metric->measurementIndex;
void Metrics::AddMeasurement(Metric& metric, int64_t value) {
const auto index = metric.measurementIndex;
metric->measurements[index] = value;
metric.measurements[index] = value;
if (metric->max == -1 || value > metric->max) {
metric->max = value;
} else if (metric->min == -1 || metric->min > value) {
metric->min = value;
if (metric.max == -1 || value > metric.max) {
metric.max = value;
} else if (metric.min == -1 || metric.min > value) {
metric.min = value;
}
if (metric->measurementSize < MAX_MEASURMENT_POINTS) {
metric->measurementSize++;
if (metric.measurementSize < MAX_MEASURMENT_POINTS) {
metric.measurementSize++;
}
metric->measurementIndex = (index + 1) % MAX_MEASURMENT_POINTS;
metric.measurementIndex = (index + 1) % MAX_MEASURMENT_POINTS;
}
const Metric* Metrics::GetMetric(MetricVariable variable) {
const auto& iter = m_Metrics.find(variable);
if (iter == m_Metrics.end()) {
return nullptr;
}
Metric* metric = iter->second;
const Metric& Metrics::GetMetric(MetricVariable variable) {
auto& metric = g_Metrics[variable];
int64_t average = 0;
for (size_t i = 0; i < metric->measurementSize; i++) {
average += metric->measurements[i];
for (size_t i = 0; i < metric.measurementSize; i++) {
average += metric.measurements[i];
}
average /= metric->measurementSize;
average /= metric.measurementSize;
metric->average = average;
metric.average = average;
return metric;
}
void Metrics::StartMeasurement(MetricVariable variable) {
const auto& iter = m_Metrics.find(variable);
auto& metric = g_Metrics[variable];
Metric* metric;
if (iter == m_Metrics.end()) {
metric = new Metric();
m_Metrics[variable] = metric;
} else {
metric = iter->second;
}
metric->activeMeasurement = std::chrono::high_resolution_clock::now();
metric.activeMeasurement = std::chrono::high_resolution_clock::now();
}
void Metrics::EndMeasurement(MetricVariable variable) {
const auto end = std::chrono::high_resolution_clock::now();
const auto& iter = m_Metrics.find(variable);
auto& metric = g_Metrics[variable];
if (iter == m_Metrics.end()) {
return;
}
Metric* metric = iter->second;
const auto elapsed = end - metric->activeMeasurement;
const auto elapsed = end - metric.activeMeasurement;
const auto nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(elapsed).count();
@@ -110,44 +82,12 @@ float Metrics::ToMiliseconds(int64_t nanoseconds) {
return static_cast<float>(nanoseconds) / 1e6;
}
std::string Metrics::MetricVariableToString(MetricVariable variable) {
switch (variable) {
case MetricVariable::GameLoop:
return "GameLoop";
case MetricVariable::PacketHandling:
return "PacketHandling";
case MetricVariable::UpdateEntities:
return "UpdateEntities";
case MetricVariable::UpdateSpawners:
return "UpdateSpawners";
case MetricVariable::Physics:
return "Physics";
case MetricVariable::UpdateReplica:
return "UpdateReplica";
case MetricVariable::Sleep:
return "Sleep";
case MetricVariable::CPUTime:
return "CPUTime";
case MetricVariable::Frame:
return "Frame";
case MetricVariable::Ghosting:
return "Ghosting";
default:
return "Invalid";
}
const std::string_view Metrics::MetricVariableToString(MetricVariable variable) {
return StringifiedEnum::ToString(variable);
}
const std::vector<MetricVariable>& Metrics::GetAllMetrics() {
return m_Variables;
}
void Metrics::Clear() {
for (const auto& pair : m_Metrics) {
delete pair.second;
}
m_Metrics.clear();
return g_Variables;
}
/* RSS Memory utilities

48
dCommon/Metrics.h Normal file
View File

@@ -0,0 +1,48 @@
#pragma once
#include "dCommonVars.h"
#include <vector>
#include <map>
#include <string_view>
#include <unordered_map>
#include <chrono>
#define MAX_MEASURMENT_POINTS 1024
enum class MetricVariable : int32_t {
GameLoop,
PacketHandling,
UpdateEntities,
UpdateSpawners,
Physics,
UpdateReplica,
Ghosting,
CPUTime,
Sleep,
Frame,
};
struct Metric {
int64_t measurements[MAX_MEASURMENT_POINTS] = {};
size_t measurementIndex = 0;
size_t measurementSize = 0;
int64_t max = -1;
int64_t min = -1;
int64_t average = 0;
std::chrono::time_point<std::chrono::high_resolution_clock> activeMeasurement;
};
namespace Metrics {
void AddMeasurement(MetricVariable variable, int64_t value);
void AddMeasurement(Metric& metric, int64_t value);
const Metric& GetMetric(MetricVariable variable);
void StartMeasurement(MetricVariable variable);
void EndMeasurement(MetricVariable variable);
float ToMiliseconds(int64_t nanoseconds);
const std::string_view MetricVariableToString(MetricVariable variable);
const std::vector<MetricVariable>& GetAllMetrics();
size_t GetPeakRSS();
size_t GetCurrentRSS();
size_t GetProcessID();
};

View File

@@ -1,61 +0,0 @@
#pragma once
#include "dCommonVars.h"
#include <vector>
#include <map>
#include <unordered_map>
#include <chrono>
#define MAX_MEASURMENT_POINTS 1024
enum class MetricVariable : int32_t
{
GameLoop,
PacketHandling,
UpdateEntities,
UpdateSpawners,
Physics,
UpdateReplica,
Ghosting,
CPUTime,
Sleep,
Frame,
};
struct Metric
{
int64_t measurements[MAX_MEASURMENT_POINTS] = {};
size_t measurementIndex = 0;
size_t measurementSize = 0;
int64_t max = -1;
int64_t min = -1;
int64_t average = 0;
std::chrono::time_point<std::chrono::high_resolution_clock> activeMeasurement;
};
class Metrics
{
public:
~Metrics();
static void AddMeasurement(MetricVariable variable, int64_t value);
static void AddMeasurement(Metric* metric, int64_t value);
static const Metric* GetMetric(MetricVariable variable);
static void StartMeasurement(MetricVariable variable);
static void EndMeasurement(MetricVariable variable);
static float ToMiliseconds(int64_t nanoseconds);
static std::string MetricVariableToString(MetricVariable variable);
static const std::vector<MetricVariable>& GetAllMetrics();
static size_t GetPeakRSS();
static size_t GetCurrentRSS();
static size_t GetProcessID();
static void Clear();
private:
Metrics();
static std::unordered_map<MetricVariable, Metric*> m_Metrics;
static std::vector<MetricVariable> m_Variables;
};

View File

@@ -1,6 +1,8 @@
#ifndef __NIPOINT3_H__
#define __NIPOINT3_H__
#ifndef GLM_ENABLE_EXPERIMENTAL
# define GLM_ENABLE_EXPERIMENTAL
#endif
/*!
\file NiPoint3.hpp
\brief Defines a point in space in XYZ coordinates

View File

@@ -1,6 +1,8 @@
#ifndef NIQUATERNION_H
#define NIQUATERNION_H
#ifndef GLM_ENABLE_EXPERIMENTAL
# define GLM_ENABLE_EXPERIMENTAL
#endif
// Custom Classes
#include "NiPoint3.h"

View File

@@ -45,6 +45,12 @@ Sd0::Sd0(std::istream& buffer) {
uint32_t bufferSize = buffer.tellg();
buffer.seekg(0, std::ios::beg);
WriteSize(firstChunk, bufferSize);
// its expected that if we got here, we got an old sd0 buffer where we ignored the sd0 part
// that means this can be at most the compressed chunk limit.
if (bufferSize > MAX_UNCOMPRESSED_CHUNK_SIZE) {
LOG("Possible bad chunk size of %i specified, rejecting.", bufferSize);
return;
}
firstChunk.resize(firstChunk.size() + bufferSize);
auto* dataStart = reinterpret_cast<char*>(firstChunk.data() + GetDataOffset(true));
if (!buffer.read(dataStart, bufferSize)) {
@@ -71,6 +77,12 @@ Sd0::Sd0(std::istream& buffer) {
WriteSize(chunk, chunkSize);
// Assuming a good buffer that is large enough to take up 2 zlib buffers
// any buffer should be compressed enough to take up less size than its uncompressed counterpart
if (chunkSize > MAX_UNCOMPRESSED_CHUNK_SIZE) {
LOG("Possible bad chunk size of %i specified, rejecting.", chunkSize);
break;
}
chunk.resize(chunkSize + dataOffset);
auto* dataStart = reinterpret_cast<char*>(chunk.data() + dataOffset);
if (!buffer.read(dataStart, chunkSize)) {
@@ -95,6 +107,11 @@ void Sd0::FromData(const uint8_t* data, size_t bufferSize) {
startOffset, numToCopy,
compressedChunk.data(), compressedChunk.size());
if (compressedSize == -1) {
LOG("Failed to compress chunk, aborting");
break;
}
auto& chunk = m_Chunks.emplace_back();
bool firstBuffer = m_Chunks.size() == 1;
auto dataOffset = GetDataOffset(firstBuffer);
@@ -119,6 +136,12 @@ std::string Sd0::GetAsStringUncompressed() const {
auto dataOffset = GetDataOffset(first);
first = false;
const auto chunkSize = chunk.size();
if (chunkSize <= static_cast<size_t>(dataOffset)) {
LOG("Bad chunkSize for data, aborting");
toReturn = "";
totalSize = 0;
break;
}
auto oldSize = toReturn.size();
toReturn.resize(oldSize + MAX_UNCOMPRESSED_CHUNK_SIZE);
@@ -128,6 +151,13 @@ std::string Sd0::GetAsStringUncompressed() const {
reinterpret_cast<uint8_t*>(toReturn.data()) + oldSize, MAX_UNCOMPRESSED_CHUNK_SIZE,
error);
if (uncompressedSize == -1) {
LOG("Failed to decompress chunk, aborting");
toReturn = "";
totalSize = 0;
break;
}
totalSize += uncompressedSize;
}

View File

@@ -3,12 +3,12 @@
#include "zlib.h"
namespace ZCompression {
int32_t GetMaxCompressedLength(int32_t nLenSrc) {
int32_t n16kBlocks = (nLenSrc + 16383) / 16384; // round up any fraction of a block
uint32_t GetMaxCompressedLength(uint32_t nLenSrc) {
uint32_t n16kBlocks = (nLenSrc + 16383) / 16384; // round up any fraction of a block
return (nLenSrc + 6 + (n16kBlocks * 5));
}
int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst) {
int32_t Compress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst) {
z_stream zInfo = { 0 };
zInfo.total_in = zInfo.avail_in = nLenSrc;
zInfo.total_out = zInfo.avail_out = nLenDst;
@@ -27,7 +27,7 @@ namespace ZCompression {
return(nRet);
}
int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr) {
int32_t Decompress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst, int32_t& nErr) {
// Get the size of the decompressed data
z_stream zInfo = { 0 };
zInfo.total_in = zInfo.avail_in = nLenSrc;

View File

@@ -3,10 +3,10 @@
#include <cstdint>
namespace ZCompression {
int32_t GetMaxCompressedLength(int32_t nLenSrc);
uint32_t GetMaxCompressedLength(uint32_t nLenSrc);
int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst);
int32_t Compress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst);
int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr);
int32_t Decompress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst, int32_t& nErr);
}

View File

@@ -7,7 +7,7 @@
#include "zlib.h"
constexpr uint32_t CRC32_INIT = 0xFFFFFFFF;
constexpr auto NULL_TERMINATOR = std::string_view{"\0\0\0", 4};
constexpr auto NULL_TERMINATOR = std::string_view{ "\0\0\0", 4 };
AssetManager::AssetManager(const std::filesystem::path& path) {
if (!std::filesystem::is_directory(path)) {
@@ -25,7 +25,7 @@ AssetManager::AssetManager(const std::filesystem::path& path) {
if (!std::filesystem::exists(m_Path / ".." / "versions")) {
throw std::runtime_error("No \"versions\" directory found in the parent directories of \"res\" - packed asset bundle cannot be loaded.");
}
m_AssetBundleType = eAssetBundleType::Packed;
m_RootPath = (m_Path / "..");
@@ -34,7 +34,7 @@ AssetManager::AssetManager(const std::filesystem::path& path) {
if (!std::filesystem::exists(m_Path / ".." / ".." / "versions")) {
throw std::runtime_error("No \"versions\" directory found in the parent directories of \"res\" - packed asset bundle cannot be loaded.");
}
m_AssetBundleType = eAssetBundleType::Packed;
m_RootPath = (m_Path / ".." / "..");
@@ -54,15 +54,15 @@ AssetManager::AssetManager(const std::filesystem::path& path) {
}
switch (m_AssetBundleType) {
case eAssetBundleType::Packed: {
this->LoadPackIndex();
break;
}
case eAssetBundleType::None:
[[fallthrough]];
case eAssetBundleType::Unpacked: {
break;
}
case eAssetBundleType::Packed: {
this->LoadPackIndex();
break;
}
case eAssetBundleType::None:
[[fallthrough]];
case eAssetBundleType::Unpacked: {
break;
}
}
}
@@ -79,7 +79,7 @@ bool AssetManager::HasFile(std::string fixedName) const {
std::replace(fixedName.begin(), fixedName.end(), '\\', '/');
if (std::filesystem::exists(m_ResPath / fixedName)) return true;
if (this->m_AssetBundleType == eAssetBundleType::Unpacked) return false;
if (this->m_AssetBundleType == eAssetBundleType::Unpacked || !m_PackIndex) return false;
std::replace(fixedName.begin(), fixedName.end(), '/', '\\');
if (fixedName.rfind("client\\res\\", 0) != 0) fixedName = "client\\res\\" + fixedName;
@@ -145,8 +145,12 @@ bool AssetManager::GetFile(std::string fixedName, char** data, uint32_t* len) co
}
const auto& pack = this->m_PackIndex->GetPacks().at(packIndex);
const bool success = pack.ReadFileFromPack(crc, data, len);
bool success = false;
try {
success = pack.ReadFileFromPack(crc, data, len);
} catch (std::exception& e) {
LOG("Failed to read file %s from pack file", fixedName.c_str());
}
return success;
}

View File

@@ -46,6 +46,7 @@ bool Pack::HasFile(const uint32_t crc) const {
}
bool Pack::ReadFileFromPack(const uint32_t crc, char** data, uint32_t* len) const {
const auto pathStr = m_FilePath.string();
// Time for some wacky C file reading for speed reasons
PackRecord pkRecord{};
@@ -65,16 +66,21 @@ bool Pack::ReadFileFromPack(const uint32_t crc, char** data, uint32_t* len) cons
bool isCompressed = (pkRecord.m_IsCompressed & 0xff) > 0;
auto inPackSize = isCompressed ? pkRecord.m_CompressedSize : pkRecord.m_UncompressedSize;
FILE* file;
FILE* file = nullptr;
#ifdef _WIN32
fopen_s(&file, m_FilePath.string().c_str(), "rb");
fopen_s(&file, pathStr.c_str(), "rb");
#elif __APPLE__
// macOS has 64bit file IO by default
file = fopen(m_FilePath.string().c_str(), "rb");
file = fopen(pathStr.c_str(), "rb");
#else
file = fopen64(m_FilePath.string().c_str(), "rb");
file = fopen64(pathStr.c_str(), "rb");
#endif
if (!file) {
LOG("No file found for path %s", pathStr.c_str());
throw std::runtime_error("Could not find file " + pathStr);
}
fseek(file, pos, SEEK_SET);
if (!isCompressed) {
@@ -102,14 +108,18 @@ bool Pack::ReadFileFromPack(const uint32_t crc, char** data, uint32_t* len) cons
int32_t readInData = fread(&size, sizeof(uint32_t), 1, file);
pos += 4; // Move pointer position 4 to the right
char* chunk = static_cast<char*>(malloc(size));
int32_t readInData2 = fread(chunk, sizeof(int8_t), size, file);
std::unique_ptr<char[]> chunk(new char[size]);
int32_t readInData2 = fread(chunk.get(), sizeof(int8_t), size, file);
pos += size; // Move pointer position the amount of bytes read to the right
int32_t err;
currentReadPos += ZCompression::Decompress(reinterpret_cast<uint8_t*>(chunk), size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, err);
const auto countToRead = ZCompression::Decompress(reinterpret_cast<uint8_t*>(chunk.get()), size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, err);
if (countToRead == -1) {
LOG("Error decompressing zlib data from file %s", pathStr.c_str());
throw std::runtime_error("Error decompressing zlib data from file " + pathStr);
}
currentReadPos += countToRead;
free(chunk);
}
*data = decompressedData;

View File

@@ -84,3 +84,7 @@ void dConfig::ProcessLine(const std::string& line) {
this->m_ConfigValues.insert(std::make_pair(key, value));
}
std::string dConfig::GetValue(const std::string& key, const char* emptyValue) {
return GetValue(key, std::string(emptyValue));
};

View File

@@ -5,6 +5,8 @@
#include <map>
#include <string>
#include "GeneralUtils.h"
class dConfig {
public:
dConfig(const std::string& filepath);
@@ -22,6 +24,14 @@ public:
*/
const std::string& GetValue(std::string key);
// Gets a value from the config and returns the parsed value, or the default value should parsing have failed.
template<typename T>
T GetValue(const std::string& key, const T emptyValue = T()) {
return GeneralUtils::TryParse<T>(GetValue(key)).value_or(emptyValue);
}
std::string GetValue(const std::string& key, const char* emptyValue);
/**
* Loads the config from a file
*/
@@ -43,3 +53,9 @@ private:
std::vector<std::function<void()>> m_ConfigHandlers;
std::string m_ConfigFilePath;
};
template<>
inline std::string dConfig::GetValue(const std::string& key, const std::string emptyValue) {
const auto& value = GetValue(key);
return value.empty() ? emptyValue : value;
};

View File

@@ -16,8 +16,8 @@
// These are the same define, but they mean two different things in different contexts
// so a different define to distinguish what calculation is happening will help clarity.
#define FRAMES_TO_MS(x) 1000 / x
#define MS_TO_FRAMES(x) 1000 / x
#define FRAMES_TO_MS(x) (1000 / (x))
#define MS_TO_FRAMES(x) (1000 / (x))
//=========== FRAME TIMINGS ===========
constexpr uint32_t highFramerate = 60;
@@ -58,6 +58,7 @@ constexpr LWOCLONEID LWOCLONEID_INVALID = -1; //!< Invalid LWOCLONEID
constexpr LWOINSTANCEID LWOINSTANCEID_INVALID = -1; //!< Invalid LWOINSTANCEID
constexpr LWOMAPID LWOMAPID_INVALID = -1; //!< Invalid LWOMAPID
constexpr uint64_t LWOZONEID_INVALID = 0; //!< Invalid LWOZONEID
constexpr uint32_t MAX_MESSAGE_LENGTH = 0x500000; //!< Prevent exceptionally large msgs from being processed. Should always be used to check user provided inputs.
constexpr float PI = 3.14159f;
@@ -110,18 +111,6 @@ private:
constexpr LWOSCENEID LWOSCENEID_INVALID = -1;
struct LWONameValue {
uint32_t length = 0; //!< The length of the name
std::u16string name; //!< The name
LWONameValue() = default;
LWONameValue(const std::u16string& name) {
this->name = name;
this->length = static_cast<uint32_t>(name.length());
}
};
struct FriendData {
public:
bool isOnline = false;

View File

@@ -20,7 +20,8 @@ enum class eCharacterVersion : uint32_t {
NJ_JAYMISSIONS,
NEXUS_FORCE_EXPLORER, // Fixes pet ids in player inventories
PET_IDS, // Fixes pet ids in player inventories
UP_TO_DATE, // will become INVENTORY_PERSISTENT_IDS
INVENTORY_PERSISTENT_IDS, // Fixes racing meta missions
UP_TO_DATE, // will become RACING_META_MISSIONS
};
#endif //!__ECHARACTERVERSION__H__