chore: Speed up and cleanup level and zone loading; Add safer asset buffer reading (#1314)

* Remove std::couts littered throughout the base

* working

End of optimizations for now

going faster

* Remove extraneous compare function

std::less<LWOSCENEID> already does this in a map.

* gaming

* Update Zone.cpp

* dlu is moving to bitbucket again

* Update Level.cpp

---------

Co-authored-by: Jettford <mrjettbradford@gmail.com>

dCommon/BinaryIO.cpp

@@ -1,14 +1,6 @@
 #include "BinaryIO.h"
 #include <string>
 
-void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outstream) {
-	//BinaryWrite(outstream, uint32_t(stringToWrite.length()));
-
-	for (size_t i = 0; i < size_t(stringToWrite.length()); ++i) {
-		BinaryIO::BinaryWrite(outstream, stringToWrite[i]);
-	}
-}
-
 //For reading null-terminated strings
 std::string BinaryIO::ReadString(std::istream& instream) {
 	std::string toReturn;
@@ -23,36 +15,3 @@ std::string BinaryIO::ReadString(std::istream& instream) {
 
 	return toReturn;
 }
-
-//For reading strings of a specific size
-std::string BinaryIO::ReadString(std::istream& instream, size_t size) {
-	std::string toReturn;
-	char buffer;
-
-	for (size_t i = 0; i < size; ++i) {
-		BinaryIO::BinaryRead(instream, buffer);
-		toReturn += buffer;
-	}
-
-	return toReturn;
-}
-
-std::string BinaryIO::ReadWString(std::istream& instream) {
-	size_t size;
-	BinaryRead(instream, size);
-	//toReturn.resize(size);
-	std::string test;
-	unsigned char buf;
-
-	for (size_t i = 0; i < size; ++i) {
-		//instream.ignore(1);
-		BinaryRead(instream, buf);
-		test += buf;
-	}
-
-	//printf("%s\n", test.c_str());
-
-	//instream.read((char*)&toReturn[0], size * 2);
-	//std::string str(toReturn.begin(), toReturn.end());
-	return test;
-}

dCommon/BinaryIO.h

@@ -1,8 +1,17 @@
 #pragma once
+
+#ifndef __BINARYIO__H__
+#define __BINARYIO__H__
+
 #include <iostream>
 #include <fstream>
+#include <string>
+
+#include "Game.h"
+#include "Logger.h"
 
 namespace BinaryIO {
+
 	template<typename T>
 	std::ostream& BinaryWrite(std::ostream& stream, const T& value) {
 		return stream.write(reinterpret_cast<const char*>(&value), sizeof(T));
@@ -15,13 +24,51 @@ namespace BinaryIO {
 		return stream.read(reinterpret_cast<char*>(&value), sizeof(T));
 	}
 
-	void WriteString(const std::string& stringToWrite, std::ofstream& outstream);
+	enum class ReadType : int8_t {
+		WideString = 0,
+		String = 1,
+	};
+
+	template<typename SizeType>
+	inline void ReadString(std::istream& stream, std::u16string& value) {
+		static_assert(std::is_integral<SizeType>::value, "SizeType must be an integral type.");
+
+		SizeType size;
+		BinaryRead(stream, size);
+
+		if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
+		value.resize(size);
+		stream.read(reinterpret_cast<char*>(value.data()), size * sizeof(uint16_t));
+	}
+
+	template<typename SizeType>
+	inline void ReadString(std::istream& stream, std::string& value, ReadType readType) {
+		static_assert(std::is_integral<SizeType>::value, "SizeType must be an integral type.");
+
+		SizeType size;
+		BinaryRead(stream, size);
+
+		if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
+		value.resize(size);
+		if (readType == ReadType::WideString) {
+			uint16_t wideChar;
+
+			// Faster to do this than to read a u16string and convert it to a string since we only go through allocator once
+			for (SizeType i = 0; i < size; ++i) {
+				BinaryRead(stream, wideChar);
+				value[i] = static_cast<char>(wideChar);
+			}
+		} else {
+			stream.read(value.data(), size);
+		}
+	}
+
 	std::string ReadString(std::istream& instream);
-	std::string ReadString(std::istream& instream, size_t size);
-	std::string ReadWString(std::istream& instream);
 
 	inline bool DoesFileExist(const std::string& name) {
 		std::ifstream f(name.c_str());
 		return f.good();
 	}
 }
+
+#endif  //!__BINARYIO__H__

dCommon/FdbToSqlite.cpp

@@ -28,19 +28,17 @@ FdbToSqlite::Convert::Convert(std::string binaryOutPath) {
 	this->m_BinaryOutPath = binaryOutPath;
 }
 
-bool FdbToSqlite::Convert::ConvertDatabase(AssetMemoryBuffer& buffer) {
+bool FdbToSqlite::Convert::ConvertDatabase(AssetStream& buffer) {
 	if (m_ConversionStarted) return false;
 
-	std::istream cdClientBuffer(&buffer);
-
 	this->m_ConversionStarted = true;
 	try {
 		CDClientDatabase::Connect(m_BinaryOutPath + "/CDServer.sqlite");
 
 		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
 
-		int32_t numberOfTables = ReadInt32(cdClientBuffer);
-		ReadTables(numberOfTables, cdClientBuffer);
+		int32_t numberOfTables = ReadInt32(buffer);
+		ReadTables(numberOfTables, buffer);
 
 		CDClientDatabase::ExecuteQuery("COMMIT;");
 	} catch (CppSQLite3Exception& e) {

dCommon/FdbToSqlite.h

@@ -7,7 +7,7 @@
 #include <iosfwd>
 #include <map>
 
-class AssetMemoryBuffer;
+#include "AssetManager.h"
 
 enum class eSqliteDataType : int32_t;
 
@@ -27,7 +27,7 @@ namespace FdbToSqlite {
 		 * 
 		 * @return true if the database was converted properly, false otherwise. 
 		 */
-		bool ConvertDatabase(AssetMemoryBuffer& buffer);
+		bool ConvertDatabase(AssetStream& buffer);
 
 		/**
 		 * @brief Reads a 32 bit int from the fdb file.

dCommon/dClient/AssetManager.cpp

@@ -152,13 +152,12 @@ bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) {
 	return success;
 }
 
-AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
-	char* buf;
-	uint32_t len;
+AssetStream AssetManager::GetFile(const char* name) {
+	char* buf; uint32_t len;
 
 	bool success = this->GetFile(name, &buf, &len);
 
-	return AssetMemoryBuffer(buf, len, success);
+	return AssetStream(buf, len, success);
 }
 
 uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {

dCommon/dClient/AssetManager.h

@@ -25,6 +25,10 @@ struct AssetMemoryBuffer : std::streambuf {
 		this->setg(base, base, base + n);
 	}
 
+	~AssetMemoryBuffer() {
+		free(m_Base);
+	}
+
 	pos_type seekpos(pos_type sp, std::ios_base::openmode which) override {
 		return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which);
 	}
@@ -40,9 +44,17 @@ struct AssetMemoryBuffer : std::streambuf {
 			setg(eback(), eback() + off, egptr());
 		return gptr() - eback();
 	}
+};
 
-	void close() {
-		free(m_Base);
+struct AssetStream : std::istream {
+	AssetStream(char* base, std::ptrdiff_t n, bool success) : std::istream(new AssetMemoryBuffer(base, n, success)) {}
+
+	~AssetStream() {
+		delete rdbuf();
+	}
+
+	operator bool() {
+		return reinterpret_cast<AssetMemoryBuffer*>(rdbuf())->m_Success;
 	}
 };
 
@@ -56,7 +68,7 @@ public:
 
 	bool HasFile(const char* name);
 	bool GetFile(const char* name, char** data, uint32_t* len);
-	AssetMemoryBuffer GetFileAsBuffer(const char* name);
+	AssetStream GetFile(const char* name);
 
 private:
 	void LoadPackIndex();

dCommon/dClient/PackIndex.cpp

@@ -8,21 +8,10 @@ PackIndex::PackIndex(const std::filesystem::path& filePath) {
 
 	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_Version);
 	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackPathCount);
-
-	for (int i = 0; i < m_PackPathCount; i++) {
-		uint32_t stringLen = 0;
-		BinaryIO::BinaryRead<uint32_t>(m_FileStream, stringLen);
-
-		std::string path;
-
-		for (int j = 0; j < stringLen; j++) {
-			char inChar;
-			BinaryIO::BinaryRead<char>(m_FileStream, inChar);
-
-			path += inChar;
-		}
-
-		m_PackPaths.push_back(path);
+	
+	m_PackPaths.resize(m_PackPathCount);
+	for (auto& item : m_PackPaths) {
+		BinaryIO::ReadString<uint32_t>(m_FileStream, item, BinaryIO::ReadType::String);
 	}
 
 	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackFileIndexCount);