From 18295017c16da07e1375a289497d84f70c0094c4 Mon Sep 17 00:00:00 2001
From: David Markowitz <EmosewaMC@gmail.com>
Date: Tue, 24 Dec 2024 14:09:39 -0800
Subject: [PATCH] use encoding

use template function

Update GeneralUtils.cpp

consolidate duplicate code

Update GeneralUtils.cpp

Update BinaryIO.cpp

compilers
---
 dCommon/BinaryIO.cpp           | 17 +++++++++++++----
 dCommon/BinaryIO.h             |  2 ++
 dCommon/FdbToSqlite.cpp        |  5 +++--
 dCommon/GeneralUtils.cpp       | 17 +++++++++++++----
 dCommon/GeneralUtils.h         |  8 ++++++++
 dMasterServer/MasterServer.cpp |  2 +-
 6 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/dCommon/BinaryIO.cpp b/dCommon/BinaryIO.cpp
index b64fb3ba..4f885c14 100644
--- a/dCommon/BinaryIO.cpp
+++ b/dCommon/BinaryIO.cpp
@@ -2,16 +2,25 @@
 #include <string>
 
 //For reading null-terminated strings
-std::string BinaryIO::ReadString(std::istream& instream) {
-	std::string toReturn;
-	char buffer;
+template<typename StringType>
+StringType ReadString(std::istream& instream) {
+	StringType toReturn{};
+	typename StringType::value_type buffer{};
 
 	BinaryIO::BinaryRead(instream, buffer);
 
 	while (buffer != 0x00) {
 		toReturn += buffer;
-		BinaryRead(instream, buffer);
+		BinaryIO::BinaryRead(instream, buffer);
 	}
 
 	return toReturn;
 }
+
+std::string BinaryIO::ReadString(std::istream& instream) {
+	return ::ReadString<std::string>(instream);
+}
+
+std::u8string BinaryIO::ReadU8String(std::istream& instream) {
+	return ::ReadString<std::u8string>(instream);
+}
diff --git a/dCommon/BinaryIO.h b/dCommon/BinaryIO.h
index d1e728ca..634d7ce7 100644
--- a/dCommon/BinaryIO.h
+++ b/dCommon/BinaryIO.h
@@ -65,6 +65,8 @@ namespace BinaryIO {
 
 	std::string ReadString(std::istream& instream);
 
+	std::u8string ReadU8String(std::istream& instream);
+
 	inline bool DoesFileExist(const std::string& name) {
 		std::ifstream f(name.c_str());
 		return f.good();
diff --git a/dCommon/FdbToSqlite.cpp b/dCommon/FdbToSqlite.cpp
index d8409cd5..75e055bb 100644
--- a/dCommon/FdbToSqlite.cpp
+++ b/dCommon/FdbToSqlite.cpp
@@ -65,13 +65,14 @@ int64_t FdbToSqlite::Convert::ReadInt64(std::istream& cdClientBuffer) {
 	return value;
 }
 
+// cdclient is encoded in latin1
 std::string FdbToSqlite::Convert::ReadString(std::istream& cdClientBuffer) {
 	int32_t prevPosition = SeekPointer(cdClientBuffer);
 
-	auto readString = BinaryIO::ReadString(cdClientBuffer);
+	const auto readString = BinaryIO::ReadU8String(cdClientBuffer);
 
 	cdClientBuffer.seekg(prevPosition);
-	return readString;
+	return GeneralUtils::Latin1ToWTF8(readString);
 }
 
 int32_t FdbToSqlite::Convert::SeekPointer(std::istream& cdClientBuffer) {
diff --git a/dCommon/GeneralUtils.cpp b/dCommon/GeneralUtils.cpp
index 34d0aefb..819fa805 100644
--- a/dCommon/GeneralUtils.cpp
+++ b/dCommon/GeneralUtils.cpp
@@ -167,17 +167,19 @@ std::u16string GeneralUtils::ASCIIToUTF16(const std::string_view string, const s
 	return ret;
 }
 
-//! Converts a (potentially-ill-formed) UTF-16 string to UTF-8
+
+//! Converts a (potentially-ill-formed) Latin1 string to UTF-8
 //! See: <http://simonsapin.github.io/wtf-8/#decoding-ill-formed-utf-16>
-std::string GeneralUtils::UTF16ToWTF8(const std::u16string_view string, const size_t size) {
+template<typename StringType>
+std::string ToWTF8(const StringType string, const size_t size) {
 	const size_t newSize = MinSize(size, string);
 	std::string ret;
 	ret.reserve(newSize);
 
 	for (size_t i = 0; i < newSize; ++i) {
-		const char16_t u = string[i];
+		const auto u = string[i];
 		if (IsLeadSurrogate(u) && (i + 1) < newSize) {
-			const char16_t next = string[i + 1];
+			const auto next = string[i + 1];
 			if (IsTrailSurrogate(next)) {
 				i += 1;
 				const char32_t cp = 0x10000
@@ -194,6 +196,13 @@ std::string GeneralUtils::UTF16ToWTF8(const std::u16string_view string, const si
 
 	return ret;
 }
+std::string GeneralUtils::Latin1ToWTF8(const std::u8string_view string, const size_t size) {
+	return ToWTF8(string, size);
+}
+
+std::string GeneralUtils::UTF16ToWTF8(const std::u16string_view string, const size_t size) {
+	return ToWTF8(string, size);
+}
 
 bool GeneralUtils::CaseInsensitiveStringCompare(const std::string_view a, const std::string_view b) {
 	return std::equal(a.begin(), a.end(), b.begin(), b.end(), [](char a, char b) { return tolower(a) == tolower(b); });
diff --git a/dCommon/GeneralUtils.h b/dCommon/GeneralUtils.h
index 2c93b656..469a16da 100644
--- a/dCommon/GeneralUtils.h
+++ b/dCommon/GeneralUtils.h
@@ -51,6 +51,14 @@ namespace GeneralUtils {
 		bool _NextUTF8Char(std::string_view& slice, uint32_t& out);
 	}
 
+	//! Converts a Latin1 string to a UTF-8 string
+	/*!
+	  \param string The string to convert
+	  \param size A size to trim the string to. Default is SIZE_MAX (No trimming)
+	  \return An UTF-8 representation of the string
+	 */
+	std::string Latin1ToWTF8(const std::u8string_view string, const size_t size = SIZE_MAX);
+
 	//! Converts a UTF-16 string to a UTF-8 string
 	/*!
 	  \param string The string to convert
diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp
index b764169a..ac47b890 100644
--- a/dMasterServer/MasterServer.cpp
+++ b/dMasterServer/MasterServer.cpp
@@ -177,7 +177,7 @@ int main(int argc, char** argv) {
 	}
 
 	// Run migrations should any need to be run.
-	MigrationRunner::RunSQLiteMigrations();	
+	MigrationRunner::RunSQLiteMigrations();
 
 	//If the first command line argument is -a or --account then make the user
 	//input a username and password, with the password being hidden.