Add an AMF Deserializer as well as corresponding Unit Tests (#599)

* Add AMFDeserializer

Add an AMFDeserializer

Reverted unrelated changes

Add unit tests for AMFDeserializer

Added unit tests for the AMFDeserializer

Finish tests

Finish the AMF deserializer tests.  This commit finishes the positive test case and implements a load test case that is expected to take less than 1.5 seconds to process.

Modularized tests

Made tests a bit modular and split into more methods

Specified binary read from file

Specified that on the IO stream we are reading a binary file otherwise windows will terminate reading the binary file on seeing a 1A byte.

Added more tests

Added tests for unimplemented values and edited a test file to be more modular

Updated test text

Fix spacing

Update AMFDeserializeTests.cpp

* Update CMakeLists.txt

* Update AMFDeserializeTests.cpp

f

Actually follow the AMF spec

Update AMFDeserializeTests.cpp

tabs

Add in commented tests

* Follow spec

formatting

Add Integer Tests

Follow Spec more

Follow spec

* Use unique_ptr

* Update AMFDeserialize.cpp

Semantics

Update AMFDeserialize.cpp

Add new lines to EOF

CMake fix

* Add better std string read

Co-authored-by: Daniel Seiler <xiphoseer@mailbox.org>

* make not static

Co-authored-by: Daniel Seiler <xiphoseer@mailbox.org>

dCommon/AMFDeserialize.cpp

@@ -0,0 +1,158 @@
+#include "AMFDeserialize.h"
+
+#include "AMFFormat.h"
+
+/**
+ * AMF3 Reference document https://rtmp.veriskope.com/pdf/amf3-file-format-spec.pdf
+ * AMF3 Deserializer written by EmosewaMC
+ */
+
+AMFValue* AMFDeserialize::Read(RakNet::BitStream* inStream) {
+	if (!inStream) return nullptr;
+	AMFValue* returnValue = nullptr;
+	// Read in the value type from the bitStream
+	int8_t marker;
+	inStream->Read(marker);
+	// Based on the typing, create the value associated with that and return the base value class
+	switch (marker) {
+		case AMFValueType::AMFUndefined: {
+			returnValue = new AMFUndefinedValue();
+			break;
+		}
+
+		case AMFValueType::AMFNull: {
+			returnValue = new AMFNullValue();
+			break;
+		}
+
+		case AMFValueType::AMFFalse: {
+			returnValue = new AMFFalseValue();
+			break;
+		}
+
+		case AMFValueType::AMFTrue: {
+			returnValue = new AMFTrueValue();
+			break;
+		}
+
+		case AMFValueType::AMFInteger: {
+			returnValue = ReadAmfInteger(inStream);
+			break;
+		}
+
+		case AMFValueType::AMFDouble: {
+			returnValue = ReadAmfDouble(inStream);
+			break;
+		}
+
+		case AMFValueType::AMFString: {
+			returnValue = ReadAmfString(inStream);
+			break;
+		}
+
+		case AMFValueType::AMFArray: {
+			returnValue = ReadAmfArray(inStream);
+			break;
+		}
+
+		// TODO We do not need these values, but if someone wants to implement them
+		// then please do so and add the corresponding unit tests.
+		case AMFValueType::AMFXMLDoc:
+		case AMFValueType::AMFDate:
+		case AMFValueType::AMFObject:
+		case AMFValueType::AMFXML:
+		case AMFValueType::AMFByteArray:
+		case AMFValueType::AMFVectorInt:
+		case AMFValueType::AMFVectorUInt:
+		case AMFValueType::AMFVectorDouble:
+		case AMFValueType::AMFVectorObject:
+		case AMFValueType::AMFDictionary: {
+			throw static_cast<AMFValueType>(marker);
+			break;
+		}
+		default:
+			throw static_cast<AMFValueType>(marker);
+			break;
+	}
+	return returnValue;
+}
+
+uint32_t AMFDeserialize::ReadU29(RakNet::BitStream* inStream) {
+	bool byteFlag = true;
+	uint32_t actualNumber{};
+	uint8_t numberOfBytesRead{};
+	while (byteFlag && numberOfBytesRead < 4) {
+		uint8_t byte{};
+		inStream->Read(byte);
+		// Parse the byte
+		if (numberOfBytesRead < 3) {
+			byteFlag = byte & static_cast<uint8_t>(1 << 7);
+			byte = byte << 1UL;
+		}
+		// Combine the read byte with our current read in number
+		actualNumber <<= 8UL;
+		actualNumber |= static_cast<uint32_t>(byte);
+		// If we are not done reading in bytes, shift right 1 bit
+		if (numberOfBytesRead < 3) actualNumber = actualNumber >> 1UL;
+		numberOfBytesRead++;
+	}
+	return actualNumber;
+}
+
+std::string AMFDeserialize::ReadString(RakNet::BitStream* inStream) {
+	auto length = ReadU29(inStream);
+	// Check if this is a reference
+	bool isReference = length % 2 == 1;
+	// Right shift by 1 bit to get index if reference or size of next string if value
+	length = length >> 1;
+	if (isReference) {
+		std::string value(length, 0);
+		inStream->Read(&value[0], length);
+		// Empty strings are never sent by reference
+		if (!value.empty()) accessedElements.push_back(value);
+		return value;
+	} else {
+		// Length is a reference to a previous index - use that as the read in value
+		return accessedElements[length];
+	}
+}
+
+AMFValue* AMFDeserialize::ReadAmfDouble(RakNet::BitStream* inStream) {
+	auto doubleValue = new AMFDoubleValue();
+	double value;
+	inStream->Read<double>(value);
+	doubleValue->SetDoubleValue(value);
+	return doubleValue;
+}
+
+AMFValue* AMFDeserialize::ReadAmfArray(RakNet::BitStream* inStream) {
+	auto arrayValue = new AMFArrayValue();
+	auto sizeOfDenseArray = (ReadU29(inStream) >> 1);
+	if (sizeOfDenseArray >= 1) {
+		char valueType;
+		inStream->Read(valueType); // Unused
+		for (uint32_t i = 0; i < sizeOfDenseArray; i++) {
+			arrayValue->PushBackValue(Read(inStream));
+		}
+	} else {
+		while (true) {
+			auto key = ReadString(inStream);
+			// No more values when we encounter an empty string
+			if (key.size() == 0) break;
+			arrayValue->InsertValue(key, Read(inStream));
+		}
+	}
+	return arrayValue;
+}
+
+AMFValue* AMFDeserialize::ReadAmfString(RakNet::BitStream* inStream) {
+	auto stringValue = new AMFStringValue();
+	stringValue->SetStringValue(ReadString(inStream));
+	return stringValue;
+}
+
+AMFValue* AMFDeserialize::ReadAmfInteger(RakNet::BitStream* inStream) {
+	auto integerValue = new AMFIntegerValue();
+	integerValue->SetIntegerValue(ReadU29(inStream));
+	return integerValue;
+}

dCommon/AMFDeserialize.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include "BitStream.h"
+
+#include <vector>
+#include <string>
+
+class AMFValue;
+class AMFDeserialize {
+	public:
+		/**
+		 * Read an AMF3 value from a bitstream.
+		 * 
+		 * @param inStream inStream to read value from.
+		 * @return Returns an AMFValue with all the information from the bitStream in it.
+		 */
+		AMFValue* Read(RakNet::BitStream* inStream);
+	private:
+		/**
+		 * @brief Private method to read a U29 integer from a bitstream
+		 * 
+		 * @param inStream bitstream to read data from
+		 * @return The number as an unsigned 29 bit integer
+		 */
+		uint32_t ReadU29(RakNet::BitStream* inStream);
+
+		/**
+		 * @brief Reads a string from a bitstream
+		 * 
+		 * @param inStream bitStream to read data from
+		 * @return The read string
+		 */
+		std::string ReadString(RakNet::BitStream* inStream);
+		
+		/**
+		 * @brief Read an AMFDouble value from a bitStream
+		 * 
+		 * @param inStream bitStream to read data from
+		 * @return Double value represented as an AMFValue
+		 */
+		AMFValue* ReadAmfDouble(RakNet::BitStream* inStream);
+
+		/**
+		 * @brief Read an AMFArray from a bitStream
+		 * 
+		 * @param inStream bitStream to read data from
+		 * @return Array value represented as an AMFValue
+		 */
+		AMFValue* ReadAmfArray(RakNet::BitStream* inStream);
+
+		/**
+		 * @brief Read an AMFString from a bitStream
+		 * 
+		 * @param inStream bitStream to read data from
+		 * @return String value represented as an AMFValue
+		 */
+		AMFValue* ReadAmfString(RakNet::BitStream* inStream);
+
+		/**
+		 * @brief Read an AMFInteger from a bitStream
+		 * 
+		 * @param inStream bitStream to read data from
+		 * @return Integer value represented as an AMFValue
+		 */
+		AMFValue* ReadAmfInteger(RakNet::BitStream* inStream);
+
+		/**
+		 * List of strings read so far saved to be read by reference.
+		 */
+		std::vector<std::string> accessedElements;
+};

dCommon/AMFFormat.h

@@ -308,6 +308,18 @@ public:
       \return Where the iterator ends
      */
     _AMFArrayList_::iterator GetDenseIteratorEnd();
+
+    //! Returns the associative map
+    /*!
+      \return The associative map
+     */
+    _AMFArrayMap_ GetAssociativeMap() { return this->associative; };
+
+    //! Returns the dense array
+    /*!
+      \return The dense array
+     */
+    _AMFArrayList_ GetDenseArray() { return this->dense; };
 };
 
 //! The anonymous object value AMF type

dCommon/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(DCOMMON_SOURCES "AMFFormat.cpp"
+		"AMFDeserialize.cpp"
 		"AMFFormat_BitStream.cpp"
 		"BinaryIO.cpp"
 		"dConfig.cpp"