Compact binary to JSON (#12)

CompactBinary: Support for converting CbObject to JSON

4 files changed, 391 insertions, 2 deletions

diff --git a/zencore/include/zencore/base64.h b/zencore/include/zencore/base64.h
new file mode 100644
index 000000000..4d78b085f
--- /dev/null
+++ b/zencore/include/zencore/base64.h
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+struct Base64
+{
+	template<typename CharType>
+	static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest);
+
+	static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; }
+};
+
+}  // namespace zen
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index 60e9ec7aa..ab01402f8 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -42,8 +42,22 @@ public:
 	}
 
 	inline uint64_t GetTicks() const { return Ticks; }
-	inline bool		operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
-	inline auto		operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+	int	 GetYear() const;
+	int	 GetMonth() const;
+	int	 GetDay() const;
+	int	 GetHour() const;
+	int	 GetHour12() const;
+	int	 GetMinute() const;
+	int	 GetSecond() const;
+	int	 GetMillisecond() const;
+	void GetDate(int& Year, int& Month, int& Day) const;
+
+	inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
+	inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+	std::string ToString(const char* Format) const;
+	std::string ToIso8601() const;
 
 private:
 	void	 Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond);
@@ -99,6 +113,25 @@ public:
 	/** The number of timespan ticks per year (365 days, not accounting for leap years). */
 	static constexpr int64_t TicksPerYear = 365 * TicksPerDay;
 
+	int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); }
+
+	int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); }
+
+	int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); }
+
+	int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); }
+
+	int GetDays() const { return (int)(Ticks / TicksPerDay); }
+
+	int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); }
+
+	int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); }
+
+	int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); }
+
+	ZENCORE_API std::string ToString(const char* Format) const;
+	ZENCORE_API std::string ToString() const;
+
 private:
 	void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano);
 
@@ -108,6 +141,8 @@ private:
 struct Guid
 {
 	uint32_t A, B, C, D;
+
+	StringBuilderBase& ToString(StringBuilderBase& OutString) const;
 };
 
 //////////////////////////////////////////////////////////////////////////
@@ -442,6 +477,11 @@ public:
 
 	/** Returns the name of the field if it has a name, otherwise an empty view. */
 	constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); }
+	/** Returns the name of the field if it has a name, otherwise an empty view. */
+	constexpr inline std::u8string_view GetU8Name() const
+	{
+		return std::u8string_view(static_cast<const char8_t*>(Payload) - NameLen, NameLen);
+	}
 
 	/** Returns the value for unchecked access. Prefer the typed accessors below. */
 	inline CbValue GetValue() const;
@@ -907,6 +947,11 @@ private:
 	inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {}
 };
 
+/**
+ * Serialize a compact binary object to JSON.
+ */
+ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder);
+
 class CbObjectView : protected CbFieldView
 {
 	friend class CbFieldView;
@@ -989,6 +1034,12 @@ public:
 	/** Whether the field has a value. */
 	using CbFieldView::operator bool;
 
+	StringBuilderBase& ToJson(StringBuilderBase& Builder) const
+	{
+		CompactBinaryToJson(*this, Builder);
+		return Builder;
+	}
+
 private:
 	friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); }
 	friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); }
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index bb9b1c896..2c0d10577 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -14,6 +14,8 @@
 #include <span>
 #include <string_view>
 
+#include <type_traits>
+
 namespace zen {
 
 //////////////////////////////////////////////////////////////////////////
@@ -94,6 +96,14 @@ public:
 	const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete;
 	const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete;
 
+	inline size_t AddUninitialized(size_t Count)
+	{
+		EnsureCapacity(Count);
+		const size_t OldCount = Size();
+		m_CurPos += Count;
+		return OldCount;
+	}
+
 	StringBuilderImpl& Append(C OneChar)
 	{
 		EnsureCapacity(1);
@@ -209,6 +219,12 @@ public:
 		return AppendRange(String.data(), String.data() + String.size());
 	}
 
+	inline void RemoveSuffix(int32_t Count)
+	{
+		ZEN_ASSERT(Count <= Size());
+		m_CurPos -= Count;
+	}
+
 	inline const C* c_str() const
 	{
 		EnsureNulTerminated();
@@ -322,6 +338,12 @@ protected:
 
 extern template class StringBuilderImpl<char>;
 
+inline StringBuilderImpl<char>&
+operator<<(StringBuilderImpl<char>& Builder, char Char)
+{
+	return Builder.Append(Char);
+}
+
 class StringBuilderBase : public StringBuilderImpl<char>
 {
 public:
@@ -661,6 +683,303 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 
 //////////////////////////////////////////////////////////////////////////
 
+/**
+ * ASCII character bitset useful for fast and readable parsing
+ *
+ * Entirely constexpr. Works with both wide and narrow strings.
+ *
+ * Example use cases:
+ *
+ *   constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n");
+ *   bool bIsWhitespace = WhitespaceCharacters.Test(MyChar);
+ *   const char* HelloWorld = AsciiSet::Skip("  \t\tHello world!", WhitespaceCharacters);
+ *
+ *   constexpr AsciiSet XmlEscapeChars("&<>\"'");
+ *   check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars));
+ *
+ *   constexpr AsciiSet Delimiters(".:;");
+ *   const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters);
+ *   FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName);
+ *
+ *   constexpr AsciiSet Slashes("/\\");
+ *   const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes);
+ *   const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName;
+ */
+class AsciiSet
+{
+public:
+	template<typename CharType, int N>
+	constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars))
+	{
+	}
+
+	/** Returns true if a character is part of the set */
+	template<typename CharType>
+	constexpr inline bool Contains(CharType Char) const
+	{
+		using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+		return !!TestImpl((UnsignedCharType)Char);
+	}
+
+	/** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */
+	template<typename CharType>
+	constexpr inline uint64_t Test(CharType Char) const
+	{
+		using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+		return TestImpl((UnsignedCharType)Char);
+	}
+
+	/** Create new set with specified character in it */
+	constexpr inline AsciiSet operator+(char Char) const
+	{
+		using UnsignedCharType = std::make_unsigned<char>::type;
+
+		InitData Bitset = {LoMask, HiMask};
+		SetImpl(Bitset, (UnsignedCharType)Char);
+		return AsciiSet(Bitset);
+	}
+
+	/** Create new set containing inverse set of characters - likely including null-terminator */
+	constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); }
+
+	////////// Algorithms for C strings //////////
+
+	/** Find first character of string inside set or end pointer. Never returns null. */
+	template<class CharType>
+	static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set)
+	{
+		for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str)
+			;
+
+		return Str;
+	}
+
+	/** Find last character of string inside set or end pointer. Never returns null. */
+	template<class CharType>
+	static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set)
+	{
+		const CharType* Last = FindFirstOrEnd(Str, Set);
+
+		for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set))
+		{
+			Last = It;
+		}
+
+		return Last;
+	}
+
+	/** Find first character of string outside of set. Never returns null. */
+	template<typename CharType>
+	static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set)
+	{
+		while (Set.Contains(*Str))
+		{
+			++Str;
+		}
+
+		return Str;
+	}
+
+	/** Test if string contains any character in set */
+	template<typename CharType>
+	static constexpr bool HasAny(const CharType* Str, AsciiSet Set)
+	{
+		return *FindFirstOrEnd(Str, Set) != '\0';
+	}
+
+	/** Test if string contains no character in set */
+	template<typename CharType>
+	static constexpr bool HasNone(const CharType* Str, AsciiSet Set)
+	{
+		return *FindFirstOrEnd(Str, Set) == '\0';
+	}
+
+	/** Test if string contains any character outside of set */
+	template<typename CharType>
+	static constexpr bool HasOnly(const CharType* Str, AsciiSet Set)
+	{
+		return *Skip(Str, Set) == '\0';
+	}
+
+	////////// Algorithms for string types like FStringView and FString //////////
+
+	/** Get initial substring with all characters in set */
+	template<class StringType>
+	static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::Members, EKeep::Head>(Str, Set);
+	}
+
+	/** Get initial substring with no characters in set */
+	template<class StringType>
+	static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Head>(Str, Set);
+	}
+
+	/** Trim initial characters in set */
+	template<class StringType>
+	static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::Members, EKeep::Tail>(Str, Set);
+	}
+
+	/** Trim initial characters not in set */
+	template<class StringType>
+	static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+	}
+
+	/** Get trailing substring with all characters in set */
+	template<class StringType>
+	static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::Members, EKeep::Tail>(Str, Set);
+	}
+
+	/** Get trailing substring with no characters in set */
+	template<class StringType>
+	static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+	}
+
+	/** Trim trailing characters in set */
+	template<class StringType>
+	static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::Members, EKeep::Head>(Str, Set);
+	}
+
+	/** Trim trailing characters not in set */
+	template<class StringType>
+	static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Head>(Str, Set);
+	}
+
+	/** Test if string contains any character in set */
+	template<class StringType>
+	static constexpr bool HasAny(const StringType& Str, AsciiSet Set)
+	{
+		return !HasNone(Str, Set);
+	}
+
+	/** Test if string contains no character in set */
+	template<class StringType>
+	static constexpr bool HasNone(const StringType& Str, AsciiSet Set)
+	{
+		uint64_t Match = 0;
+		for (auto Char : Str)
+		{
+			Match |= Set.Test(Char);
+		}
+		return Match == 0;
+	}
+
+	/** Test if string contains any character outside of set */
+	template<class StringType>
+	static constexpr bool HasOnly(const StringType& Str, AsciiSet Set)
+	{
+		auto End = Str.data() + Str.size();
+		return FindFirst<EInclude::Members>(Set, GetData(Str), End) == End;
+	}
+
+private:
+	enum class EDir
+	{
+		Forward,
+		Reverse
+	};
+	enum class EInclude
+	{
+		Members,
+		NonMembers
+	};
+	enum class EKeep
+	{
+		Head,
+		Tail
+	};
+
+	template<EInclude Include, typename CharType>
+	static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End)
+	{
+		for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It)
+			;
+		return It;
+	}
+
+	template<EInclude Include, typename CharType>
+	static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End)
+	{
+		for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It)
+			;
+		return It;
+	}
+
+	template<EDir Dir, EInclude Include, EKeep Keep, class StringType>
+	static constexpr StringType Scan(const StringType& Str, AsciiSet Set)
+	{
+		auto Begin = Str.data();
+		auto End   = Begin + Str.size();
+		auto It	   = Dir == EDir::Forward ? FindFirst<Include>(Set, Begin, End) : FindLast<Include>(Set, End - 1, Begin - 1) + 1;
+
+		return Keep == EKeep::Head ? StringType(Begin, static_cast<int32_t>(It - Begin)) : StringType(It, static_cast<int32_t>(End - It));
+	}
+
+	// Work-around for constexpr limitations
+	struct InitData
+	{
+		uint64_t Lo, Hi;
+	};
+	static constexpr uint64_t NilMask = uint64_t(1) << '\0';
+
+	static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char)
+	{
+		uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+		uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+		uint64_t Bit  = uint64_t(1) << uint8_t(Char & 0x3f);
+
+		Bitset.Lo |= Bit & IsLo;
+		Bitset.Hi |= Bit & IsHi;
+	}
+
+	constexpr inline uint64_t TestImpl(uint32_t Char) const
+	{
+		uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+		uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+		uint64_t Bit  = uint64_t(1) << (Char & 0x3f);
+
+		return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask);
+	}
+
+	template<typename CharType, int N>
+	static constexpr InitData StringToBitset(const CharType (&Chars)[N])
+	{
+		using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+		InitData Bitset = {0, 0};
+		for (int I = 0; I < N - 1; ++I)
+		{
+			SetImpl(Bitset, UnsignedCharType(Chars[I]));
+		}
+
+		return Bitset;
+	}
+
+	constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {}
+
+	constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {}
+
+	uint64_t LoMask, HiMask;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
 void string_forcelink();  // internal
 
 }  // namespace zen
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
index f6093cb96..4b9c1af1b 100644
--- a/zencore/include/zencore/zencore.h
+++ b/zencore/include/zencore/zencore.h
@@ -102,9 +102,11 @@
 // Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable.
 #	define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name))
 #	define ZEN_FORCENOINLINE	   __declspec(noinline) /* Force code to NOT be inline */
+#	define LINE_TERMINATOR_ANSI   "\r\n"
 #else
 #	define ZEN_CODE_SECTION(Name)
 #	define ZEN_FORCENOINLINE
+#	define LINE_TERMINATOR_ANSI "\n"
 #endif
 
 #if ZEN_ARCH_ARM64