Merged latest from main

11 files changed, 827 insertions, 68 deletions

diff --git a/zencore/include/zencore/base64.h b/zencore/include/zencore/base64.h
new file mode 100644
index 000000000..4d78b085f
--- /dev/null
+++ b/zencore/include/zencore/base64.h
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+struct Base64
+{
+	template<typename CharType>
+	static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest);
+
+	static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; }
+};
+
+}  // namespace zen
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index 4fce129ea..ab01402f8 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -30,6 +30,7 @@ class CbArrayView;
 class BinaryReader;
 class BinaryWriter;
 class CompressedBuffer;
+class CbValue;
 
 class DateTime
 {
@@ -41,8 +42,22 @@ public:
 	}
 
 	inline uint64_t GetTicks() const { return Ticks; }
-	inline bool		operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
-	inline auto		operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+	int	 GetYear() const;
+	int	 GetMonth() const;
+	int	 GetDay() const;
+	int	 GetHour() const;
+	int	 GetHour12() const;
+	int	 GetMinute() const;
+	int	 GetSecond() const;
+	int	 GetMillisecond() const;
+	void GetDate(int& Year, int& Month, int& Day) const;
+
+	inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
+	inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+	std::string ToString(const char* Format) const;
+	std::string ToIso8601() const;
 
 private:
 	void	 Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond);
@@ -98,6 +113,25 @@ public:
 	/** The number of timespan ticks per year (365 days, not accounting for leap years). */
 	static constexpr int64_t TicksPerYear = 365 * TicksPerDay;
 
+	int GetFractionTicks() const { return (int)(Ticks % TicksPerSecond); }
+
+	int GetFractionMicro() const { return (int)((Ticks % TicksPerSecond) / TicksPerMicrosecond); }
+
+	int GetFractionMilli() const { return (int)((Ticks % TicksPerSecond) / TicksPerMillisecond); }
+
+	int GetFractionNano() const { return (int)((Ticks % TicksPerSecond) * NanosecondsPerTick); }
+
+	int GetDays() const { return (int)(Ticks / TicksPerDay); }
+
+	int GetHours() const { return (int)((Ticks / TicksPerHour) % 24); }
+
+	int GetMinutes() const { return (int)((Ticks / TicksPerMinute) % 60); }
+
+	int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); }
+
+	ZENCORE_API std::string ToString(const char* Format) const;
+	ZENCORE_API std::string ToString() const;
+
 private:
 	void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano);
 
@@ -107,6 +141,8 @@ private:
 struct Guid
 {
 	uint32_t A, B, C, D;
+
+	StringBuilderBase& ToString(StringBuilderBase& OutString) const;
 };
 
 //////////////////////////////////////////////////////////////////////////
@@ -318,6 +354,9 @@ public:
 	static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; }
 	static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; }
 
+	static constexpr inline bool IsCustomById(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomById; }
+	static constexpr inline bool IsCustomByName(CbFieldType Type) { return GetType(Type) == CbFieldType::CustomByName; }
+
 	static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; }
 	static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; }
 
@@ -365,6 +404,46 @@ public:
 	virtual void VisitTimeSpan(TimeSpan Value)				= 0;
 };
 
+/** A custom compact binary field type with an integer identifier. */
+struct CbCustomById
+{
+	/** An identifier for the sub-type of the field. */
+	uint64_t Id = 0;
+	/** A view of the value. Lifetime is tied to the field that the value is associated with. */
+	MemoryView Data;
+};
+
+/** A custom compact binary field type with a string identifier. */
+struct CbCustomByName
+{
+	/** An identifier for the sub-type of the field. Lifetime is tied to the field that the name is associated with. */
+	std::u8string_view Name;
+	/** A view of the value. Lifetime is tied to the field that the value is associated with. */
+	MemoryView Data;
+};
+
+namespace CompactBinaryPrivate {
+	/** Parameters for converting to an integer. */
+	struct IntegerParams
+	{
+		/** Whether the output type has a sign bit. */
+		uint32_t IsSigned : 1;
+		/** Bits of magnitude. (7 for int8) */
+		uint32_t MagnitudeBits : 31;
+	};
+
+	/** Make integer params for the given integer type. */
+	template<typename IntType>
+	static constexpr inline IntegerParams MakeIntegerParams()
+	{
+		IntegerParams Params;
+		Params.IsSigned		 = IntType(-1) < IntType(0);
+		Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
+		return Params;
+	}
+
+}  // namespace CompactBinaryPrivate
+
 /**
  * An atom of data in the compact binary format.
  *
@@ -393,13 +472,25 @@ public:
 
 	ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType);
 
+	/** Construct a field from a value, without access to the name. */
+	inline explicit CbFieldView(const CbValue& Value);
+
 	/** Returns the name of the field if it has a name, otherwise an empty view. */
 	constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); }
+	/** Returns the name of the field if it has a name, otherwise an empty view. */
+	constexpr inline std::u8string_view GetU8Name() const
+	{
+		return std::u8string_view(static_cast<const char8_t*>(Payload) - NameLen, NameLen);
+	}
+
+	/** Returns the value for unchecked access. Prefer the typed accessors below. */
+	inline CbValue GetValue() const;
 
 	ZENCORE_API MemoryView	 AsBinaryView(MemoryView Default = MemoryView());
 	ZENCORE_API CbObjectView AsObjectView();
 	ZENCORE_API CbArrayView	 AsArrayView();
 	ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view());
+	ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view());
 
 	ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const;
 
@@ -448,6 +539,11 @@ public:
 	/** Access the field as a OID. Returns the provided default on error. */
 	ZENCORE_API Oid AsObjectId(const Oid& Default);
 
+	/** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */
+	ZENCORE_API CbCustomById AsCustomById(CbCustomById Default);
+	/** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */
+	ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default);
+
 	/** Access the field as a date/time tick count. Returns the provided default on error. */
 	ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0);
 
@@ -590,25 +686,6 @@ protected:
 	}
 
 private:
-	/** Parameters for converting to an integer. */
-	struct IntegerParams
-	{
-		/** Whether the output type has a sign bit. */
-		uint32_t IsSigned : 1;
-		/** Bits of magnitude. (7 for int8) */
-		uint32_t MagnitudeBits : 31;
-	};
-
-	/** Make integer params for the given integer type. */
-	template<typename IntType>
-	static constexpr inline IntegerParams MakeIntegerParams()
-	{
-		IntegerParams Params;
-		Params.IsSigned		 = IntType(-1) < IntType(0);
-		Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
-		return Params;
-	}
-
 	/**
 	 * Access the field as the given integer type.
 	 *
@@ -617,11 +694,12 @@ private:
 	template<typename IntType>
 	inline IntType AsInteger(IntType Default)
 	{
-		return IntType(AsInteger(uint64_t(Default), MakeIntegerParams<IntType>()));
+		return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams<IntType>()));
 	}
 
-	ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params);
+	ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params);
 
+private:
 	/** The field type, with the transient HasFieldType flag if the field contains its type. */
 	CbFieldType Type = CbFieldType::None;
 	/** The error (if any) that occurred on the last field access. */
@@ -869,6 +947,11 @@ private:
 	inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {}
 };
 
+/**
+ * Serialize a compact binary object to JSON.
+ */
+ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder);
+
 class CbObjectView : protected CbFieldView
 {
 	friend class CbFieldView;
@@ -951,6 +1034,12 @@ public:
 	/** Whether the field has a value. */
 	using CbFieldView::operator bool;
 
+	StringBuilderBase& ToJson(StringBuilderBase& Builder) const
+	{
+		CompactBinaryToJson(*this, Builder);
+		return Builder;
+	}
+
 private:
 	friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); }
 	friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); }
diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h
new file mode 100644
index 000000000..5795ef957
--- /dev/null
+++ b/zencore/include/zencore/compactbinaryvalue.h
@@ -0,0 +1,290 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zencore/endian.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/memory.h>
+
+namespace zen {
+
+namespace CompactBinaryPrivate {
+
+	template<typename T>
+	static constexpr inline T ReadUnaligned(const void* const Memory)
+	{
+#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
+		return *static_cast<const T*>(Memory);
+#else
+		T Value;
+		memcpy(&Value, Memory, sizeof(Value));
+		return Value;
+#endif
+	}
+}  // namespace CompactBinaryPrivate
+/**
+ * A type that provides unchecked access to compact binary values.
+ *
+ * The main purpose of the type is to efficiently switch on field type. For every other use case,
+ * prefer to use the field, array, and object types directly. The accessors here do not check the
+ * type before reading the value, which means they can read out of bounds even on a valid compact
+ * binary value if the wrong accessor is used.
+ */
+class CbValue
+{
+public:
+	CbValue(CbFieldType Type, const void* Value);
+
+	CbObjectView AsObjectView() const;
+	CbArrayView	 AsArrayView() const;
+
+	MemoryView AsBinary() const;
+
+	/** Access as a string. Checks for range errors and uses the default if OutError is not null. */
+	std::string_view AsString(CbFieldError* OutError = nullptr, std::string_view Default = std::string_view()) const;
+
+	/** Access as a string as UTF8. Checks for range errors and uses the default if OutError is not null. */
+	std::u8string_view AsU8String(CbFieldError* OutError = nullptr, std::u8string_view Default = std::u8string_view()) const;
+
+	/**
+	 * Access as an integer, with both positive and negative values returned as unsigned.
+	 *
+	 * Checks for range errors and uses the default if OutError is not null.
+	 */
+	uint64_t AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError = nullptr, uint64_t Default = 0) const;
+
+	uint64_t AsIntegerPositive() const;
+	int64_t	 AsIntegerNegative() const;
+
+	float  AsFloat32() const;
+	double AsFloat64() const;
+
+	bool AsBool() const;
+
+	inline IoHash AsObjectAttachment() const { return AsHash(); }
+	inline IoHash AsBinaryAttachment() const { return AsHash(); }
+	inline IoHash AsAttachment() const { return AsHash(); }
+
+	IoHash AsHash() const;
+	Guid   AsUuid() const;
+
+	int64_t AsDateTimeTicks() const;
+	int64_t AsTimeSpanTicks() const;
+
+	Oid AsObjectId() const;
+
+	CbCustomById   AsCustomById() const;
+	CbCustomByName AsCustomByName() const;
+
+	inline CbFieldType GetType() const { return Type; }
+	inline const void* GetData() const { return Data; }
+
+private:
+	const void* Data;
+	CbFieldType Type;
+};
+
+inline CbFieldView::CbFieldView(const CbValue& InValue) : Type(InValue.GetType()), Payload(InValue.GetData())
+{
+}
+
+inline CbValue
+CbFieldView::GetValue() const
+{
+	return CbValue(CbFieldTypeOps::GetType(Type), Payload);
+}
+
+inline CbValue::CbValue(CbFieldType InType, const void* InValue) : Data(InValue), Type(InType)
+{
+}
+
+inline CbObjectView
+CbValue::AsObjectView() const
+{
+	return CbObjectView(*this);
+}
+
+inline CbArrayView
+CbValue::AsArrayView() const
+{
+	return CbArrayView(*this);
+}
+
+inline MemoryView
+CbValue::AsBinary() const
+{
+	const uint8_t* const Bytes = static_cast<const uint8_t*>(Data);
+	uint32_t			 ValueSizeByteCount;
+	const uint64_t		 ValueSize = ReadVarUInt(Bytes, ValueSizeByteCount);
+	return MakeMemoryView(Bytes + ValueSizeByteCount, ValueSize);
+}
+
+inline std::string_view
+CbValue::AsString(CbFieldError* OutError, std::string_view Default) const
+{
+	const char* const Chars = static_cast<const char*>(Data);
+	uint32_t		  ValueSizeByteCount;
+	const uint64_t	  ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
+
+	if (OutError)
+	{
+		if (ValueSize >= (uint64_t(1) << 31))
+		{
+			*OutError = CbFieldError::RangeError;
+			return Default;
+		}
+		*OutError = CbFieldError::None;
+	}
+
+	return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+}
+
+inline std::u8string_view
+CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const
+{
+	const char8_t* const Chars = static_cast<const char8_t*>(Data);
+	uint32_t			 ValueSizeByteCount;
+	const uint64_t		 ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
+
+	if (OutError)
+	{
+		if (ValueSize >= (uint64_t(1) << 31))
+		{
+			*OutError = CbFieldError::RangeError;
+			return Default;
+		}
+		*OutError = CbFieldError::None;
+	}
+
+	return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+}
+
+inline uint64_t
+CbValue::AsInteger(CompactBinaryPrivate::IntegerParams Params, CbFieldError* OutError, uint64_t Default) const
+{
+	// A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero.
+	const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1);
+	const uint64_t IsNegative	  = uint8_t(Type) & 1;
+
+	uint32_t	   MagnitudeByteCount;
+	const uint64_t Magnitude = ReadVarUInt(Data, MagnitudeByteCount);
+	const uint64_t Value	 = Magnitude ^ -int64_t(IsNegative);
+
+	if (OutError)
+	{
+		const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned);
+		*OutError				 = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
+
+		const uint64_t UseValueMask = -int64_t(IsInRange);
+		return (Value & UseValueMask) | (Default & ~UseValueMask);
+	}
+
+	return Value;
+}
+
+inline uint64_t
+CbValue::AsIntegerPositive() const
+{
+	uint32_t MagnitudeByteCount;
+	return ReadVarUInt(Data, MagnitudeByteCount);
+}
+
+inline int64_t
+CbValue::AsIntegerNegative() const
+{
+	uint32_t MagnitudeByteCount;
+	return int64_t(ReadVarUInt(Data, MagnitudeByteCount)) ^ -int64_t(1);
+}
+
+inline float
+CbValue::AsFloat32() const
+{
+	const uint32_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint32_t>(Data));
+	return reinterpret_cast<const float&>(Value);
+}
+
+inline double
+CbValue::AsFloat64() const
+{
+	const uint64_t Value = FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<uint64_t>(Data));
+	return reinterpret_cast<const double&>(Value);
+}
+
+inline bool
+CbValue::AsBool() const
+{
+	return uint8_t(Type) & 1;
+}
+
+inline IoHash
+CbValue::AsHash() const
+{
+	return IoHash::MakeFrom(Data);
+}
+
+inline Guid
+CbValue::AsUuid() const
+{
+	Guid Value;
+	memcpy(&Value, Data, sizeof(Guid));
+	Value.A = FromNetworkOrder(Value.A);
+	Value.B = FromNetworkOrder(Value.B);
+	Value.C = FromNetworkOrder(Value.C);
+	Value.D = FromNetworkOrder(Value.D);
+	return Value;
+}
+
+inline int64_t
+CbValue::AsDateTimeTicks() const
+{
+	return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data));
+}
+
+inline int64_t
+CbValue::AsTimeSpanTicks() const
+{
+	return FromNetworkOrder(CompactBinaryPrivate::ReadUnaligned<int64_t>(Data));
+}
+
+inline Oid
+CbValue::AsObjectId() const
+{
+	return Oid::FromMemory(Data);
+}
+
+inline CbCustomById
+CbValue::AsCustomById() const
+{
+	const uint8_t* Bytes = static_cast<const uint8_t*>(Data);
+	uint32_t	   DataSizeByteCount;
+	const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount);
+	Bytes += DataSizeByteCount;
+
+	CbCustomById Value;
+	uint32_t	 TypeIdByteCount;
+	Value.Id   = ReadVarUInt(Bytes, TypeIdByteCount);
+	Value.Data = MakeMemoryView(Bytes + TypeIdByteCount, DataSize - TypeIdByteCount);
+	return Value;
+}
+
+inline CbCustomByName
+CbValue::AsCustomByName() const
+{
+	const uint8_t* Bytes = static_cast<const uint8_t*>(Data);
+	uint32_t	   DataSizeByteCount;
+	const uint64_t DataSize = ReadVarUInt(Bytes, DataSizeByteCount);
+	Bytes += DataSizeByteCount;
+
+	uint32_t	   TypeNameLenByteCount;
+	const uint64_t TypeNameLen = ReadVarUInt(Bytes, TypeNameLenByteCount);
+	Bytes += TypeNameLenByteCount;
+
+	CbCustomByName Value;
+	Value.Name = std::u8string_view(reinterpret_cast<const char8_t*>(Bytes), static_cast<std::u8string_view::size_type>(TypeNameLen));
+	Value.Data = MakeMemoryView(Bytes + TypeNameLen, DataSize - TypeNameLen - TypeNameLenByteCount);
+	return Value;
+}
+
+}  // namespace zen
diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h
index 1dc6209d6..e0e4aaae0 100644
--- a/zencore/include/zencore/except.h
+++ b/zencore/include/zencore/except.h
@@ -27,7 +27,7 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const st
 ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message);
 
 ZENCORE_API std::string GetLastErrorAsString();
-ZENCORE_API std::string GetErrorAsString(uint32_t ErrorCode);
+ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode);
 
 inline int32_t
 GetLastError()
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index 66ab37e5c..6678528f6 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -14,19 +14,16 @@ class IoBuffer;
 
 /** Delete directory (after deleting any contents)
  */
-ZENCORE_API bool DeleteDirectories(const wchar_t* dir);
 ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir);
 
 /** Ensure directory exists.
 
 	Will also create any required parent directories
  */
-ZENCORE_API bool CreateDirectories(const wchar_t* dir);
 ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
 
 /** Ensure directory exists and delete contents (if any) before returning
  */
-ZENCORE_API bool CleanDirectory(const wchar_t* dir);
 ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
 
 /** Map native file handle to a path
@@ -82,6 +79,6 @@ public:
 
 //////////////////////////////////////////////////////////////////////////
 
-void filesystem_forcelink(); // internal
+void filesystem_forcelink();  // internal
 
 }  // namespace zen
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 298952dd6..6f3609d19 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -11,6 +11,7 @@
 
 namespace zen {
 
+struct IoHash;
 struct IoBufferExtendedCore;
 
 enum class ZenContentType : uint8_t
@@ -352,6 +353,8 @@ public:
 	inline static IoBuffer		MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
 };
 
+IoHash HashBuffer(IoBuffer& Buffer);
+
 void iobuffer_forcelink();
 
 }  // namespace zen
diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h
index 221f5f358..0838cfe80 100644
--- a/zencore/include/zencore/logging.h
+++ b/zencore/include/zencore/logging.h
@@ -39,46 +39,46 @@ using zen::Log;
 
 // Helper macros for logging
 
-#define ZEN_TRACE(fmtstr, ...)                 \
-	do                                         \
-	{                                          \
-		using namespace std::literals;         \
-		Log().trace(fmtstr##sv, ##__VA_ARGS__);\
+#define ZEN_TRACE(fmtstr, ...)                  \
+	do                                          \
+	{                                           \
+		using namespace std::literals;          \
+		Log().trace(fmtstr##sv, ##__VA_ARGS__); \
 	} while (false)
 
-#define ZEN_DEBUG(fmtstr, ...)                 \
+#define ZEN_DEBUG(fmtstr, ...)                  \
+	do                                          \
+	{                                           \
+		using namespace std::literals;          \
+		Log().debug(fmtstr##sv, ##__VA_ARGS__); \
+	} while (false)
+
+#define ZEN_INFO(fmtstr, ...)                  \
 	do                                         \
 	{                                          \
 		using namespace std::literals;         \
-		Log().debug(fmtstr##sv, ##__VA_ARGS__);\
+		Log().info(fmtstr##sv, ##__VA_ARGS__); \
 	} while (false)
 
-#define ZEN_INFO(fmtstr, ...)                 \
-	do                                        \
-	{                                         \
-		using namespace std::literals;        \
-		Log().info(fmtstr##sv, ##__VA_ARGS__);\
-	} while (false)
-
-#define ZEN_WARN(fmtstr, ...)                 \
-	do                                        \
-	{                                         \
-		using namespace std::literals;        \
-		Log().warn(fmtstr##sv, ##__VA_ARGS__);\
-	} while (false)
-
-#define ZEN_ERROR(fmtstr, ...)                 \
+#define ZEN_WARN(fmtstr, ...)                  \
 	do                                         \
 	{                                          \
 		using namespace std::literals;         \
-		Log().error(fmtstr##sv, ##__VA_ARGS__);\
+		Log().warn(fmtstr##sv, ##__VA_ARGS__); \
+	} while (false)
+
+#define ZEN_ERROR(fmtstr, ...)                  \
+	do                                          \
+	{                                           \
+		using namespace std::literals;          \
+		Log().error(fmtstr##sv, ##__VA_ARGS__); \
 	} while (false)
 
-#define ZEN_CRITICAL(fmtstr, ...)                 \
-	do                                            \
-	{                                             \
-		using namespace std::literals;            \
-		Log().critical(fmtstr##sv, ##__VA_ARGS__);\
+#define ZEN_CRITICAL(fmtstr, ...)                  \
+	do                                             \
+	{                                              \
+		using namespace std::literals;             \
+		Log().critical(fmtstr##sv, ##__VA_ARGS__); \
 	} while (false)
 
 #define ZEN_CONSOLE(fmtstr, ...)                    \
diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h
index 3d4db1081..aba391c85 100644
--- a/zencore/include/zencore/memory.h
+++ b/zencore/include/zencore/memory.h
@@ -354,6 +354,12 @@ MakeMemoryView(const void* Data, const void* DataEnd)
 	return MemoryView(Data, DataEnd);
 }
 
+[[nodiscard]] inline MemoryView
+MakeMemoryView(const void* Data, uint64_t Size)
+{
+	return MemoryView(Data, reinterpret_cast<const uint8_t*>(Data) + Size);
+}
+
 /**
  * Make a non-owning mutable view of the memory of the initializer list.
  *
diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h
index 7290fd914..0554f620d 100644
--- a/zencore/include/zencore/stats.h
+++ b/zencore/include/zencore/stats.h
@@ -12,30 +12,44 @@ template<typename T>
 class Gauge
 {
 public:
-	Gauge() : m_value{0} {}
+	Gauge() : m_Value{0} {}
+
+	T	 Value() const { return m_Value; }
+	void SetValue(T Value) { m_Value = Value; }
 
 private:
-	T m_value;
+	std::atomic<T> m_Value;
 };
 
+/** Stats counter
+ *
+ * A counter is modified by adding or subtracting a value from a current value.
+ * This would typically be used to track number of requests in flight, number
+ * of active jobs etc
+ *
+ */
 class Counter
 {
 public:
 	inline void		SetValue(uint64_t Value) { m_count = Value; }
 	inline uint64_t Value() const { return m_count; }
 
-	inline void Increment(int64_t AddValue) { m_count += AddValue; }
-	inline void Decrement(int64_t SubValue) { m_count -= SubValue; }
-	inline void Clear() { m_count = 0; }
+	inline void Increment(int64_t AddValue) { m_count.fetch_add(AddValue); }
+	inline void Decrement(int64_t SubValue) { m_count.fetch_sub(SubValue); }
+	inline void Clear() { m_count.store(0, std::memory_order_release); }
 
 private:
-	std::atomic_uint64_t m_count{0};
+	std::atomic<uint64_t> m_count{0};
 };
 
-/// <summary>
-/// Exponential Weighted Moving Average
-/// </summary>
-class EWMA
+/** Exponential Weighted Moving Average
+
+	This is very raw, to use as little state as possible. If we
+	want to use this more broadly in user code we should perhaps
+	add a more user-friendly wrapper
+ */
+
+class RawEWMA
 {
 public:
 	/// <summary>
@@ -49,16 +63,38 @@ public:
 	double Rate() const;
 
 private:
-	double m_rate = 0;
+	std::atomic<double> m_Rate = 0;
 };
 
 /// <summary>
-/// Tracks rate of events over time (i.e requests/sec)
+/// Tracks rate of events over time (i.e requests/sec), using
+/// exponential moving averages
 /// </summary>
 class Meter
 {
 public:
+	Meter();
+	~Meter();
+
+	double Rate1();					  // One-minute rate
+	double Rate5();					  // Five-minute rate
+	double Rate15();				  // Fifteen-minute rate
+	double MeanRate();				  // Mean rate since instantiation of this meter
+	void   Mark(uint64_t Count = 1);  // Register one or more events
+
 private:
+	std::atomic<uint64_t> m_TotalCount{0};	  // Accumulator counting number of marks since beginning
+	std::atomic<uint64_t> m_PendingCount{0};  // Pending EWMA update accumulator
+	std::atomic<uint64_t> m_StartTick{0};	  // Time this was instantiated (for mean)
+	std::atomic<uint64_t> m_LastTick{0};	  // Timestamp of last EWMA tick
+	std::atomic<int64_t>  m_Remain{0};		  // Tracks the "modulo" of tick time
+	bool				  m_IsFirstTick = true;
+	RawEWMA				  m_RateM1;
+	RawEWMA				  m_RateM5;
+	RawEWMA				  m_RateM15;
+
+	void TickIfNecessary();
+	void Tick();
 };
 
 extern void stats_forcelink();
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index bb9b1c896..2c0d10577 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -14,6 +14,8 @@
 #include <span>
 #include <string_view>
 
+#include <type_traits>
+
 namespace zen {
 
 //////////////////////////////////////////////////////////////////////////
@@ -94,6 +96,14 @@ public:
 	const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete;
 	const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete;
 
+	inline size_t AddUninitialized(size_t Count)
+	{
+		EnsureCapacity(Count);
+		const size_t OldCount = Size();
+		m_CurPos += Count;
+		return OldCount;
+	}
+
 	StringBuilderImpl& Append(C OneChar)
 	{
 		EnsureCapacity(1);
@@ -209,6 +219,12 @@ public:
 		return AppendRange(String.data(), String.data() + String.size());
 	}
 
+	inline void RemoveSuffix(int32_t Count)
+	{
+		ZEN_ASSERT(Count <= Size());
+		m_CurPos -= Count;
+	}
+
 	inline const C* c_str() const
 	{
 		EnsureNulTerminated();
@@ -322,6 +338,12 @@ protected:
 
 extern template class StringBuilderImpl<char>;
 
+inline StringBuilderImpl<char>&
+operator<<(StringBuilderImpl<char>& Builder, char Char)
+{
+	return Builder.Append(Char);
+}
+
 class StringBuilderBase : public StringBuilderImpl<char>
 {
 public:
@@ -661,6 +683,303 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 
 //////////////////////////////////////////////////////////////////////////
 
+/**
+ * ASCII character bitset useful for fast and readable parsing
+ *
+ * Entirely constexpr. Works with both wide and narrow strings.
+ *
+ * Example use cases:
+ *
+ *   constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n");
+ *   bool bIsWhitespace = WhitespaceCharacters.Test(MyChar);
+ *   const char* HelloWorld = AsciiSet::Skip("  \t\tHello world!", WhitespaceCharacters);
+ *
+ *   constexpr AsciiSet XmlEscapeChars("&<>\"'");
+ *   check(AsciiSet::HasNone(EscapedXmlString, XmlEscapeChars));
+ *
+ *   constexpr AsciiSet Delimiters(".:;");
+ *   const TCHAR* DelimiterOrEnd = AsciiSet::FindFirstOrEnd(PrefixedName, Delimiters);
+ *   FString Prefix(PrefixedName, DelimiterOrEnd - PrefixedName);
+ *
+ *   constexpr AsciiSet Slashes("/\\");
+ *   const TCHAR* SlashOrEnd = AsciiSet::FindLastOrEnd(PathName, Slashes);
+ *   const TCHAR* FileName = *SlashOrEnd ? SlashOrEnd + 1 : PathName;
+ */
+class AsciiSet
+{
+public:
+	template<typename CharType, int N>
+	constexpr AsciiSet(const CharType (&Chars)[N]) : AsciiSet(StringToBitset(Chars))
+	{
+	}
+
+	/** Returns true if a character is part of the set */
+	template<typename CharType>
+	constexpr inline bool Contains(CharType Char) const
+	{
+		using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+		return !!TestImpl((UnsignedCharType)Char);
+	}
+
+	/** Returns non-zero if a character is part of the set. Prefer Contains() to avoid VS2019 conversion warnings. */
+	template<typename CharType>
+	constexpr inline uint64_t Test(CharType Char) const
+	{
+		using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+		return TestImpl((UnsignedCharType)Char);
+	}
+
+	/** Create new set with specified character in it */
+	constexpr inline AsciiSet operator+(char Char) const
+	{
+		using UnsignedCharType = std::make_unsigned<char>::type;
+
+		InitData Bitset = {LoMask, HiMask};
+		SetImpl(Bitset, (UnsignedCharType)Char);
+		return AsciiSet(Bitset);
+	}
+
+	/** Create new set containing inverse set of characters - likely including null-terminator */
+	constexpr inline AsciiSet operator~() const { return AsciiSet(~LoMask, ~HiMask); }
+
+	////////// Algorithms for C strings //////////
+
+	/** Find first character of string inside set or end pointer. Never returns null. */
+	template<class CharType>
+	static constexpr const CharType* FindFirstOrEnd(const CharType* Str, AsciiSet Set)
+	{
+		for (AsciiSet SetOrNil(Set.LoMask | NilMask, Set.HiMask); !SetOrNil.Test(*Str); ++Str)
+			;
+
+		return Str;
+	}
+
+	/** Find last character of string inside set or end pointer. Never returns null. */
+	template<class CharType>
+	static constexpr const CharType* FindLastOrEnd(const CharType* Str, AsciiSet Set)
+	{
+		const CharType* Last = FindFirstOrEnd(Str, Set);
+
+		for (const CharType* It = Last; *It; It = FindFirstOrEnd(It + 1, Set))
+		{
+			Last = It;
+		}
+
+		return Last;
+	}
+
+	/** Find first character of string outside of set. Never returns null. */
+	template<typename CharType>
+	static constexpr const CharType* Skip(const CharType* Str, AsciiSet Set)
+	{
+		while (Set.Contains(*Str))
+		{
+			++Str;
+		}
+
+		return Str;
+	}
+
+	/** Test if string contains any character in set */
+	template<typename CharType>
+	static constexpr bool HasAny(const CharType* Str, AsciiSet Set)
+	{
+		return *FindFirstOrEnd(Str, Set) != '\0';
+	}
+
+	/** Test if string contains no character in set */
+	template<typename CharType>
+	static constexpr bool HasNone(const CharType* Str, AsciiSet Set)
+	{
+		return *FindFirstOrEnd(Str, Set) == '\0';
+	}
+
+	/** Test if string contains any character outside of set */
+	template<typename CharType>
+	static constexpr bool HasOnly(const CharType* Str, AsciiSet Set)
+	{
+		return *Skip(Str, Set) == '\0';
+	}
+
+	////////// Algorithms for string types like FStringView and FString //////////
+
+	/** Get initial substring with all characters in set */
+	template<class StringType>
+	static constexpr StringType FindPrefixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::Members, EKeep::Head>(Str, Set);
+	}
+
+	/** Get initial substring with no characters in set */
+	template<class StringType>
+	static constexpr StringType FindPrefixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Head>(Str, Set);
+	}
+
+	/** Trim initial characters in set */
+	template<class StringType>
+	static constexpr StringType TrimPrefixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::Members, EKeep::Tail>(Str, Set);
+	}
+
+	/** Trim initial characters not in set */
+	template<class StringType>
+	static constexpr StringType TrimPrefixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Forward, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+	}
+
+	/** Get trailing substring with all characters in set */
+	template<class StringType>
+	static constexpr StringType FindSuffixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::Members, EKeep::Tail>(Str, Set);
+	}
+
+	/** Get trailing substring with no characters in set */
+	template<class StringType>
+	static constexpr StringType FindSuffixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Tail>(Str, Set);
+	}
+
+	/** Trim trailing characters in set */
+	template<class StringType>
+	static constexpr StringType TrimSuffixWith(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::Members, EKeep::Head>(Str, Set);
+	}
+
+	/** Trim trailing characters not in set */
+	template<class StringType>
+	static constexpr StringType TrimSuffixWithout(const StringType& Str, AsciiSet Set)
+	{
+		return Scan<EDir::Reverse, EInclude::NonMembers, EKeep::Head>(Str, Set);
+	}
+
+	/** Test if string contains any character in set */
+	template<class StringType>
+	static constexpr bool HasAny(const StringType& Str, AsciiSet Set)
+	{
+		return !HasNone(Str, Set);
+	}
+
+	/** Test if string contains no character in set */
+	template<class StringType>
+	static constexpr bool HasNone(const StringType& Str, AsciiSet Set)
+	{
+		uint64_t Match = 0;
+		for (auto Char : Str)
+		{
+			Match |= Set.Test(Char);
+		}
+		return Match == 0;
+	}
+
+	/** Test if string contains any character outside of set */
+	template<class StringType>
+	static constexpr bool HasOnly(const StringType& Str, AsciiSet Set)
+	{
+		auto End = Str.data() + Str.size();
+		return FindFirst<EInclude::Members>(Set, GetData(Str), End) == End;
+	}
+
+private:
+	enum class EDir
+	{
+		Forward,
+		Reverse
+	};
+	enum class EInclude
+	{
+		Members,
+		NonMembers
+	};
+	enum class EKeep
+	{
+		Head,
+		Tail
+	};
+
+	template<EInclude Include, typename CharType>
+	static constexpr const CharType* FindFirst(AsciiSet Set, const CharType* It, const CharType* End)
+	{
+		for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); ++It)
+			;
+		return It;
+	}
+
+	template<EInclude Include, typename CharType>
+	static constexpr const CharType* FindLast(AsciiSet Set, const CharType* It, const CharType* End)
+	{
+		for (; It != End && (Include == EInclude::Members) == !!Set.Test(*It); --It)
+			;
+		return It;
+	}
+
+	template<EDir Dir, EInclude Include, EKeep Keep, class StringType>
+	static constexpr StringType Scan(const StringType& Str, AsciiSet Set)
+	{
+		auto Begin = Str.data();
+		auto End   = Begin + Str.size();
+		auto It	   = Dir == EDir::Forward ? FindFirst<Include>(Set, Begin, End) : FindLast<Include>(Set, End - 1, Begin - 1) + 1;
+
+		return Keep == EKeep::Head ? StringType(Begin, static_cast<int32_t>(It - Begin)) : StringType(It, static_cast<int32_t>(End - It));
+	}
+
+	// Work-around for constexpr limitations
+	struct InitData
+	{
+		uint64_t Lo, Hi;
+	};
+	static constexpr uint64_t NilMask = uint64_t(1) << '\0';
+
+	static constexpr inline void SetImpl(InitData& Bitset, uint32_t Char)
+	{
+		uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+		uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+		uint64_t Bit  = uint64_t(1) << uint8_t(Char & 0x3f);
+
+		Bitset.Lo |= Bit & IsLo;
+		Bitset.Hi |= Bit & IsHi;
+	}
+
+	constexpr inline uint64_t TestImpl(uint32_t Char) const
+	{
+		uint64_t IsLo = uint64_t(0) - (Char >> 6 == 0);
+		uint64_t IsHi = uint64_t(0) - (Char >> 6 == 1);
+		uint64_t Bit  = uint64_t(1) << (Char & 0x3f);
+
+		return (Bit & IsLo & LoMask) | (Bit & IsHi & HiMask);
+	}
+
+	template<typename CharType, int N>
+	static constexpr InitData StringToBitset(const CharType (&Chars)[N])
+	{
+		using UnsignedCharType = std::make_unsigned<CharType>::type;
+
+		InitData Bitset = {0, 0};
+		for (int I = 0; I < N - 1; ++I)
+		{
+			SetImpl(Bitset, UnsignedCharType(Chars[I]));
+		}
+
+		return Bitset;
+	}
+
+	constexpr AsciiSet(InitData Bitset) : LoMask(Bitset.Lo), HiMask(Bitset.Hi) {}
+
+	constexpr AsciiSet(uint64_t Lo, uint64_t Hi) : LoMask(Lo), HiMask(Hi) {}
+
+	uint64_t LoMask, HiMask;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
 void string_forcelink();  // internal
 
 }  // namespace zen
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
index f6093cb96..4b9c1af1b 100644
--- a/zencore/include/zencore/zencore.h
+++ b/zencore/include/zencore/zencore.h
@@ -102,9 +102,11 @@
 // Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable.
 #	define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name))
 #	define ZEN_FORCENOINLINE	   __declspec(noinline) /* Force code to NOT be inline */
+#	define LINE_TERMINATOR_ANSI   "\r\n"
 #else
 #	define ZEN_CODE_SECTION(Name)
 #	define ZEN_FORCENOINLINE
+#	define LINE_TERMINATOR_ANSI "\n"
 #endif
 
 #if ZEN_ARCH_ARM64