aboutsummaryrefslogtreecommitdiff
path: root/zencore
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2021-05-11 13:05:39 +0200
committerStefan Boberg <[email protected]>2021-05-11 13:05:39 +0200
commitf8d9ac5d13dd37b8b57af0478e77ba1e75c813aa (patch)
tree1daf7621e110d48acd5e12e3073ce48ef0dd11b2 /zencore
downloadzen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.tar.xz
zen-f8d9ac5d13dd37b8b57af0478e77ba1e75c813aa.zip
Adding zenservice code
Diffstat (limited to 'zencore')
-rw-r--r--zencore/blake3.cpp153
-rw-r--r--zencore/compactbinary.cpp1279
-rw-r--r--zencore/compactbinarybuilder.cpp1530
-rw-r--r--zencore/compactbinarypackage.cpp945
-rw-r--r--zencore/compactbinaryvalidation.cpp607
-rw-r--r--zencore/compress.cpp11
-rw-r--r--zencore/except.cpp17
-rw-r--r--zencore/filesystem.cpp592
-rw-r--r--zencore/httpclient.cpp23
-rw-r--r--zencore/httpserver.cpp1459
-rw-r--r--zencore/include/zencore/atomic.h43
-rw-r--r--zencore/include/zencore/blake3.h57
-rw-r--r--zencore/include/zencore/compactbinary.h1335
-rw-r--r--zencore/include/zencore/compactbinarybuilder.h633
-rw-r--r--zencore/include/zencore/compactbinarypackage.h305
-rw-r--r--zencore/include/zencore/compactbinaryvalidation.h192
-rw-r--r--zencore/include/zencore/compress.h53
-rw-r--r--zencore/include/zencore/endian.h61
-rw-r--r--zencore/include/zencore/enumflags.h61
-rw-r--r--zencore/include/zencore/except.h60
-rw-r--r--zencore/include/zencore/filesystem.h74
-rw-r--r--zencore/include/zencore/fmtutils.h49
-rw-r--r--zencore/include/zencore/httpclient.h18
-rw-r--r--zencore/include/zencore/httpserver.h373
-rw-r--r--zencore/include/zencore/intmath.h140
-rw-r--r--zencore/include/zencore/iobuffer.h272
-rw-r--r--zencore/include/zencore/iohash.h95
-rw-r--r--zencore/include/zencore/md5.h50
-rw-r--r--zencore/include/zencore/memory.h213
-rw-r--r--zencore/include/zencore/meta.h30
-rw-r--r--zencore/include/zencore/refcount.h144
-rw-r--r--zencore/include/zencore/scopeguard.h33
-rw-r--r--zencore/include/zencore/sha1.h76
-rw-r--r--zencore/include/zencore/sharedbuffer.h169
-rw-r--r--zencore/include/zencore/snapshot_manifest.h57
-rw-r--r--zencore/include/zencore/stats.h66
-rw-r--r--zencore/include/zencore/stream.h318
-rw-r--r--zencore/include/zencore/streamutil.h118
-rw-r--r--zencore/include/zencore/string.h595
-rw-r--r--zencore/include/zencore/thread.h118
-rw-r--r--zencore/include/zencore/timer.h41
-rw-r--r--zencore/include/zencore/trace.h91
-rw-r--r--zencore/include/zencore/uid.h78
-rw-r--r--zencore/include/zencore/varint.h255
-rw-r--r--zencore/include/zencore/windows.h10
-rw-r--r--zencore/include/zencore/xxhash.h87
-rw-r--r--zencore/include/zencore/zencore.h134
-rw-r--r--zencore/iobuffer.cpp341
-rw-r--r--zencore/iohash.cpp73
-rw-r--r--zencore/iothreadpool.cpp36
-rw-r--r--zencore/iothreadpool.h31
-rw-r--r--zencore/md5.cpp446
-rw-r--r--zencore/memory.cpp165
-rw-r--r--zencore/refcount.cpp96
-rw-r--r--zencore/sha1.cpp439
-rw-r--r--zencore/sharedbuffer.cpp110
-rw-r--r--zencore/snapshot_manifest.cpp281
-rw-r--r--zencore/stats.cpp73
-rw-r--r--zencore/stream.cpp307
-rw-r--r--zencore/streamutil.cpp104
-rw-r--r--zencore/string.cpp913
-rw-r--r--zencore/thread.cpp192
-rw-r--r--zencore/timer.cpp67
-rw-r--r--zencore/trace.cpp51
-rw-r--r--zencore/uid.cpp196
-rw-r--r--zencore/xxhash.cpp50
-rw-r--r--zencore/zencore.cpp70
-rw-r--r--zencore/zencore.vcxproj188
-rw-r--r--zencore/zencore.vcxproj.filters78
69 files changed, 17427 insertions, 0 deletions
diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp
new file mode 100644
index 000000000..ec5d496d5
--- /dev/null
+++ b/zencore/blake3.cpp
@@ -0,0 +1,153 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/blake3.h>
+
+#include <zencore/string.h>
+#include <zencore/zencore.h>
+#include "../3rdparty/BLAKE3/c/blake3.h"
+
+#pragma comment(lib, "blake3.lib")
+
+#include <doctest/doctest.h>
+#include <string.h>
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+void
+blake3_forcelink()
+{
+}
+
+BLAKE3 BLAKE3::Zero; // Initialized to all zeroes
+
+BLAKE3
+BLAKE3::HashMemory(const void* data, size_t byteCount)
+{
+ BLAKE3 b3;
+
+ blake3_hasher b3h;
+ blake3_hasher_init(&b3h);
+ blake3_hasher_update(&b3h, data, byteCount);
+ blake3_hasher_finalize(&b3h, b3.Hash, sizeof b3.Hash);
+
+ return b3;
+}
+
+BLAKE3
+BLAKE3::FromHexString(const char* string)
+{
+ BLAKE3 b3;
+
+ ParseHexBytes(string, 2 * sizeof b3.Hash, b3.Hash);
+
+ return b3;
+}
+
+const char*
+BLAKE3::ToHexString(char* outString /* 40 characters + NUL terminator */) const
+{
+ ToHexBytes(Hash, sizeof(BLAKE3), outString);
+ outString[2 * sizeof(BLAKE3)] = '\0';
+
+ return outString;
+}
+
+StringBuilderBase&
+BLAKE3::ToHexString(StringBuilderBase& outBuilder) const
+{
+ char str[65];
+ ToHexString(str);
+
+ outBuilder.AppendRange(str, &str[65]);
+
+ return outBuilder;
+}
+
+BLAKE3Stream::BLAKE3Stream()
+{
+ blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState);
+ static_assert(sizeof(blake3_hasher) <= sizeof(m_HashState));
+ blake3_hasher_init(b3h);
+}
+
+void
+BLAKE3Stream::Reset()
+{
+ blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState);
+ blake3_hasher_init(b3h);
+}
+
+BLAKE3Stream&
+BLAKE3Stream::Append(const void* data, size_t byteCount)
+{
+ blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState);
+ blake3_hasher_update(b3h, data, byteCount);
+
+ return *this;
+}
+
+BLAKE3
+BLAKE3Stream::GetHash()
+{
+ BLAKE3 b3;
+
+ blake3_hasher* b3h = reinterpret_cast<blake3_hasher*>(m_HashState);
+ blake3_hasher_finalize(b3h, b3.Hash, sizeof b3.Hash);
+
+ return b3;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+doctest::String
+toString(const BLAKE3& value)
+{
+ char text[2 * sizeof(BLAKE3) + 1];
+ value.ToHexString(text);
+
+ return text;
+}
+
+TEST_CASE("BLAKE3")
+{
+ SUBCASE("Basics")
+ {
+ BLAKE3 b3 = BLAKE3::HashMemory(nullptr, 0);
+ CHECK(BLAKE3::FromHexString("af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") == b3);
+
+ BLAKE3::String_t b3s;
+ std::string b3ss = b3.ToHexString(b3s);
+ CHECK(b3ss == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262");
+ }
+
+ SUBCASE("hashes")
+ {
+ CHECK(BLAKE3::FromHexString("00307ced6a8b278d5e3a9f77b138d0e9d2209717c9d45b205f427a73565cc5fb") == BLAKE3::HashMemory("abc123", 6));
+ CHECK(BLAKE3::FromHexString("a7142c8c3905cd11b1e35105c7ac588b75d6798822f71e1145187ad46f3e8df4") ==
+ BLAKE3::HashMemory("1234567890123456789012345678901234567890", 40));
+ CHECK(BLAKE3::FromHexString("70e708532559265c4662d0285e5e0a4be8bd972bd1f255a93ddf342243adc427") ==
+ BLAKE3::HashMemory("The HttpSendHttpResponse function sends an HTTP response to the specified HTTP request.", 87));
+ }
+
+ SUBCASE("streamHashes")
+ {
+ auto streamHash = [](const void* data, size_t dataBytes) -> BLAKE3 {
+ BLAKE3Stream b3s;
+ b3s.Append(data, dataBytes);
+ return b3s.GetHash();
+ };
+
+ CHECK(BLAKE3::FromHexString("00307ced6a8b278d5e3a9f77b138d0e9d2209717c9d45b205f427a73565cc5fb") == streamHash("abc123", 6));
+ CHECK(BLAKE3::FromHexString("a7142c8c3905cd11b1e35105c7ac588b75d6798822f71e1145187ad46f3e8df4") ==
+ streamHash("1234567890123456789012345678901234567890", 40));
+ CHECK(BLAKE3::FromHexString("70e708532559265c4662d0285e5e0a4be8bd972bd1f255a93ddf342243adc427") ==
+ streamHash("The HttpSendHttpResponse function sends an HTTP response to the specified HTTP request.", 87));
+ }
+}
+
+} // namespace zen
diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
new file mode 100644
index 000000000..4ee9e9281
--- /dev/null
+++ b/zencore/compactbinary.cpp
@@ -0,0 +1,1279 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/compactbinary.h"
+
+#include <zencore/endian.h>
+#include <zencore/stream.h>
+#include <zencore/trace.h>
+
+#include <doctest/doctest.h>
+#include <ryml/ryml.hpp>
+#include <string_view>
+
+namespace zen {
+
+const int DaysPerMonth[] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365};
+
+bool
+IsLeapYear(int Year)
+{
+ if ((Year % 4) == 0)
+ {
+ return (((Year % 100) != 0) || ((Year % 400) == 0));
+ }
+
+ return false;
+}
+
+void
+DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, int MilliSecond)
+{
+ int TotalDays = 0;
+
+ if ((Month > 2) && IsLeapYear(Year))
+ {
+ ++TotalDays;
+ }
+
+ --Year; // the current year is not a full year yet
+ --Month; // the current month is not a full month yet
+
+ TotalDays += Year * 365;
+ TotalDays += Year / 4; // leap year day every four years...
+ TotalDays -= Year / 100; // ...except every 100 years...
+ TotalDays += Year / 400; // ...but also every 400 years
+ TotalDays += DaysToMonth[Month]; // days in this year up to last month
+ TotalDays += Day - 1; // days in this month minus today
+
+ Ticks = TotalDays * TimeSpan::TicksPerDay + Hour * TimeSpan::TicksPerHour + Minute * TimeSpan::TicksPerMinute +
+ Second * TimeSpan::TicksPerSecond + MilliSecond * TimeSpan::TicksPerMillisecond;
+}
+
+void
+TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano)
+{
+ int64_t TotalTicks = 0;
+
+ TotalTicks += Days * TicksPerDay;
+ TotalTicks += Hours * TicksPerHour;
+ TotalTicks += Minutes * TicksPerMinute;
+ TotalTicks += Seconds * TicksPerSecond;
+ TotalTicks += FractionNano / NanosecondsPerTick;
+
+ Ticks = TotalTicks;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace usonprivate {
+
+ static constexpr const uint8_t GEmptyObjectPayload[] = {uint8_t(CbFieldType::Object), 0x00};
+ static constexpr const uint8_t GEmptyArrayPayload[] = {uint8_t(CbFieldType::Array), 0x01, 0x00};
+
+ template<typename T>
+ static constexpr inline T ReadUnaligned(const void* const Memory)
+ {
+#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
+ return *static_cast<const T*>(Memory);
+#else
+ T Value;
+ memcpy(&Value, Memory, sizeof(Value));
+ return Value;
+#endif
+ }
+} // namespace usonprivate
+
+//////////////////////////////////////////////////////////////////////////
+
+CbFieldView::CbFieldView(const void* DataPointer, CbFieldType FieldType)
+{
+ const uint8_t* Bytes = static_cast<const uint8_t*>(DataPointer);
+ const CbFieldType LocalType = CbFieldTypeOps::HasFieldType(FieldType) ? (CbFieldType(*Bytes++) | CbFieldType::HasFieldType) : FieldType;
+
+ uint32_t NameLenByteCount = 0;
+ const uint64_t NameLen64 = CbFieldTypeOps::HasFieldName(LocalType) ? ReadVarUInt(Bytes, NameLenByteCount) : 0;
+ Bytes += NameLen64 + NameLenByteCount;
+
+ Type = LocalType;
+ NameLen = uint32_t(std::clamp<uint64_t>(NameLen64, 0, ~uint32_t(0)));
+ Payload = Bytes;
+}
+
+void
+CbFieldView::IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+{
+ switch (CbFieldTypeOps::GetType(Type))
+ {
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ return CbObjectView::FromFieldView(*this).IterateAttachments(Visitor);
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ return CbArrayView::FromFieldView(*this).IterateAttachments(Visitor);
+ case CbFieldType::CompactBinaryAttachment:
+ case CbFieldType::BinaryAttachment:
+ return Visitor(*this);
+ default:
+ return;
+ }
+}
+
+CbObjectView
+CbFieldView::AsObjectView()
+{
+ if (CbFieldTypeOps::IsObject(Type))
+ {
+ Error = CbFieldError::None;
+ return CbObjectView::FromFieldView(*this);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return CbObjectView();
+ }
+}
+
+CbArrayView
+CbFieldView::AsArrayView()
+{
+ if (CbFieldTypeOps::IsArray(Type))
+ {
+ Error = CbFieldError::None;
+ return CbArrayView::FromFieldView(*this);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return CbArrayView();
+ }
+}
+
+MemoryView
+CbFieldView::AsBinaryView(const MemoryView Default)
+{
+ if (CbFieldTypeOps::IsBinary(Type))
+ {
+ const uint8_t* const PayloadBytes = static_cast<const uint8_t*>(Payload);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(PayloadBytes, ValueSizeByteCount);
+
+ Error = CbFieldError::None;
+ return MemoryView(PayloadBytes + ValueSizeByteCount, ValueSize);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+std::string_view
+CbFieldView::AsString(const std::string_view Default)
+{
+ if (CbFieldTypeOps::IsString(Type))
+ {
+ const char* const PayloadChars = static_cast<const char*>(Payload);
+ uint32_t ValueSizeByteCount;
+ const uint64_t ValueSize = ReadVarUInt(PayloadChars, ValueSizeByteCount);
+
+ if (ValueSize >= (uint64_t(1) << 31))
+ {
+ Error = CbFieldError::RangeError;
+ return Default;
+ }
+
+ Error = CbFieldError::None;
+ return std::string_view(PayloadChars + ValueSizeByteCount, ValueSize);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+uint64_t
+CbFieldView::AsInteger(const uint64_t Default, const IntegerParams Params)
+{
+ if (CbFieldTypeOps::IsInteger(Type))
+ {
+ // A shift of a 64-bit value by 64 is undefined so shift by one less because magnitude is never zero.
+ const uint64_t OutOfRangeMask = uint64_t(-2) << (Params.MagnitudeBits - 1);
+ const uint64_t IsNegative = uint8_t(Type) & 1;
+
+ uint32_t MagnitudeByteCount;
+ const uint64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount);
+ const uint64_t Value = Magnitude ^ -int64_t(IsNegative);
+
+ const uint64_t IsInRange = (!(Magnitude & OutOfRangeMask)) & ((!IsNegative) | Params.IsSigned);
+ Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
+
+ const uint64_t UseValueMask = -int64_t(IsInRange);
+ return (Value & UseValueMask) | (Default & ~UseValueMask);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+float
+CbFieldView::AsFloat(const float Default)
+{
+ switch (CbFieldTypeOps::GetType(Type))
+ {
+ case CbFieldType::IntegerPositive:
+ case CbFieldType::IntegerNegative:
+ {
+ const uint64_t IsNegative = uint8_t(Type) & 1;
+ constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*FLT_MANT_DIG*/ 24) - 1);
+
+ uint32_t MagnitudeByteCount;
+ const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative;
+ const uint64_t IsInRange = !(Magnitude & OutOfRangeMask);
+ Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
+ return IsInRange ? float(IsNegative ? -Magnitude : Magnitude) : Default;
+ }
+ case CbFieldType::Float32:
+ {
+ Error = CbFieldError::None;
+ const uint32_t Value = FromNetworkOrder(usonprivate::ReadUnaligned<uint32_t>(Payload));
+ return reinterpret_cast<const float&>(Value);
+ }
+ case CbFieldType::Float64:
+ Error = CbFieldError::RangeError;
+ return Default;
+ default:
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+double
+CbFieldView::AsDouble(const double Default)
+{
+ switch (CbFieldTypeOps::GetType(Type))
+ {
+ case CbFieldType::IntegerPositive:
+ case CbFieldType::IntegerNegative:
+ {
+ const uint64_t IsNegative = uint8_t(Type) & 1;
+ constexpr uint64_t OutOfRangeMask = ~((uint64_t(1) << /*DBL_MANT_DIG*/ 53) - 1);
+
+ uint32_t MagnitudeByteCount;
+ const int64_t Magnitude = ReadVarUInt(Payload, MagnitudeByteCount) + IsNegative;
+ const uint64_t IsInRange = !(Magnitude & OutOfRangeMask);
+ Error = IsInRange ? CbFieldError::None : CbFieldError::RangeError;
+ return IsInRange ? double(IsNegative ? -Magnitude : Magnitude) : Default;
+ }
+ case CbFieldType::Float32:
+ {
+ Error = CbFieldError::None;
+ const uint32_t Value = FromNetworkOrder(usonprivate::ReadUnaligned<uint32_t>(Payload));
+ return reinterpret_cast<const float&>(Value);
+ }
+ case CbFieldType::Float64:
+ {
+ Error = CbFieldError::None;
+ const uint64_t Value = FromNetworkOrder(usonprivate::ReadUnaligned<uint64_t>(Payload));
+ return reinterpret_cast<const double&>(Value);
+ }
+ default:
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+bool
+CbFieldView::AsBool(const bool bDefault)
+{
+ const CbFieldType LocalType = Type;
+ const bool bIsBool = CbFieldTypeOps::IsBool(LocalType);
+ Error = bIsBool ? CbFieldError::None : CbFieldError::TypeError;
+ return (uint8_t(bIsBool) & uint8_t(LocalType) & 1) | ((!bIsBool) & bDefault);
+}
+
+IoHash
+CbFieldView::AsCompactBinaryAttachment(const IoHash& Default)
+{
+ if (CbFieldTypeOps::IsCompactBinaryAttachment(Type))
+ {
+ Error = CbFieldError::None;
+ return IoHash::MakeFrom(Payload);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+IoHash
+CbFieldView::AsBinaryAttachment(const IoHash& Default)
+{
+ if (CbFieldTypeOps::IsBinaryAttachment(Type))
+ {
+ Error = CbFieldError::None;
+ return IoHash::MakeFrom(Payload);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+IoHash
+CbFieldView::AsAttachment(const IoHash& Default)
+{
+ if (CbFieldTypeOps::IsAttachment(Type))
+ {
+ Error = CbFieldError::None;
+ return IoHash::MakeFrom(Payload);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+IoHash
+CbFieldView::AsHash(const IoHash& Default)
+{
+ if (CbFieldTypeOps::IsHash(Type))
+ {
+ Error = CbFieldError::None;
+ return IoHash::MakeFrom(Payload);
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+Guid
+CbFieldView::AsUuid()
+{
+ return AsUuid(Guid());
+}
+
+Guid
+CbFieldView::AsUuid(const Guid& Default)
+{
+ if (CbFieldTypeOps::IsUuid(Type))
+ {
+ Error = CbFieldError::None;
+ Guid Value;
+ memcpy(&Value, Payload, sizeof(Guid));
+ Value.A = FromNetworkOrder(Value.A);
+ Value.B = FromNetworkOrder(Value.B);
+ Value.C = FromNetworkOrder(Value.C);
+ Value.D = FromNetworkOrder(Value.D);
+ return Value;
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+Oid
+CbFieldView::AsObjectId()
+{
+ return AsObjectId(Oid());
+}
+
+Oid
+CbFieldView::AsObjectId(const Oid& Default)
+{
+ if (CbFieldTypeOps::IsObjectId(Type))
+ {
+ Error = CbFieldError::None;
+ Oid Value;
+ memcpy(&Value, Payload, sizeof(Oid));
+ return Value;
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+int64_t
+CbFieldView::AsDateTimeTicks(const int64_t Default)
+{
+ if (CbFieldTypeOps::IsDateTime(Type))
+ {
+ Error = CbFieldError::None;
+ return FromNetworkOrder(usonprivate::ReadUnaligned<int64_t>(Payload));
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+DateTime
+CbFieldView::AsDateTime()
+{
+ return DateTime(AsDateTimeTicks(0));
+}
+
+DateTime
+CbFieldView::AsDateTime(DateTime Default)
+{
+ return DateTime(AsDateTimeTicks(Default.GetTicks()));
+}
+
+int64_t
+CbFieldView::AsTimeSpanTicks(const int64_t Default)
+{
+ if (CbFieldTypeOps::IsTimeSpan(Type))
+ {
+ Error = CbFieldError::None;
+ return FromNetworkOrder(usonprivate::ReadUnaligned<int64_t>(Payload));
+ }
+ else
+ {
+ Error = CbFieldError::TypeError;
+ return Default;
+ }
+}
+
+TimeSpan
+CbFieldView::AsTimeSpan()
+{
+ return TimeSpan(AsTimeSpanTicks(0));
+}
+
+TimeSpan
+CbFieldView::AsTimeSpan(TimeSpan Default)
+{
+ return TimeSpan(AsTimeSpanTicks(Default.GetTicks()));
+}
+
+uint64_t
+CbFieldView::GetSize() const
+{
+ return sizeof(CbFieldType) + GetViewNoType().GetSize();
+}
+
+uint64_t
+CbFieldView::GetPayloadSize() const
+{
+ switch (CbFieldTypeOps::GetType(Type))
+ {
+ case CbFieldType::None:
+ case CbFieldType::Null:
+ return 0;
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ case CbFieldType::Binary:
+ case CbFieldType::String:
+ {
+ uint32_t PayloadSizeByteCount;
+ const uint64_t PayloadSize = ReadVarUInt(Payload, PayloadSizeByteCount);
+ return PayloadSize + PayloadSizeByteCount;
+ }
+ case CbFieldType::IntegerPositive:
+ case CbFieldType::IntegerNegative:
+ return MeasureVarUInt(Payload);
+ case CbFieldType::Float32:
+ return 4;
+ case CbFieldType::Float64:
+ return 8;
+ case CbFieldType::BoolFalse:
+ case CbFieldType::BoolTrue:
+ return 0;
+ case CbFieldType::CompactBinaryAttachment:
+ case CbFieldType::BinaryAttachment:
+ case CbFieldType::Hash:
+ return 20;
+ case CbFieldType::Uuid:
+ return 16;
+ case CbFieldType::ObjectId:
+ return 12;
+ case CbFieldType::DateTime:
+ case CbFieldType::TimeSpan:
+ return 8;
+ default:
+ return 0;
+ }
+}
+
+IoHash
+CbFieldView::GetHash() const
+{
+ IoHashStream HashStream;
+ GetHash(HashStream);
+ return HashStream.GetHash();
+}
+
+void
+CbFieldView::GetHash(IoHashStream& Hash) const
+{
+ const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type);
+ Hash.Append(&SerializedType, sizeof(SerializedType));
+ auto View = GetViewNoType();
+ Hash.Append(View.GetData(), View.GetSize());
+}
+
+bool
+CbFieldView::Equals(const CbFieldView& Other) const
+{
+ return CbFieldTypeOps::GetSerializedType(Type) == CbFieldTypeOps::GetSerializedType(Other.Type) &&
+ GetViewNoType().EqualBytes(Other.GetViewNoType());
+}
+
+void
+CbFieldView::CopyTo(MutableMemoryView Buffer) const
+{
+ const MemoryView Source = GetViewNoType();
+ ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize());
+ // TEXT("A buffer of %" UINT64_FMT " bytes was provided when %" UINT64_FMT " bytes are required"),
+ // Buffer.GetSize(),
+ // sizeof(CbFieldType) + Source.GetSize());
+ *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetSerializedType(Type);
+ Buffer.RightChopInline(sizeof(CbFieldType));
+ memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize());
+}
+
+void
+CbFieldView::CopyTo(BinaryWriter& Ar) const
+{
+ const MemoryView Source = GetViewNoType();
+ CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type);
+ Ar.Write(&SerializedType, sizeof(SerializedType));
+ Ar.Write(Source.GetData(), Source.GetSize());
+}
+
+MemoryView
+CbFieldView::GetView() const
+{
+ const uint32_t TypeSize = CbFieldTypeOps::HasFieldType(Type) ? sizeof(CbFieldType) : 0;
+ const uint32_t NameSize = CbFieldTypeOps::HasFieldName(Type) ? NameLen + MeasureVarUInt(NameLen) : 0;
+ const uint64_t PayloadSize = GetPayloadSize();
+ return MemoryView(static_cast<const uint8_t*>(Payload) - TypeSize - NameSize, TypeSize + NameSize + PayloadSize);
+}
+
+MemoryView
+CbFieldView::GetViewNoType() const
+{
+ const uint32_t NameSize = CbFieldTypeOps::HasFieldName(Type) ? NameLen + MeasureVarUInt(NameLen) : 0;
+ const uint64_t PayloadSize = GetPayloadSize();
+ return MemoryView(static_cast<const uint8_t*>(Payload) - NameSize, NameSize + PayloadSize);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+CbArrayView::CbArrayView() : CbFieldView(usonprivate::GEmptyArrayPayload)
+{
+}
+
+uint64_t
+CbArrayView::Num() const
+{
+ const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload());
+ PayloadBytes += MeasureVarUInt(PayloadBytes);
+ uint32_t NumByteCount;
+ return ReadVarUInt(PayloadBytes, NumByteCount);
+}
+
+CbFieldViewIterator
+CbArrayView::CreateViewIterator() const
+{
+ const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload());
+ uint32_t PayloadSizeByteCount;
+ const uint64_t PayloadSize = ReadVarUInt(PayloadBytes, PayloadSizeByteCount);
+ PayloadBytes += PayloadSizeByteCount;
+ const uint64_t NumByteCount = MeasureVarUInt(PayloadBytes);
+ if (PayloadSize > NumByteCount)
+ {
+ const void* const PayloadEnd = PayloadBytes + PayloadSize;
+ PayloadBytes += NumByteCount;
+ const CbFieldType UniformType =
+ CbFieldTypeOps::GetType(GetType()) == CbFieldType::UniformArray ? CbFieldType(*PayloadBytes++) : CbFieldType::HasFieldType;
+ return CbFieldViewIterator::MakeRange(MemoryView(PayloadBytes, PayloadEnd), UniformType);
+ }
+ return CbFieldViewIterator();
+}
+
+void
+CbArrayView::VisitFields(ICbVisitor&)
+{
+}
+
+uint64_t
+CbArrayView::GetSize() const
+{
+ return sizeof(CbFieldType) + GetPayloadSize();
+}
+
+IoHash
+CbArrayView::GetHash() const
+{
+ IoHashStream Hash;
+ GetHash(Hash);
+ return Hash.GetHash();
+}
+
+void
+CbArrayView::GetHash(IoHashStream& HashStream) const
+{
+ const CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType());
+ HashStream.Append(&SerializedType, sizeof(SerializedType));
+ auto _ = GetPayloadView();
+ HashStream.Append(_.GetData(), _.GetSize());
+}
+
+bool
+CbArrayView::Equals(const CbArrayView& Other) const
+{
+ return CbFieldTypeOps::GetType(GetType()) == CbFieldTypeOps::GetType(Other.GetType()) &&
+ GetPayloadView().EqualBytes(Other.GetPayloadView());
+}
+
+void
+CbArrayView::CopyTo(MutableMemoryView Buffer) const
+{
+ const MemoryView Source = GetPayloadView();
+ ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize());
+ // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."),
+ // Buffer.GetSize(),
+ // sizeof(CbFieldType) + Source.GetSize());
+
+ *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType());
+ Buffer.RightChopInline(sizeof(CbFieldType));
+ memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize());
+}
+
+void
+CbArrayView::CopyTo(BinaryWriter& Ar) const
+{
+ const MemoryView Source = GetPayloadView();
+ CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType());
+ Ar.Write(&SerializedType, sizeof(SerializedType));
+ Ar.Write(Source.GetData(), Source.GetSize());
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+CbObjectView::CbObjectView() : CbFieldView(usonprivate::GEmptyObjectPayload)
+{
+}
+
+CbFieldViewIterator
+CbObjectView::CreateViewIterator() const
+{
+ const uint8_t* PayloadBytes = static_cast<const uint8_t*>(GetPayload());
+ uint32_t PayloadSizeByteCount;
+ const uint64_t PayloadSize = ReadVarUInt(PayloadBytes, PayloadSizeByteCount);
+
+ PayloadBytes += PayloadSizeByteCount;
+
+ if (PayloadSize)
+ {
+ const void* const PayloadEnd = PayloadBytes + PayloadSize;
+ const CbFieldType UniformType =
+ CbFieldTypeOps::GetType(GetType()) == CbFieldType::UniformObject ? CbFieldType(*PayloadBytes++) : CbFieldType::HasFieldType;
+ return CbFieldViewIterator::MakeRange(MemoryView(PayloadBytes, PayloadEnd), UniformType);
+ }
+
+ return CbFieldViewIterator();
+}
+
+void
+CbObjectView::VisitFields(ICbVisitor&)
+{
+}
+
+CbFieldView
+CbObjectView::FindView(const std::string_view Name) const
+{
+ for (const CbFieldView Field : *this)
+ {
+ if (Name == Field.GetName())
+ {
+ return Field;
+ }
+ }
+ return CbFieldView();
+}
+
+CbFieldView
+CbObjectView::FindViewIgnoreCase(const std::string_view Name) const
+{
+ for (const CbFieldView Field : *this)
+ {
+ if (Name == Field.GetName())
+ {
+ return Field;
+ }
+ }
+ return CbFieldView();
+}
+
+uint64_t
+CbObjectView::GetSize() const
+{
+ return sizeof(CbFieldType) + GetPayloadSize();
+}
+
+IoHash
+CbObjectView::GetHash() const
+{
+ IoHashStream Hash;
+ GetHash(Hash);
+ return Hash.GetHash();
+}
+
+void
+CbObjectView::GetHash(IoHashStream& HashStream) const
+{
+ const CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType());
+ HashStream.Append(&SerializedType, sizeof(SerializedType));
+ HashStream.Append(GetPayloadView());
+}
+
+bool
+CbObjectView::Equals(const CbObjectView& Other) const
+{
+ return CbFieldTypeOps::GetType(GetType()) == CbFieldTypeOps::GetType(Other.GetType()) &&
+ GetPayloadView().EqualBytes(Other.GetPayloadView());
+}
+
+void
+CbObjectView::CopyTo(MutableMemoryView Buffer) const
+{
+ const MemoryView Source = GetPayloadView();
+ ZEN_ASSERT(Buffer.GetSize() == (sizeof(CbFieldType) + Source.GetSize()));
+ // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."),
+ // Buffer.GetSize(),
+ // sizeof(CbFieldType) + Source.GetSize());
+ *static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType());
+ Buffer.RightChopInline(sizeof(CbFieldType));
+ memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize());
+}
+
+void
+CbObjectView::CopyTo(BinaryWriter& Ar) const
+{
+ const MemoryView Source = GetPayloadView();
+ CbFieldType SerializedType = CbFieldTypeOps::GetType(GetType());
+ Ar.Write(&SerializedType, sizeof(SerializedType));
+ Ar.Write(Source.GetData(), Source.GetSize());
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template<typename FieldType>
+uint64_t
+TCbFieldIterator<FieldType>::GetRangeSize() const
+{
+ MemoryView View;
+ if (TryGetSerializedRangeView(View))
+ {
+ return View.GetSize();
+ }
+ else
+ {
+ uint64_t Size = 0;
+ for (CbFieldViewIterator It(*this); It; ++It)
+ {
+ Size += It.GetSize();
+ }
+ return Size;
+ }
+}
+
+template<typename FieldType>
+IoHash
+TCbFieldIterator<FieldType>::GetRangeHash() const
+{
+ IoHashStream Hash;
+ GetRangeHash(Hash);
+ return IoHash(Hash.GetHash());
+}
+
+template<typename FieldType>
+void
+TCbFieldIterator<FieldType>::GetRangeHash(IoHashStream& Hash) const
+{
+ MemoryView View;
+ if (TryGetSerializedRangeView(View))
+ {
+ Hash.Append(View.GetData(), View.GetSize());
+ }
+ else
+ {
+ for (CbFieldViewIterator It(*this); It; ++It)
+ {
+ It.GetHash(Hash);
+ }
+ }
+}
+
+template<typename FieldType>
+void
+TCbFieldIterator<FieldType>::CopyRangeTo(MutableMemoryView InBuffer) const
+{
+ MemoryView Source;
+ if (TryGetSerializedRangeView(Source))
+ {
+ ZEN_ASSERT(InBuffer.GetSize() == Source.GetSize());
+ // TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."),
+ // InBuffer.GetSize(),
+ // Source.GetSize());
+ memcpy(InBuffer.GetData(), Source.GetData(), Source.GetSize());
+ }
+ else
+ {
+ for (CbFieldViewIterator It(*this); It; ++It)
+ {
+ const uint64_t Size = It.GetSize();
+ It.CopyTo(InBuffer.Left(Size));
+ InBuffer.RightChopInline(Size);
+ }
+ }
+}
+
+template class TCbFieldIterator<CbFieldView>;
+template class TCbFieldIterator<CbField>;
+
+template<typename FieldType>
+void
+TCbFieldIterator<FieldType>::IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const
+{
+ if (CbFieldTypeOps::HasFieldType(FieldType::GetType()))
+ {
+ // Always iterate over non-uniform ranges because we do not know if they contain an attachment.
+ for (CbFieldViewIterator It(*this); It; ++It)
+ {
+ if (CbFieldTypeOps::MayContainAttachments(It.GetType()))
+ {
+ It.IterateAttachments(Visitor);
+ }
+ }
+ }
+ else
+ {
+ // Only iterate over uniform ranges if the uniform type may contain an attachment.
+ if (CbFieldTypeOps::MayContainAttachments(FieldType::GetType()))
+ {
+ for (CbFieldViewIterator It(*this); It; ++It)
+ {
+ It.IterateAttachments(Visitor);
+ }
+ }
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+CbFieldIterator
+CbFieldIterator::CloneRange(const CbFieldViewIterator& It)
+{
+ MemoryView View;
+ if (It.TryGetSerializedRangeView(View))
+ {
+ return MakeRange(SharedBuffer::Clone(View));
+ }
+ else
+ {
+ UniqueBuffer Buffer = UniqueBuffer::Alloc(It.GetRangeSize());
+ It.CopyRangeTo(MutableMemoryView(Buffer.GetData(), Buffer.GetSize()));
+ return MakeRange(SharedBuffer(std::move(Buffer)));
+ }
+}
+
+SharedBuffer
+CbFieldIterator::GetRangeBuffer() const
+{
+ const MemoryView RangeView = GetRangeView();
+ const SharedBuffer& OuterBuffer = GetOuterBuffer();
+ return OuterBuffer.GetView() == RangeView ? OuterBuffer : SharedBuffer::MakeView(RangeView, OuterBuffer);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+uint64_t
+MeasureCompactBinary(MemoryView View, CbFieldType Type)
+{
+ uint64_t Size;
+ return TryMeasureCompactBinary(View, Type, Size, Type) ? Size : 0;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+bool
+TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize, CbFieldType Type)
+{
+ uint64_t Size = 0;
+
+ if (CbFieldTypeOps::HasFieldType(Type))
+ {
+ if (View.GetSize() == 0)
+ {
+ OutType = CbFieldType::None;
+ OutSize = 1;
+ return false;
+ }
+
+ Type = *static_cast<const CbFieldType*>(View.GetData());
+ View.RightChopInline(1);
+ Size += 1;
+ }
+
+ bool bDynamicSize = false;
+ uint64_t FixedSize = 0;
+ switch (CbFieldTypeOps::GetType(Type))
+ {
+ case CbFieldType::Null:
+ break;
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ case CbFieldType::Binary:
+ case CbFieldType::String:
+ case CbFieldType::IntegerPositive:
+ case CbFieldType::IntegerNegative:
+ bDynamicSize = true;
+ break;
+ case CbFieldType::Float32:
+ FixedSize = 4;
+ break;
+ case CbFieldType::Float64:
+ FixedSize = 8;
+ break;
+ case CbFieldType::BoolFalse:
+ case CbFieldType::BoolTrue:
+ break;
+ case CbFieldType::CompactBinaryAttachment:
+ case CbFieldType::BinaryAttachment:
+ case CbFieldType::Hash:
+ FixedSize = 20;
+ break;
+ case CbFieldType::Uuid:
+ FixedSize = 16;
+ break;
+ case CbFieldType::ObjectId:
+ FixedSize = 12;
+ break;
+ case CbFieldType::DateTime:
+ case CbFieldType::TimeSpan:
+ FixedSize = 8;
+ break;
+ case CbFieldType::None:
+ default:
+ OutType = CbFieldType::None;
+ OutSize = 0;
+ return false;
+ }
+
+ OutType = Type;
+
+ if (CbFieldTypeOps::HasFieldName(Type))
+ {
+ if (View.GetSize() == 0)
+ {
+ OutSize = Size + 1;
+ return false;
+ }
+
+ uint32_t NameLenByteCount = MeasureVarUInt(View.GetData());
+ if (View.GetSize() < NameLenByteCount)
+ {
+ OutSize = Size + NameLenByteCount;
+ return false;
+ }
+
+ const uint64_t NameLen = ReadVarUInt(View.GetData(), NameLenByteCount);
+ const uint64_t NameSize = NameLen + NameLenByteCount;
+
+ if (bDynamicSize && View.GetSize() < NameSize)
+ {
+ OutSize = Size + NameSize;
+ return false;
+ }
+
+ View.RightChopInline(NameSize);
+ Size += NameSize;
+ }
+
+ switch (CbFieldTypeOps::GetType(Type))
+ {
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ case CbFieldType::Binary:
+ case CbFieldType::String:
+ if (View.GetSize() == 0)
+ {
+ OutSize = Size + 1;
+ return false;
+ }
+ else
+ {
+ uint32_t PayloadSizeByteCount = MeasureVarUInt(View.GetData());
+ if (View.GetSize() < PayloadSizeByteCount)
+ {
+ OutSize = Size + PayloadSizeByteCount;
+ return false;
+ }
+ const uint64_t PayloadSize = ReadVarUInt(View.GetData(), PayloadSizeByteCount);
+ OutSize = Size + PayloadSize + PayloadSizeByteCount;
+ }
+ return true;
+
+ case CbFieldType::IntegerPositive:
+ case CbFieldType::IntegerNegative:
+ if (View.GetSize() == 0)
+ {
+ OutSize = Size + 1;
+ return false;
+ }
+ OutSize = Size + MeasureVarUInt(View.GetData());
+ return true;
+
+ default:
+ OutSize = Size + FixedSize;
+ return true;
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+CbField
+LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator)
+{
+ std::vector<uint8_t> HeaderBytes;
+ CbFieldType FieldType;
+ uint64_t FieldSize = 1;
+
+ // Read in small increments until the total field size is known, to avoid reading too far.
+ for (;;)
+ {
+ const int32_t ReadSize = int32_t(FieldSize - HeaderBytes.size());
+ const size_t ReadOffset = HeaderBytes.size();
+ HeaderBytes.resize(ReadOffset + ReadSize);
+
+ Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize);
+ if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize))
+ {
+ break;
+ }
+ ZEN_ASSERT(FieldSize > 0, "Failed to load from invalid compact binary data.");
+ }
+
+ // Allocate the buffer, copy the header, and read the remainder of the field.
+ UniqueBuffer Buffer = Allocator(FieldSize);
+ ZEN_ASSERT(Buffer.GetSize() == FieldSize);
+ MutableMemoryView View = Buffer.GetView();
+ memcpy(View.GetData(), HeaderBytes.data(), HeaderBytes.size());
+ View.RightChopInline(HeaderBytes.size());
+ if (!View.IsEmpty())
+ {
+ Ar.Read(View.GetData(), View.GetSize());
+ }
+ return CbField(SharedBuffer(std::move(Buffer)));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+SaveCompactBinary(BinaryWriter& Ar, const CbFieldView& Field)
+{
+ Field.CopyTo(Ar);
+}
+
+void
+SaveCompactBinary(BinaryWriter& Ar, const CbArrayView& Array)
+{
+ Array.CopyTo(Ar);
+}
+
+void
+SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object)
+{
+ Object.CopyTo(Ar);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+StringBuilderBase&
+ToString(CbObjectView& Root, StringBuilderBase& OutString)
+{
+ ryml::Tree Tree;
+
+ ryml::NodeRef r = Tree.rootref();
+ r |= ryml::MAP;
+
+ for (CbFieldViewIterator It = Root.CreateViewIterator(); It; ++It)
+ {
+ }
+
+ return OutString;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+uson_forcelink()
+{
+}
+
+TEST_CASE("uson")
+{
+ using namespace std::literals;
+
+ SUBCASE("CbField")
+ {
+ constexpr CbFieldView DefaultField;
+ static_assert(!DefaultField.HasName(), "Error in HasName()");
+ static_assert(!DefaultField.HasValue(), "Error in HasValue()");
+ static_assert(!DefaultField.HasError(), "Error in HasError()");
+ static_assert(DefaultField.GetError() == CbFieldError::None, "Error in GetError()");
+
+ CHECK(DefaultField.GetSize() == 1);
+ CHECK(DefaultField.GetName().size() == 0);
+ CHECK(DefaultField.HasName() == false);
+ CHECK(DefaultField.HasValue() == false);
+ CHECK(DefaultField.HasError() == false);
+ CHECK(DefaultField.GetError() == CbFieldError::None);
+
+ const uint8_t Type = (uint8_t)CbFieldType::None;
+ CHECK(DefaultField.GetHash() == IoHash::HashMemory(&Type, sizeof Type));
+
+ CHECK(DefaultField.GetView() == MemoryView{});
+ MemoryView SerializedView;
+ CHECK(DefaultField.TryGetSerializedView(SerializedView) == false);
+ }
+
+ SUBCASE("CbField(None)")
+ {
+ CbFieldView NoneField(nullptr, CbFieldType::None);
+ CHECK(NoneField.GetSize() == 1);
+ CHECK(NoneField.GetName().size() == 0);
+ CHECK(NoneField.HasName() == false);
+ CHECK(NoneField.HasValue() == false);
+ CHECK(NoneField.HasError() == false);
+ CHECK(NoneField.GetError() == CbFieldError::None);
+ CHECK(NoneField.GetHash() == CbFieldView().GetHash());
+ CHECK(NoneField.GetView() == MemoryView());
+ MemoryView SerializedView;
+ CHECK(NoneField.TryGetSerializedView(SerializedView) == false);
+ }
+
+ SUBCASE("CbField(None|Type|Name)")
+ {
+ constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
+ constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'};
+ CbFieldView NoneField(NoneBytes);
+
+ CHECK(NoneField.GetSize() == sizeof(NoneBytes));
+ CHECK(NoneField.GetName().compare("Name"sv) == 0);
+ CHECK(NoneField.HasName() == true);
+ CHECK(NoneField.HasValue() == false);
+ CHECK(NoneField.GetHash() == IoHash::HashMemory(NoneBytes, sizeof NoneBytes));
+ CHECK(NoneField.GetView() == MemoryView(NoneBytes, sizeof NoneBytes));
+ MemoryView SerializedView;
+ CHECK(NoneField.TryGetSerializedView(SerializedView) == true);
+ CHECK(SerializedView == MemoryView(NoneBytes, sizeof NoneBytes));
+
+ uint8_t CopyBytes[sizeof(NoneBytes)];
+ NoneField.CopyTo(MutableMemoryView(CopyBytes, sizeof CopyBytes));
+ CHECK(MemoryView(NoneBytes, sizeof NoneBytes).EqualBytes(MemoryView(CopyBytes, sizeof CopyBytes)));
+ }
+
+ SUBCASE("CbField(None|Type)")
+ {
+ constexpr CbFieldType FieldType = CbFieldType::None;
+ constexpr const char NoneBytes[] = {char(FieldType)};
+ CbFieldView NoneField(NoneBytes);
+
+ CHECK(NoneField.GetSize() == sizeof NoneBytes);
+ CHECK(NoneField.GetName().size() == 0);
+ CHECK(NoneField.HasName() == false);
+ CHECK(NoneField.HasValue() == false);
+ CHECK(NoneField.GetHash() == CbFieldView().GetHash());
+ CHECK(NoneField.GetView() == MemoryView(NoneBytes, sizeof NoneBytes));
+ MemoryView SerializedView;
+ CHECK(NoneField.TryGetSerializedView(SerializedView) == true);
+ CHECK(SerializedView == MemoryView(NoneBytes, sizeof NoneBytes));
+ }
+
+ SUBCASE("CbField(None|Name)")
+ {
+ constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
+ constexpr const char NoneBytes[] = {char(FieldType), 4, 'N', 'a', 'm', 'e'};
+ CbFieldView NoneField(NoneBytes + 1, FieldType);
+ CHECK(NoneField.GetSize() == uint64_t(sizeof NoneBytes));
+ CHECK(NoneField.GetName().compare("Name") == 0);
+ CHECK(NoneField.HasName() == true);
+ CHECK(NoneField.HasValue() == false);
+ CHECK(NoneField.GetHash() == IoHash::HashMemory(NoneBytes, sizeof NoneBytes));
+ CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1));
+ MemoryView SerializedView;
+ CHECK(NoneField.TryGetSerializedView(SerializedView) == false);
+
+ uint8_t CopyBytes[sizeof(NoneBytes)];
+ NoneField.CopyTo(MutableMemoryView(CopyBytes, sizeof CopyBytes));
+ CHECK(MemoryView(NoneBytes, sizeof NoneBytes).EqualBytes(MemoryView(CopyBytes, sizeof CopyBytes)));
+ }
+
+ SUBCASE("CbField(None|EmptyName)")
+ {
+ constexpr CbFieldType FieldType = CbFieldType::None | CbFieldType::HasFieldName;
+ constexpr const uint8_t NoneBytes[] = {uint8_t(FieldType), 0};
+ CbFieldView NoneField(NoneBytes + 1, FieldType);
+ CHECK(NoneField.GetSize() == sizeof NoneBytes);
+ CHECK(NoneField.GetName().empty() == true);
+ CHECK(NoneField.HasName() == true);
+ CHECK(NoneField.HasValue() == false);
+ CHECK(NoneField.GetHash() == IoHash::HashMemory(NoneBytes, sizeof NoneBytes));
+ CHECK(NoneField.GetView() == MemoryView(&NoneBytes[1], sizeof NoneBytes - 1));
+ MemoryView SerializedView;
+ CHECK(NoneField.TryGetSerializedView(SerializedView) == false);
+ }
+
+ static_assert(!std::is_constructible<CbFieldView, const CbObjectView&>::value, "Invalid constructor for CbField");
+ static_assert(!std::is_assignable<CbFieldView, const CbObjectView&>::value, "Invalid assignment for CbField");
+ static_assert(!std::is_convertible<CbFieldView, CbObjectView>::value, "Invalid conversion to CbObject");
+ static_assert(!std::is_assignable<CbObjectView, const CbFieldView&>::value, "Invalid assignment for CbObject");
+}
+
+TEST_CASE("uson.null")
+{
+ using namespace std::literals;
+
+ SUBCASE("CbField(Null)")
+ {
+ CbFieldView NullField(nullptr, CbFieldType::Null);
+ CHECK(NullField.GetSize() == 1);
+ CHECK(NullField.IsNull() == true);
+ CHECK(NullField.HasValue() == true);
+ CHECK(NullField.HasError() == false);
+ CHECK(NullField.GetError() == CbFieldError::None);
+ const uint8_t Null[]{uint8_t(CbFieldType::Null)};
+ CHECK(NullField.GetHash() == IoHash::HashMemory(Null, sizeof Null));
+ }
+
+ SUBCASE("CbField(None)")
+ {
+ CbFieldView Field;
+ CHECK(Field.IsNull() == false);
+ }
+}
+
+} // namespace zen
diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp
new file mode 100644
index 000000000..d1422e5a2
--- /dev/null
+++ b/zencore/compactbinarybuilder.cpp
@@ -0,0 +1,1530 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/compactbinarybuilder.h"
+
+#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/endian.h>
+#include <zencore/stream.h>
+#include <zencore/string.h>
+
+#define _USE_MATH_DEFINES
+#include <math.h>
+
+#include <doctest/doctest.h>
+
+namespace zen {
+
+template<typename T>
+uint64_t
+AddUninitialized(std::vector<T>& Vector, uint64_t Count)
+{
+ const uint64_t Offset = Vector.size();
+ Vector.resize(Offset + Count);
+ return Offset;
+}
+
+template<typename T>
+uint64_t
+Append(std::vector<T>& Vector, const T* Data, uint64_t Count)
+{
+ const uint64_t Offset = Vector.size();
+ Vector.resize(Offset + Count);
+
+ memcpy(Vector.data() + Offset, Data, sizeof(T) * Count);
+
+ return Offset;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+enum class CbWriter::StateFlags : uint8_t
+{
+ None = 0,
+ /** Whether a name has been written for the current field. */
+ Name = 1 << 0,
+ /** Whether this state is in the process of writing a field. */
+ Field = 1 << 1,
+ /** Whether this state is for array fields. */
+ Array = 1 << 2,
+ /** Whether this state is for object fields. */
+ Object = 1 << 3,
+};
+
+ENUM_CLASS_FLAGS(CbWriter::StateFlags);
+
+/** Whether the field type can be used in a uniform array or uniform object. */
+static constexpr bool
+IsUniformType(const CbFieldType Type)
+{
+ if (CbFieldTypeOps::HasFieldName(Type))
+ {
+ return true;
+ }
+
+ switch (Type)
+ {
+ case CbFieldType::None:
+ case CbFieldType::Null:
+ case CbFieldType::BoolFalse:
+ case CbFieldType::BoolTrue:
+ return false;
+ default:
+ return true;
+ }
+}
+
+/** Append the payload from the compact binary value to the array and return its type. */
+static inline CbFieldType
+AppendCompactBinary(const CbFieldView& Value, std::vector<uint8_t>& OutData)
+{
+ struct FCopy : public CbFieldView
+ {
+ using CbFieldView::GetPayloadView;
+ using CbFieldView::GetType;
+ };
+ const FCopy& ValueCopy = static_cast<const FCopy&>(Value);
+ const MemoryView SourceView = ValueCopy.GetPayloadView();
+ const uint64_t TargetOffset = OutData.size();
+ OutData.resize(TargetOffset + SourceView.GetSize());
+ memcpy(OutData.data() + TargetOffset, SourceView.GetData(), SourceView.GetSize());
+ return CbFieldTypeOps::GetType(ValueCopy.GetType());
+}
+
+CbWriter::CbWriter()
+{
+ States.emplace_back();
+}
+
+CbWriter::CbWriter(const int64_t InitialSize) : CbWriter()
+{
+ Data.reserve(InitialSize);
+}
+
+CbWriter::~CbWriter()
+{
+}
+
+void
+CbWriter::Reset()
+{
+ Data.resize(0);
+ States.resize(0);
+ States.emplace_back();
+}
+
+CbFieldIterator
+CbWriter::Save()
+{
+ const uint64_t Size = GetSaveSize();
+ UniqueBuffer Buffer = UniqueBuffer::Alloc(Size);
+ const CbFieldViewIterator Output = Save(MutableMemoryView(Buffer.GetData(), Buffer.GetSize()));
+ SharedBuffer SharedBuf(std::move(Buffer));
+
+ return CbFieldIterator::MakeRangeView(Output, SharedBuf);
+}
+
+CbFieldViewIterator
+CbWriter::Save(const MutableMemoryView Buffer)
+{
+ ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None);
+ // TEXT("It is invalid to save while there are incomplete write operations."));
+ ZEN_ASSERT(Data.size() > 0); // TEXT("It is invalid to save when nothing has been written."));
+ ZEN_ASSERT(Buffer.GetSize() == Data.size());
+ // TEXT("Buffer is %" UINT64_FMT " bytes but %" INT64_FMT " is required."),
+ // Buffer.GetSize(),
+ // Data.Num());
+ memcpy(Buffer.GetData(), Data.data(), Data.size());
+ return CbFieldViewIterator::MakeRange(Buffer);
+}
+
+void
+CbWriter::Save(BinaryWriter& Writer)
+{
+ ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None);
+ // TEXT("It is invalid to save while there are incomplete write operations."));
+ ZEN_ASSERT(Data.size() > 0); // TEXT("It is invalid to save when nothing has been written."));
+ Writer.Write(Data.data(), Data.size());
+}
+
+uint64_t
+CbWriter::GetSaveSize() const
+{
+ return Data.size();
+}
+
+void
+CbWriter::BeginField()
+{
+ WriterState& State = States.back();
+ if ((State.Flags & StateFlags::Field) == StateFlags::None)
+ {
+ State.Flags |= StateFlags::Field;
+ State.Offset = Data.size();
+ Data.push_back(0);
+ }
+ else
+ {
+ ZEN_ASSERT((State.Flags & StateFlags::Name) == StateFlags::Name);
+ // TEXT("A new field cannot be written until the previous field '%.*hs' is finished."),
+ // GetActiveName().Len(),
+ // GetActiveName().GetData());
+ }
+}
+
+void
+CbWriter::EndField(CbFieldType Type)
+{
+ WriterState& State = States.back();
+
+ if ((State.Flags & StateFlags::Name) == StateFlags::Name)
+ {
+ Type |= CbFieldType::HasFieldName;
+ }
+ else
+ {
+ ZEN_ASSERT((State.Flags & StateFlags::Object) == StateFlags::None);
+ // TEXT("It is invalid to write an object field without a unique non-empty name."));
+ }
+
+ if (State.Count == 0)
+ {
+ State.UniformType = Type;
+ }
+ else if (State.UniformType != Type)
+ {
+ State.UniformType = CbFieldType::None;
+ }
+
+ State.Flags &= ~(StateFlags::Name | StateFlags::Field);
+ ++State.Count;
+ Data[State.Offset] = uint8_t(Type);
+}
+
+ZEN_NOINLINE
+CbWriter&
+CbWriter::SetName(const std::string_view Name)
+{
+ WriterState& State = States.back();
+ ZEN_ASSERT((State.Flags & StateFlags::Array) != StateFlags::Array);
+ // TEXT("It is invalid to write a name for an array field. Name '%.*hs'"),
+ // Name.Len(),
+ // Name.GetData());
+ ZEN_ASSERT(!Name.empty());
+ // TEXT("%s"),
+ //(State.Flags & EStateFlags::Object) == EStateFlags::Object
+ // ? TEXT("It is invalid to write an empty name for an object field. Specify a unique non-empty name.")
+ // : TEXT("It is invalid to write an empty name for a top-level field. Specify a name or avoid this call."));
+ ZEN_ASSERT((State.Flags & (StateFlags::Name | StateFlags::Field)) == StateFlags::None);
+ // TEXT("A new field '%.*hs' cannot be written until the previous field '%.*hs' is finished."),
+ // Name.Len(),
+ // Name.GetData(),
+ // GetActiveName().Len(),
+ // GetActiveName().GetData());
+
+ BeginField();
+ State.Flags |= StateFlags::Name;
+ const uint32_t NameLenByteCount = MeasureVarUInt(uint32_t(Name.size()));
+ const int64_t NameLenOffset = Data.size();
+ Data.resize(NameLenOffset + NameLenByteCount);
+
+ WriteVarUInt(uint64_t(Name.size()), Data.data() + NameLenOffset);
+
+ const uint8_t* NamePtr = reinterpret_cast<const uint8_t*>(Name.data());
+ Data.insert(Data.end(), NamePtr, NamePtr + Name.size());
+ return *this;
+}
+
+void
+CbWriter::SetNameOrAddString(const std::string_view NameOrValue)
+{
+ // A name is only written if it would begin a new field inside of an object.
+ if ((States.back().Flags & (StateFlags::Name | StateFlags::Field | StateFlags::Object)) == StateFlags::Object)
+ {
+ SetName(NameOrValue);
+ }
+ else
+ {
+ AddString(NameOrValue);
+ }
+}
+
+std::string_view
+CbWriter::GetActiveName() const
+{
+ const WriterState& State = States.back();
+ if ((State.Flags & StateFlags::Name) == StateFlags::Name)
+ {
+ const uint8_t* const EncodedName = Data.data() + State.Offset + sizeof(CbFieldType);
+ uint32_t NameLenByteCount;
+ const uint64_t NameLen = ReadVarUInt(EncodedName, NameLenByteCount);
+ const size_t ClampedNameLen = std::clamp<uint64_t>(NameLen, 0, ~uint64_t(0));
+ return std::string_view(reinterpret_cast<const char*>(EncodedName + NameLenByteCount), ClampedNameLen);
+ }
+ return std::string_view();
+}
+
+void
+CbWriter::MakeFieldsUniform(const int64_t FieldBeginOffset, const int64_t FieldEndOffset)
+{
+ MutableMemoryView SourceView(Data.data() + FieldBeginOffset, uint64_t(FieldEndOffset - FieldBeginOffset));
+ MutableMemoryView TargetView = SourceView;
+ TargetView.RightChopInline(sizeof(CbFieldType));
+
+ while (!SourceView.IsEmpty())
+ {
+ const uint64_t FieldSize = MeasureCompactBinary(SourceView) - sizeof(CbFieldType);
+ SourceView.RightChopInline(sizeof(CbFieldType));
+ if (TargetView.GetData() != SourceView.GetData())
+ {
+ memmove(TargetView.GetData(), SourceView.GetData(), FieldSize);
+ }
+ SourceView.RightChopInline(FieldSize);
+ TargetView.RightChopInline(FieldSize);
+ }
+
+ if (!TargetView.IsEmpty())
+ {
+ const auto EraseBegin = Data.begin() + (FieldEndOffset - TargetView.GetSize());
+ const auto EraseEnd = EraseBegin + TargetView.GetSize();
+
+ Data.erase(EraseBegin, EraseEnd);
+ }
+}
+
+void
+CbWriter::AddField(const CbFieldView& Value)
+{
+ ZEN_ASSERT(Value.HasValue()); // , TEXT("It is invalid to write a field with no value."));
+ BeginField();
+ EndField(AppendCompactBinary(Value, Data));
+}
+
+void
+CbWriter::AddField(const CbField& Value)
+{
+ AddField(CbFieldView(Value));
+}
+
+void
+CbWriter::BeginObject()
+{
+ BeginField();
+ States.push_back(WriterState());
+ States.back().Flags |= StateFlags::Object;
+}
+
+void
+CbWriter::EndObject()
+{
+ ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Object) == StateFlags::Object);
+
+ // TEXT("It is invalid to end an object when an object is not at the top of the stack."));
+ ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None);
+ // TEXT("It is invalid to end an object until the previous field is finished."));
+
+ const bool bUniform = IsUniformType(States.back().UniformType);
+ const uint64_t Count = States.back().Count;
+ States.pop_back();
+
+ // Calculate the offset of the payload.
+ const WriterState& State = States.back();
+ int64_t PayloadOffset = State.Offset + 1;
+ if ((State.Flags & StateFlags::Name) == StateFlags::Name)
+ {
+ uint32_t NameLenByteCount;
+ const uint64_t NameLen = ReadVarUInt(Data.data() + PayloadOffset, NameLenByteCount);
+ PayloadOffset += NameLen + NameLenByteCount;
+ }
+
+ // Remove redundant field types for uniform objects.
+ if (bUniform && Count > 1)
+ {
+ MakeFieldsUniform(PayloadOffset, Data.size());
+ }
+
+ // Insert the object size.
+ const uint64_t Size = uint64_t(Data.size() - PayloadOffset);
+ const uint32_t SizeByteCount = MeasureVarUInt(Size);
+ Data.insert(Data.begin() + PayloadOffset, SizeByteCount, 0);
+ WriteVarUInt(Size, Data.data() + PayloadOffset);
+
+ EndField(bUniform ? CbFieldType::UniformObject : CbFieldType::Object);
+}
+
+void
+CbWriter::AddObject(const CbObjectView& Value)
+{
+ BeginField();
+ EndField(AppendCompactBinary(Value.AsFieldView(), Data));
+}
+
+void
+CbWriter::AddObject(const CbObject& Value)
+{
+ AddObject(CbObjectView(Value));
+}
+
+ZEN_NOINLINE
+void
+CbWriter::BeginArray()
+{
+ BeginField();
+ States.push_back(WriterState());
+ States.back().Flags |= StateFlags::Array;
+}
+
+void
+CbWriter::EndArray()
+{
+ ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Array) == StateFlags::Array);
+ // TEXT("Invalid attempt to end an array when an array is not at the top of the stack."));
+ ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None);
+ // TEXT("It is invalid to end an array until the previous field is finished."));
+ const bool bUniform = IsUniformType(States.back().UniformType);
+ const uint64_t Count = States.back().Count;
+ States.pop_back();
+
+ // Calculate the offset of the payload.
+ const WriterState& State = States.back();
+ int64_t PayloadOffset = State.Offset + 1;
+ if ((State.Flags & StateFlags::Name) == StateFlags::Name)
+ {
+ uint32_t NameLenByteCount;
+ const uint64_t NameLen = ReadVarUInt(Data.data() + PayloadOffset, NameLenByteCount);
+ PayloadOffset += NameLen + NameLenByteCount;
+ }
+
+ // Remove redundant field types for uniform arrays.
+ if (bUniform && Count > 1)
+ {
+ MakeFieldsUniform(PayloadOffset, Data.size());
+ }
+
+ // Insert the array size and field count.
+ const uint32_t CountByteCount = MeasureVarUInt(Count);
+ const uint64_t Size = uint64_t(Data.size() - PayloadOffset) + CountByteCount;
+ const uint32_t SizeByteCount = MeasureVarUInt(Size);
+ Data.insert(Data.begin() + PayloadOffset, SizeByteCount + CountByteCount, 0);
+ WriteVarUInt(Size, Data.data() + PayloadOffset);
+ WriteVarUInt(Count, Data.data() + PayloadOffset + SizeByteCount);
+
+ EndField(bUniform ? CbFieldType::UniformArray : CbFieldType::Array);
+}
+
+void
+CbWriter::AddArray(const CbArrayView& Value)
+{
+ BeginField();
+ EndField(AppendCompactBinary(Value.AsFieldView(), Data));
+}
+
+void
+CbWriter::AddArray(const CbArray& Value)
+{
+ AddArray(CbArrayView(Value));
+}
+
+void
+CbWriter::AddNull()
+{
+ BeginField();
+ EndField(CbFieldType::Null);
+}
+
+void
+CbWriter::AddBinary(const void* const Value, const uint64_t Size)
+{
+ BeginField();
+ const uint32_t SizeByteCount = MeasureVarUInt(Size);
+ const int64_t SizeOffset = Data.size();
+ Data.resize(Data.size() + SizeByteCount);
+ WriteVarUInt(Size, Data.data() + SizeOffset);
+ Data.insert(Data.end(), static_cast<const uint8_t*>(Value), static_cast<const uint8_t*>(Value) + Size);
+ EndField(CbFieldType::Binary);
+}
+
+void
+CbWriter::AddBinary(IoBuffer Buffer)
+{
+ AddBinary(Buffer.Data(), Buffer.Size());
+}
+
+void
+CbWriter::AddBinary(SharedBuffer Buffer)
+{
+ AddBinary(Buffer.GetData(), Buffer.GetSize());
+}
+
+void
+CbWriter::AddString(const std::string_view Value)
+{
+ BeginField();
+ const uint64_t Size = uint64_t(Value.size());
+ const uint32_t SizeByteCount = MeasureVarUInt(Size);
+ const int64_t Offset = Data.size();
+
+ Data.resize(Offset + SizeByteCount + Size);
+
+ uint8_t* StringData = Data.data() + Offset;
+ WriteVarUInt(Size, StringData);
+ StringData += SizeByteCount;
+ if (Size > 0)
+ {
+ memcpy(StringData, Value.data(), Value.size() * sizeof(char));
+ }
+ EndField(CbFieldType::String);
+}
+
+void
+CbWriter::AddString(const std::wstring_view Value)
+{
+ BeginField();
+ ExtendableStringBuilder<128> Utf8;
+ WideToUtf8(Value, Utf8);
+
+ const uint32_t Size = uint32_t(Utf8.Size());
+ const uint32_t SizeByteCount = MeasureVarUInt(Size);
+ const int64_t Offset = Data.size();
+ Data.resize(Offset + SizeByteCount + Size);
+ uint8_t* StringData = Data.data() + Offset;
+ WriteVarUInt(Size, StringData);
+ StringData += SizeByteCount;
+ if (Size > 0)
+ {
+ memcpy(reinterpret_cast<char*>(StringData), Utf8.Data(), Utf8.Size());
+ }
+ EndField(CbFieldType::String);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddInteger(const int32_t Value)
+{
+ if (Value >= 0)
+ {
+ return AddInteger(uint32_t(Value));
+ }
+ BeginField();
+ const uint32_t Magnitude = ~uint32_t(Value);
+ const uint32_t MagnitudeByteCount = MeasureVarUInt(Magnitude);
+ const int64_t Offset = Data.size();
+ Data.resize(Offset + MagnitudeByteCount);
+ WriteVarUInt(Magnitude, Data.data() + Offset);
+ EndField(CbFieldType::IntegerNegative);
+}
+
+void
+CbWriter::AddInteger(const int64_t Value)
+{
+ if (Value >= 0)
+ {
+ return AddInteger(uint64_t(Value));
+ }
+ BeginField();
+ const uint64_t Magnitude = ~uint64_t(Value);
+ const uint32_t MagnitudeByteCount = MeasureVarUInt(Magnitude);
+ const uint64_t Offset = AddUninitialized(Data, MagnitudeByteCount);
+ WriteVarUInt(Magnitude, Data.data() + Offset);
+ EndField(CbFieldType::IntegerNegative);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddInteger(const uint32_t Value)
+{
+ BeginField();
+ const uint32_t ValueByteCount = MeasureVarUInt(Value);
+ const uint64_t Offset = AddUninitialized(Data, ValueByteCount);
+ WriteVarUInt(Value, Data.data() + Offset);
+ EndField(CbFieldType::IntegerPositive);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddInteger(const uint64_t Value)
+{
+ BeginField();
+ const uint32_t ValueByteCount = MeasureVarUInt(Value);
+ const uint64_t Offset = AddUninitialized(Data, ValueByteCount);
+ WriteVarUInt(Value, Data.data() + Offset);
+ EndField(CbFieldType::IntegerPositive);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddFloat(const float Value)
+{
+ BeginField();
+ const uint32_t RawValue = FromNetworkOrder(reinterpret_cast<const uint32_t&>(Value));
+ Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint32_t));
+ EndField(CbFieldType::Float32);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddFloat(const double Value)
+{
+ const float Value32 = float(Value);
+ if (Value == double(Value32))
+ {
+ return AddFloat(Value32);
+ }
+ BeginField();
+ const uint64_t RawValue = FromNetworkOrder(reinterpret_cast<const uint64_t&>(Value));
+ Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t));
+ EndField(CbFieldType::Float64);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddBool(const bool bValue)
+{
+ BeginField();
+ EndField(bValue ? CbFieldType::BoolTrue : CbFieldType::BoolFalse);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddCompactBinaryAttachment(const IoHash& Value)
+{
+ BeginField();
+ Append(Data, Value.Hash, sizeof Value.Hash);
+ EndField(CbFieldType::CompactBinaryAttachment);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddBinaryAttachment(const IoHash& Value)
+{
+ BeginField();
+ Append(Data, Value.Hash, sizeof Value.Hash);
+ EndField(CbFieldType::BinaryAttachment);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddAttachment(const CbAttachment& Attachment)
+{
+ BeginField();
+ const IoHash& Value = Attachment.GetHash();
+ Append(Data, Value.Hash, sizeof Value.Hash);
+ EndField(CbFieldType::BinaryAttachment);
+}
+
+ZEN_NOINLINE
+void
+CbWriter::AddHash(const IoHash& Value)
+{
+ BeginField();
+ Append(Data, Value.Hash, sizeof Value.Hash);
+ EndField(CbFieldType::Hash);
+}
+
+void
+CbWriter::AddUuid(const Guid& Value)
+{
+ const auto AppendSwappedBytes = [this](uint32_t In) {
+ In = FromNetworkOrder(In);
+ Append(Data, reinterpret_cast<const uint8_t*>(&In), sizeof In);
+ };
+ BeginField();
+ AppendSwappedBytes(Value.A);
+ AppendSwappedBytes(Value.B);
+ AppendSwappedBytes(Value.C);
+ AppendSwappedBytes(Value.D);
+ EndField(CbFieldType::Uuid);
+}
+
+void
+CbWriter::AddObjectId(const Oid& Value)
+{
+ BeginField();
+ Append(Data, reinterpret_cast<const uint8_t*>(&Value.OidBits), sizeof Value.OidBits);
+ EndField(CbFieldType::ObjectId);
+}
+
+void
+CbWriter::AddDateTimeTicks(const int64_t Ticks)
+{
+ BeginField();
+ const uint64_t RawValue = FromNetworkOrder(uint64_t(Ticks));
+ Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t));
+ EndField(CbFieldType::DateTime);
+}
+
+void
+CbWriter::AddDateTime(const DateTime Value)
+{
+ AddDateTimeTicks(Value.GetTicks());
+}
+
+void
+CbWriter::AddTimeSpanTicks(const int64_t Ticks)
+{
+ BeginField();
+ const uint64_t RawValue = FromNetworkOrder(uint64_t(Ticks));
+ Append(Data, reinterpret_cast<const uint8_t*>(&RawValue), sizeof(uint64_t));
+ EndField(CbFieldType::TimeSpan);
+}
+
+void
+CbWriter::AddTimeSpan(const TimeSpan Value)
+{
+ AddTimeSpanTicks(Value.GetTicks());
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+CbWriter&
+operator<<(CbWriter& Writer, const DateTime Value)
+{
+ Writer.AddDateTime(Value);
+ return Writer;
+}
+
+CbWriter&
+operator<<(CbWriter& Writer, const TimeSpan Value)
+{
+ Writer.AddTimeSpan(Value);
+ return Writer;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void
+usonbuilder_forcelink()
+{
+}
+
+doctest::String
+toString(const DateTime&)
+{
+ // TODO:implement
+ return "";
+}
+
+doctest::String
+toString(const TimeSpan&)
+{
+ // TODO:implement
+ return "";
+}
+
+TEST_CASE("usonbuilder.object")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("EmptyObject")
+ {
+ Writer.BeginObject();
+ Writer.EndObject();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsObject() == true);
+ CHECK(Field.AsObjectView().CreateViewIterator().HasValue() == false);
+ }
+
+ SUBCASE("NamedEmptyObject")
+ {
+ Writer.SetName("Object"sv);
+ Writer.BeginObject();
+ Writer.EndObject();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsObject() == true);
+ CHECK(Field.AsObjectView().CreateViewIterator().HasValue() == false);
+ }
+
+ SUBCASE("BasicObject")
+ {
+ Writer.BeginObject();
+ Writer.SetName("Integer"sv).AddInteger(0);
+ Writer.SetName("Float"sv).AddFloat(0.0f);
+ Writer.EndObject();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsObject() == true);
+
+ CbObjectView Object = Field.AsObjectView();
+ CHECK(Object["Integer"sv].IsInteger() == true);
+ CHECK(Object["Float"sv].IsFloat() == true);
+ }
+
+ SUBCASE("UniformObject")
+ {
+ Writer.BeginObject();
+ Writer.SetName("Field1"sv).AddInteger(0);
+ Writer.SetName("Field2"sv).AddInteger(1);
+ Writer.EndObject();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsObject() == true);
+
+ CbObjectView Object = Field.AsObjectView();
+ CHECK(Object["Field1"sv].IsInteger() == true);
+ CHECK(Object["Field2"sv].IsInteger() == true);
+ }
+}
+
+TEST_CASE("usonbuilder.array")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("EmptyArray")
+ {
+ Writer.BeginArray();
+ Writer.EndArray();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsArray() == true);
+ CHECK(Field.AsArrayView().Num() == 0);
+ }
+
+ SUBCASE("NamedEmptyArray")
+ {
+ Writer.SetName("Array"sv);
+ Writer.BeginArray();
+ Writer.EndArray();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsArray() == true);
+ CHECK(Field.AsArrayView().Num() == 0);
+ }
+
+ SUBCASE("BasicArray")
+ {
+ Writer.BeginArray();
+ Writer.AddInteger(0);
+ Writer.AddFloat(0.0f);
+ Writer.EndArray();
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsArray() == true);
+ CbFieldViewIterator Iterator = Field.AsArrayView().CreateViewIterator();
+ CHECK(Iterator.IsInteger() == true);
+ ++Iterator;
+ CHECK(Iterator.IsFloat() == true);
+ ++Iterator;
+ CHECK(Iterator.HasValue() == false);
+ }
+
+ SUBCASE("UniformArray")
+ {
+ Writer.BeginArray();
+ Writer.AddInteger(0);
+ Writer.AddInteger(1);
+ Writer.EndArray();
+
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.IsArray() == true);
+ CbFieldViewIterator Iterator = Field.AsArrayView().CreateViewIterator();
+ CHECK(Iterator.IsInteger() == true);
+ ++Iterator;
+ CHECK(Iterator.IsInteger() == true);
+ ++Iterator;
+ CHECK(Iterator.HasValue() == false);
+ }
+}
+
+TEST_CASE("usonbuilder.null")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("Null")
+ {
+ Writer.AddNull();
+ CbField Field = Writer.Save();
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.HasName() == false);
+ CHECK(Field.IsNull() == true);
+ }
+
+ SUBCASE("NullWithName")
+ {
+ Writer.SetName("Null"sv);
+ Writer.AddNull();
+ CbField Field = Writer.Save();
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ CHECK(Field.HasName() == true);
+ CHECK(Field.GetName().compare("Null"sv) == 0);
+ CHECK(Field.IsNull() == true);
+ }
+
+ SUBCASE("Null Array/Object Uniformity")
+ {
+ Writer.BeginArray();
+ Writer.AddNull();
+ Writer.AddNull();
+ Writer.AddNull();
+ Writer.EndArray();
+
+ Writer.BeginObject();
+ Writer.SetName("N1"sv).AddNull();
+ Writer.SetName("N2"sv).AddNull();
+ Writer.SetName("N3"sv).AddNull();
+ Writer.EndObject();
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ }
+
+ SUBCASE("Null with Save(Buffer)")
+ {
+ constexpr int NullCount = 3;
+ for (int Index = 0; Index < NullCount; ++Index)
+ {
+ Writer.AddNull();
+ }
+ uint8_t Buffer[NullCount]{};
+ MutableMemoryView BufferView(Buffer, sizeof Buffer);
+ CbFieldViewIterator Fields = Writer.Save(BufferView);
+
+ CHECK(ValidateCompactBinaryRange(BufferView, CbValidateMode::All) == CbValidateError::None);
+
+ for (int Index = 0; Index < NullCount; ++Index)
+ {
+ CHECK(Fields.IsNull() == true);
+ ++Fields;
+ }
+ CHECK(Fields.HasValue() == false);
+ }
+}
+
+TEST_CASE("usonbuilder.binary")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+}
+
+TEST_CASE("usonbuilder.string")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("Empty Strings")
+ {
+ Writer.AddString(std::string_view());
+ Writer.AddString(std::wstring_view());
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ for (CbFieldView Field : Fields)
+ {
+ CHECK(Field.HasName() == false);
+ CHECK(Field.IsString() == true);
+ CHECK(Field.AsString().empty() == true);
+ }
+ }
+
+ SUBCASE("Test Basic Strings")
+ {
+ Writer.SetName("String"sv).AddString("Value"sv);
+ Writer.SetName("String"sv).AddString(L"Value"sv);
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ for (CbFieldView Field : Fields)
+ {
+ CHECK(Field.GetName().compare("String"sv) == 0);
+ CHECK(Field.HasName() == true);
+ CHECK(Field.IsString() == true);
+ CHECK(Field.AsString().compare("Value"sv) == 0);
+ }
+ }
+
+ SUBCASE("Long Strings")
+ {
+ constexpr int DotCount = 256;
+ StringBuilder<DotCount + 1> Dots;
+ for (int Index = 0; Index < DotCount; ++Index)
+ {
+ Dots.Append('.');
+ }
+ Writer.AddString(Dots);
+ Writer.AddString(std::wstring().append(256, L'.'));
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ for (CbFieldView Field : Fields)
+ {
+ CHECK((Field.AsString() == std::string_view(Dots)));
+ }
+ }
+
+ SUBCASE("Non-ASCII String")
+ {
+ wchar_t Value[2] = {0xd83d, 0xde00};
+ Writer.AddString("\xf0\x9f\x98\x80"sv);
+ Writer.AddString(std::wstring_view(Value, ZEN_ARRAY_COUNT(Value)));
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ for (CbFieldView Field : Fields)
+ {
+ CHECK((Field.AsString() == "\xf0\x9f\x98\x80"sv));
+ }
+ }
+}
+
+TEST_CASE("usonbuilder.integer")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ auto TestInt32 = [&Writer](int32_t Value) {
+ Writer.Reset();
+ Writer.AddInteger(Value);
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ CHECK(Field.AsInt32() == Value);
+ CHECK(Field.HasError() == false);
+ };
+
+ auto TestUInt32 = [&Writer](uint32_t Value) {
+ Writer.Reset();
+ Writer.AddInteger(Value);
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ CHECK(Field.AsUInt32() == Value);
+ CHECK(Field.HasError() == false);
+ };
+
+ auto TestInt64 = [&Writer](int64_t Value) {
+ Writer.Reset();
+ Writer.AddInteger(Value);
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ CHECK(Field.AsInt64() == Value);
+ CHECK(Field.HasError() == false);
+ };
+
+ auto TestUInt64 = [&Writer](uint64_t Value) {
+ Writer.Reset();
+ Writer.AddInteger(Value);
+ CbField Field = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Field.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ CHECK(Field.AsUInt64() == Value);
+ CHECK(Field.HasError() == false);
+ };
+
+ TestUInt32(uint32_t(0x00));
+ TestUInt32(uint32_t(0x7f));
+ TestUInt32(uint32_t(0x80));
+ TestUInt32(uint32_t(0xff));
+ TestUInt32(uint32_t(0x0100));
+ TestUInt32(uint32_t(0x7fff));
+ TestUInt32(uint32_t(0x8000));
+ TestUInt32(uint32_t(0xffff));
+ TestUInt32(uint32_t(0x0001'0000));
+ TestUInt32(uint32_t(0x7fff'ffff));
+ TestUInt32(uint32_t(0x8000'0000));
+ TestUInt32(uint32_t(0xffff'ffff));
+
+ TestUInt64(uint64_t(0x0000'0001'0000'0000));
+ TestUInt64(uint64_t(0x7fff'ffff'ffff'ffff));
+ TestUInt64(uint64_t(0x8000'0000'0000'0000));
+ TestUInt64(uint64_t(0xffff'ffff'ffff'ffff));
+
+ TestInt32(int32_t(0x01));
+ TestInt32(int32_t(0x80));
+ TestInt32(int32_t(0x81));
+ TestInt32(int32_t(0x8000));
+ TestInt32(int32_t(0x8001));
+ TestInt32(int32_t(0x7fff'ffff));
+ TestInt32(int32_t(0x8000'0000));
+ TestInt32(int32_t(0x8000'0001));
+
+ TestInt64(int64_t(0x0000'0001'0000'0000));
+ TestInt64(int64_t(0x8000'0000'0000'0000));
+ TestInt64(int64_t(0x7fff'ffff'ffff'ffff));
+ TestInt64(int64_t(0x8000'0000'0000'0001));
+ TestInt64(int64_t(0xffff'ffff'ffff'ffff));
+}
+
+TEST_CASE("usonbuilder.float")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("Float32")
+ {
+ constexpr float Values[] = {
+ 0.0f,
+ 1.0f,
+ -1.0f,
+ 3.14159265358979323846f, // PI
+ 3.402823466e+38f, // FLT_MAX
+ 1.175494351e-38f // FLT_MIN
+ };
+
+ for (float Value : Values)
+ {
+ Writer.AddFloat(Value);
+ }
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ const float* CheckValue = Values;
+ for (CbFieldView Field : Fields)
+ {
+ CHECK(Field.AsFloat() == *CheckValue++);
+ CHECK(Field.HasError() == false);
+ }
+ }
+
+ SUBCASE("Float64")
+ {
+ constexpr double Values[] = {
+ 0.0f,
+ 1.0f,
+ -1.0f,
+ 3.14159265358979323846, // PI
+ 1.9999998807907104,
+ 1.9999999403953552,
+ 3.4028234663852886e38,
+ 6.8056469327705771e38,
+ 2.2250738585072014e-308, // DBL_MIN
+ 1.7976931348623158e+308 // DBL_MAX
+ };
+
+ for (double Value : Values)
+ {
+ Writer.AddFloat(Value);
+ }
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ const double* CheckValue = Values;
+ for (CbFieldView Field : Fields)
+ {
+ CHECK(Field.AsDouble() == *CheckValue++);
+ CHECK(Field.HasError() == false);
+ }
+ }
+}
+
+TEST_CASE("usonbuilder.bool")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("Bool")
+ {
+ Writer.AddBool(true);
+ Writer.AddBool(false);
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ CHECK(Fields.AsBool() == true);
+ CHECK(Fields.HasError() == false);
+ ++Fields;
+ CHECK(Fields.AsBool() == false);
+ CHECK(Fields.HasError() == false);
+ ++Fields;
+ CHECK(Fields.HasValue() == false);
+ }
+
+ SUBCASE("Bool Array/Object Uniformity")
+ {
+ Writer.BeginArray();
+ Writer.AddBool(false);
+ Writer.AddBool(false);
+ Writer.AddBool(false);
+ Writer.EndArray();
+
+ Writer.BeginObject();
+ Writer.SetName("B1"sv).AddBool(false);
+ Writer.SetName("B2"sv).AddBool(false);
+ Writer.SetName("B3"sv).AddBool(false);
+ Writer.EndObject();
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ }
+}
+
+TEST_CASE("usonbuilder.usonattachment")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+}
+
+TEST_CASE("usonbuilder.binaryattachment")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+}
+
+TEST_CASE("usonbuilder.hash")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+}
+
+TEST_CASE("usonbuilder.uuid")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+}
+
+TEST_CASE("usonbuilder.datetime")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ const DateTime Values[] = {DateTime(0), DateTime(2020, 5, 13, 15, 10)};
+ for (DateTime Value : Values)
+ {
+ Writer.AddDateTime(Value);
+ }
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ const DateTime* CheckValue = Values;
+ for (CbFieldView Field : Fields)
+ {
+ CHECK(Field.AsDateTime() == *CheckValue++);
+ CHECK(Field.HasError() == false);
+ }
+}
+
+TEST_CASE("usonbuilder.timespan")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ const TimeSpan Values[] = {TimeSpan(0), TimeSpan(1, 2, 4, 8)};
+ for (TimeSpan Value : Values)
+ {
+ Writer.AddTimeSpan(Value);
+ }
+
+ CbFieldIterator Fields = Writer.Save();
+
+ CHECK(ValidateCompactBinary(Fields.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+
+ const TimeSpan* CheckValue = Values;
+ for (CbFieldView Field : Fields)
+ {
+ CHECK(Field.AsTimeSpan() == *CheckValue++);
+ CHECK(Field.HasError() == false);
+ }
+}
+
+TEST_CASE("usonbuilder.complex")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("complex")
+ {
+ CbObject Object;
+
+ {
+ Writer.BeginObject();
+
+ const uint8_t LocalField[] = {uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), 1, 'I', 42};
+ Writer.AddField("FieldCopy"sv, CbFieldView(LocalField));
+ Writer.AddField("FieldRefCopy"sv, CbField(SharedBuffer::Clone(MakeMemoryView(LocalField))));
+
+ const uint8_t LocalObject[] = {uint8_t(CbFieldType::Object | CbFieldType::HasFieldName),
+ 1,
+ 'O',
+ 7,
+ uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName),
+ 1,
+ 'I',
+ 42,
+ uint8_t(CbFieldType::Null | CbFieldType::HasFieldName),
+ 1,
+ 'N'};
+ Writer.AddObject("ObjectCopy"sv, CbObjectView(LocalObject));
+ Writer.AddObject("ObjectRefCopy"sv, CbObject(SharedBuffer::Clone(MakeMemoryView(LocalObject))));
+
+ const uint8_t LocalArray[] = {uint8_t(CbFieldType::UniformArray | CbFieldType::HasFieldName),
+ 1,
+ 'A',
+ 4,
+ 2,
+ uint8_t(CbFieldType::IntegerPositive),
+ 42,
+ 21};
+ Writer.AddArray("ArrayCopy"sv, CbArrayView(LocalArray));
+ Writer.AddArray("ArrayRefCopy"sv, CbArray(SharedBuffer::Clone(MakeMemoryView(LocalArray))));
+
+ Writer.AddNull("Null"sv);
+
+ Writer.BeginObject("Binary"sv);
+ {
+ Writer.AddBinary("Empty"sv, MemoryView());
+ Writer.AddBinary("Value"sv, MakeMemoryView("BinaryValue"));
+ Writer.AddBinary("LargeValue"sv, MakeMemoryView(std::wstring().append(256, L'.')));
+ Writer.AddBinary("LargeRefValue"sv, SharedBuffer::Clone(MakeMemoryView(std::wstring().append(256, L'!'))));
+ }
+ Writer.EndObject();
+
+ Writer.BeginObject("Strings"sv);
+ {
+ Writer.AddString("AnsiString"sv, "AnsiValue"sv);
+ Writer.AddString("WideString"sv, std::wstring().append(256, L'.'));
+ Writer.AddString("EmptyAnsiString"sv, std::string_view());
+ Writer.AddString("EmptyWideString"sv, std::wstring_view());
+ Writer.AddString("AnsiStringLiteral", "AnsiValue");
+ Writer.AddString("WideStringLiteral", L"AnsiValue");
+ }
+ Writer.EndObject();
+
+ Writer.BeginArray("Integers"sv);
+ {
+ Writer.AddInteger(int32_t(-1));
+ Writer.AddInteger(int64_t(-1));
+ Writer.AddInteger(uint32_t(1));
+ Writer.AddInteger(uint64_t(1));
+ Writer.AddInteger(std::numeric_limits<int32_t>::min());
+ Writer.AddInteger(std::numeric_limits<int32_t>::max());
+ Writer.AddInteger(std::numeric_limits<uint32_t>::max());
+ Writer.AddInteger(std::numeric_limits<int64_t>::min());
+ Writer.AddInteger(std::numeric_limits<int64_t>::max());
+ Writer.AddInteger(std::numeric_limits<uint64_t>::max());
+ }
+ Writer.EndArray();
+
+ Writer.BeginArray("UniformIntegers"sv);
+ {
+ Writer.AddInteger(0);
+ Writer.AddInteger(std::numeric_limits<int32_t>::max());
+ Writer.AddInteger(std::numeric_limits<uint32_t>::max());
+ Writer.AddInteger(std::numeric_limits<int64_t>::max());
+ Writer.AddInteger(std::numeric_limits<uint64_t>::max());
+ }
+ Writer.EndArray();
+
+ Writer.AddFloat("Float32"sv, 1.0f);
+ Writer.AddFloat("Float64as32"sv, 2.0);
+ Writer.AddFloat("Float64"sv, 3.0e100);
+
+ Writer.AddBool("False"sv, false);
+ Writer.AddBool("True"sv, true);
+
+ Writer.AddCompactBinaryAttachment("CompactBinaryAttachment"sv, IoHash());
+ Writer.AddBinaryAttachment("BinaryAttachment"sv, IoHash());
+ Writer.AddAttachment("Attachment"sv, CbAttachment());
+
+ Writer.AddHash("Hash"sv, IoHash());
+ Writer.AddUuid("Uuid"sv, Guid());
+
+ Writer.AddDateTimeTicks("DateTimeZero"sv, 0);
+ Writer.AddDateTime("DateTime2020"sv, DateTime(2020, 5, 13, 15, 10));
+
+ Writer.AddTimeSpanTicks("TimeSpanZero"sv, 0);
+ Writer.AddTimeSpan("TimeSpan"sv, TimeSpan(1, 2, 4, 8));
+
+ Writer.BeginObject("NestedObjects"sv);
+ {
+ Writer.BeginObject("Empty"sv);
+ Writer.EndObject();
+
+ Writer.BeginObject("Null"sv);
+ Writer.AddNull("Null"sv);
+ Writer.EndObject();
+ }
+ Writer.EndObject();
+
+ Writer.BeginArray("NestedArrays"sv);
+ {
+ Writer.BeginArray();
+ Writer.EndArray();
+
+ Writer.BeginArray();
+ Writer.AddNull();
+ Writer.AddNull();
+ Writer.AddNull();
+ Writer.EndArray();
+
+ Writer.BeginArray();
+ Writer.AddBool(false);
+ Writer.AddBool(false);
+ Writer.AddBool(false);
+ Writer.EndArray();
+
+ Writer.BeginArray();
+ Writer.AddBool(true);
+ Writer.AddBool(true);
+ Writer.AddBool(true);
+ Writer.EndArray();
+ }
+ Writer.EndArray();
+
+ Writer.BeginArray("ArrayOfObjects"sv);
+ {
+ Writer.BeginObject();
+ Writer.EndObject();
+
+ Writer.BeginObject();
+ Writer.AddNull("Null"sv);
+ Writer.EndObject();
+ }
+ Writer.EndArray();
+
+ Writer.BeginArray("LargeArray"sv);
+ for (int Index = 0; Index < 256; ++Index)
+ {
+ Writer.AddInteger(Index - 128);
+ }
+ Writer.EndArray();
+
+ Writer.BeginArray("LargeUniformArray"sv);
+ for (int Index = 0; Index < 256; ++Index)
+ {
+ Writer.AddInteger(Index);
+ }
+ Writer.EndArray();
+
+ Writer.BeginArray("NestedUniformArray"sv);
+ for (int Index = 0; Index < 16; ++Index)
+ {
+ Writer.BeginArray();
+ for (int Value = 0; Value < 4; ++Value)
+ {
+ Writer.AddInteger(Value);
+ }
+ Writer.EndArray();
+ }
+ Writer.EndArray();
+
+ Writer.EndObject();
+ Object = Writer.Save().AsObject();
+ }
+ CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ }
+}
+
+TEST_CASE("usonbuilder.stream")
+{
+ using namespace std::literals;
+
+ FixedCbWriter<256> Writer;
+
+ SUBCASE("basic")
+ {
+ CbObject Object;
+ {
+ Writer.BeginObject();
+
+ const uint8_t LocalField[] = {uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName), 1, 'I', 42};
+ Writer << "FieldCopy"sv << CbFieldView(LocalField);
+
+ const uint8_t LocalObject[] = {uint8_t(CbFieldType::Object | CbFieldType::HasFieldName),
+ 1,
+ 'O',
+ 7,
+ uint8_t(CbFieldType::IntegerPositive | CbFieldType::HasFieldName),
+ 1,
+ 'I',
+ 42,
+ uint8_t(CbFieldType::Null | CbFieldType::HasFieldName),
+ 1,
+ 'N'};
+ Writer << "ObjectCopy"sv << CbObjectView(LocalObject);
+
+ const uint8_t LocalArray[] = {uint8_t(CbFieldType::UniformArray | CbFieldType::HasFieldName),
+ 1,
+ 'A',
+ 4,
+ 2,
+ uint8_t(CbFieldType::IntegerPositive),
+ 42,
+ 21};
+ Writer << "ArrayCopy"sv << CbArrayView(LocalArray);
+
+ Writer << "Null"sv << nullptr;
+
+ Writer << "Strings"sv;
+ Writer.BeginObject();
+ Writer << "AnsiString"sv
+ << "AnsiValue"sv
+ << "AnsiStringLiteral"sv
+ << "AnsiValue"
+ << "WideString"sv << L"WideValue"sv << "WideStringLiteral"sv << L"WideValue";
+ Writer.EndObject();
+
+ Writer << "Integers"sv;
+ Writer.BeginArray();
+ Writer << int32_t(-1) << int64_t(-1) << uint32_t(1) << uint64_t(1);
+ Writer.EndArray();
+
+ Writer << "Float32"sv << 1.0f;
+ Writer << "Float64"sv << 2.0;
+
+ Writer << "False"sv << false << "True"sv << true;
+
+ Writer << "Attachment"sv << CbAttachment();
+
+ Writer << "Hash"sv << IoHash();
+ Writer << "Uuid"sv << Guid();
+
+ Writer << "DateTime"sv << DateTime(2020, 5, 13, 15, 10);
+ Writer << "TimeSpan"sv << TimeSpan(1, 2, 4, 8);
+
+ Writer << "LiteralName" << nullptr;
+
+ Writer.EndObject();
+ Object = Writer.Save().AsObject();
+ }
+
+ CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None);
+ }
+}
+
+} // namespace zen
diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
new file mode 100644
index 000000000..21883f5b1
--- /dev/null
+++ b/zencore/compactbinarypackage.cpp
@@ -0,0 +1,945 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/compactbinarypackage.h"
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/endian.h>
+#include <zencore/stream.h>
+#include <zencore/trace.h>
+
+#include <doctest/doctest.h>
+
+namespace zen {
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+CbAttachment::CbAttachment(CbFieldIterator InValue, const IoHash* const InHash)
+{
+ if (InValue)
+ {
+ if (!InValue.IsOwned())
+ {
+ InValue = CbFieldIterator::CloneRange(InValue);
+ }
+
+ CompactBinary = CbFieldViewIterator(InValue);
+ Buffer = std::move(InValue).GetOuterBuffer();
+ }
+
+ if (InHash)
+ {
+ Hash = *InHash;
+ if (CompactBinary)
+ {
+ ZEN_ASSERT_SLOW(Hash == CompactBinary.GetRangeHash());
+ }
+ else
+ {
+#if 0
+ zenfs_assertSlow(Hash.IsZero(), TEXT("A null or empty field range must use a hash of zero."));
+#endif
+ }
+ }
+ else if (CompactBinary)
+ {
+ Hash = CompactBinary.GetRangeHash();
+ }
+}
+
+CbAttachment::CbAttachment(SharedBuffer InBuffer, const IoHash* const InHash) : Buffer(std::move(InBuffer))
+{
+ Buffer.MakeOwned();
+ if (InHash)
+ {
+ Hash = *InHash;
+ if (Buffer.GetSize())
+ {
+ ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()));
+ }
+ else
+ {
+ ZEN_ASSERT_SLOW(Hash == IoHash::Zero, TEXT("A null or empty buffer must use a hash of zero."));
+ }
+ }
+ else if (Buffer.GetSize())
+ {
+ Hash = IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize());
+ }
+ else
+ {
+ Buffer.Reset();
+ }
+}
+
+SharedBuffer
+CbAttachment::AsBinaryView() const
+{
+ if (!CompactBinary)
+ {
+ return Buffer;
+ }
+
+ MemoryView SerializedView;
+ if (CompactBinary.TryGetSerializedRangeView(SerializedView))
+ {
+ return SerializedView == Buffer.GetView() ? Buffer : SharedBuffer::MakeView(SerializedView, Buffer);
+ }
+
+ return CbFieldIterator::CloneRange(CompactBinary).GetRangeBuffer();
+}
+
+CbFieldIterator
+CbAttachment::AsCompactBinary() const
+{
+ return CompactBinary ? CbFieldIterator::MakeRangeView(CompactBinary, Buffer) : CbFieldIterator();
+}
+
+void
+CbAttachment::Load(IoBuffer& InBuffer, BufferAllocator Allocator)
+{
+ MemoryInStream InStream(InBuffer.Data(), InBuffer.Size());
+ BinaryReader Reader(InStream);
+
+ Load(Reader, Allocator);
+}
+
+void
+CbAttachment::Load(CbFieldIterator& Fields)
+{
+ ZEN_ASSERT(Fields.IsBinary()); //, TEXT("Attachments must start with a binary field."));
+ const MemoryView View = Fields.AsBinaryView();
+ if (View.GetSize() > 0)
+ {
+ Buffer = SharedBuffer::MakeView(View, Fields.GetOuterBuffer());
+ Buffer.MakeOwned();
+ ++Fields;
+ Hash = Fields.AsAttachment();
+ ZEN_ASSERT(!Fields.HasError()); // TEXT("Attachments must be a non-empty binary value with a content hash."));
+ if (Fields.IsCompactBinaryAttachment())
+ {
+ CompactBinary = CbFieldViewIterator::MakeRange(Buffer);
+ }
+ ++Fields;
+ }
+ else
+ {
+ ++Fields;
+ Buffer.Reset();
+ CompactBinary.Reset();
+ Hash = IoHash::Zero;
+ }
+}
+
+void
+CbAttachment::Load(BinaryReader& Reader, BufferAllocator Allocator)
+{
+ CbField BufferField = LoadCompactBinary(Reader, Allocator);
+ ZEN_ASSERT(BufferField.IsBinary(), "Attachments must start with a binary field");
+ const MemoryView View = BufferField.AsBinaryView();
+ if (View.GetSize() > 0)
+ {
+ Buffer = SharedBuffer::MakeView(View, BufferField.GetOuterBuffer());
+ Buffer.MakeOwned();
+ CompactBinary = CbFieldViewIterator();
+
+ std::vector<uint8_t> HashBuffer;
+ CbField HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer {
+ HashBuffer.resize(Size);
+ return UniqueBuffer::MakeView(HashBuffer.data(), Size);
+ });
+ Hash = HashField.AsAttachment();
+ ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash.");
+ if (HashField.IsCompactBinaryAttachment())
+ {
+ CompactBinary = CbFieldViewIterator::MakeRange(Buffer);
+ }
+ }
+ else
+ {
+ Buffer.Reset();
+ CompactBinary.Reset();
+ Hash = IoHash::Zero;
+ }
+}
+
+void
+CbAttachment::Save(CbWriter& Writer) const
+{
+ if (CompactBinary)
+ {
+ MemoryView SerializedView;
+ if (CompactBinary.TryGetSerializedRangeView(SerializedView))
+ {
+ Writer.AddBinary(SerializedView);
+ }
+ else
+ {
+ Writer.AddBinary(AsBinaryView());
+ }
+ Writer.AddCompactBinaryAttachment(Hash);
+ }
+ else if (Buffer && Buffer.GetSize())
+ {
+ Writer.AddBinary(Buffer);
+ Writer.AddBinaryAttachment(Hash);
+ }
+ else // Null
+ {
+ Writer.AddBinary(MemoryView());
+ }
+}
+
+void
+CbAttachment::Save(BinaryWriter& Writer) const
+{
+ CbWriter TempWriter;
+ Save(TempWriter);
+ TempWriter.Save(Writer);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void
+CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentResolver* InResolver)
+{
+ if (InObject.CreateIterator())
+ {
+ Object = InObject.IsOwned() ? std::move(InObject) : CbObject::Clone(InObject);
+ if (InObjectHash)
+ {
+ ObjectHash = *InObjectHash;
+ ZEN_ASSERT_SLOW(ObjectHash == Object.GetHash());
+ }
+ else
+ {
+ ObjectHash = Object.GetHash();
+ }
+ if (InResolver)
+ {
+ GatherAttachments(Object.CreateIterator(), *InResolver);
+ }
+ }
+ else
+ {
+ Object.Reset();
+ ObjectHash = IoHash::Zero;
+ }
+}
+
+void
+CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver)
+{
+ if (!Attachment.IsNull())
+ {
+ auto It = std::lower_bound(begin(Attachments), end(Attachments), Attachment);
+ if (It != Attachments.end() && *It == Attachment)
+ {
+ CbAttachment& Existing = *It;
+ if (Attachment.IsCompactBinary() && !Existing.IsCompactBinary())
+ {
+ Existing = CbAttachment(CbFieldIterator::MakeRange(Existing.AsBinaryView()));
+ }
+ }
+ else
+ {
+ Attachments.insert(It, Attachment);
+ }
+
+ if (Attachment.IsCompactBinary() && Resolver)
+ {
+ GatherAttachments(Attachment.AsCompactBinary(), *Resolver);
+ }
+ }
+}
+
+int32_t
+CbPackage::RemoveAttachment(const IoHash& Hash)
+{
+ return gsl::narrow_cast<int32_t>(
+ std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; }));
+}
+
+bool
+CbPackage::Equals(const CbPackage& Package) const
+{
+ return ObjectHash == Package.ObjectHash && Attachments == Package.Attachments;
+}
+
+const CbAttachment*
+CbPackage::FindAttachment(const IoHash& Hash) const
+{
+ auto It = std::find_if(begin(Attachments), end(Attachments), [&Hash](const CbAttachment& Attachment) -> bool {
+ return Attachment.GetHash() == Hash;
+ });
+
+ if (It == end(Attachments))
+ return nullptr;
+
+ return &*It;
+}
+
+void
+CbPackage::GatherAttachments(const CbFieldViewIterator& Fields, AttachmentResolver Resolver)
+{
+ Fields.IterateRangeAttachments([this, &Resolver](CbFieldView Field) {
+ const IoHash& Hash = Field.AsAttachment();
+
+ if (SharedBuffer Buffer = Resolver(Hash))
+ {
+ if (Field.IsCompactBinaryAttachment())
+ {
+ AddAttachment(CbAttachment(CbFieldIterator::MakeRange(std::move(Buffer)), Hash), &Resolver);
+ }
+ else
+ {
+ AddAttachment(CbAttachment(std::move(Buffer), Hash));
+ }
+ }
+ });
+}
+
+void
+CbPackage::Load(IoBuffer& InBuffer, BufferAllocator Allocator)
+{
+ MemoryInStream InStream(InBuffer.Data(), InBuffer.Size());
+ BinaryReader Reader(InStream);
+
+ Load(Reader, Allocator);
+}
+
+void
+CbPackage::Load(CbFieldIterator& Fields)
+{
+ *this = CbPackage();
+ while (Fields)
+ {
+ if (Fields.IsNull())
+ {
+ ++Fields;
+ break;
+ }
+ else if (Fields.IsBinary())
+ {
+ CbAttachment Attachment;
+ Attachment.Load(Fields);
+ AddAttachment(Attachment);
+ }
+ else
+ {
+ ZEN_ASSERT(Fields.IsObject(), TEXT("Expected Object, Binary, or Null field when loading a package."));
+ Object = Fields.AsObject();
+ Object.MakeOwned();
+ ++Fields;
+ if (Object.CreateIterator())
+ {
+ ObjectHash = Fields.AsCompactBinaryAttachment();
+ ZEN_ASSERT(!Fields.HasError(), TEXT("Object must be followed by a CompactBinaryReference with the object hash."));
+ ++Fields;
+ }
+ else
+ {
+ Object.Reset();
+ }
+ }
+ }
+}
+
+void
+CbPackage::Load(BinaryReader& Reader, BufferAllocator Allocator)
+{
+ uint8_t StackBuffer[64];
+ const auto StackAllocator = [&Allocator, &StackBuffer](uint64_t Size) -> UniqueBuffer {
+ if (Size <= sizeof(StackBuffer))
+ {
+ return UniqueBuffer::MakeView(StackBuffer, Size);
+ }
+
+ return Allocator(Size);
+ };
+
+ *this = CbPackage();
+
+ for (;;)
+ {
+ CbField ValueField = LoadCompactBinary(Reader, StackAllocator);
+ if (ValueField.IsNull())
+ {
+ break;
+ }
+ else if (ValueField.IsBinary())
+ {
+ const MemoryView View = ValueField.AsBinaryView();
+ if (View.GetSize() > 0)
+ {
+ SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer());
+ Buffer.MakeOwned();
+ CbField HashField = LoadCompactBinary(Reader, StackAllocator);
+ const IoHash& Hash = HashField.AsAttachment();
+ ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash.");
+ if (HashField.IsCompactBinaryAttachment())
+ {
+ AddAttachment(CbAttachment(CbFieldIterator::MakeRange(std::move(Buffer)), Hash));
+ }
+ else
+ {
+ AddAttachment(CbAttachment(std::move(Buffer), Hash));
+ }
+ }
+ }
+ else
+ {
+ ZEN_ASSERT(ValueField.IsObject(), "Expected Object, Binary, or Null field when loading a package");
+ Object = ValueField.AsObject();
+ Object.MakeOwned();
+ if (Object.CreateViewIterator())
+ {
+ CbField HashField = LoadCompactBinary(Reader, StackAllocator);
+ ObjectHash = HashField.AsCompactBinaryAttachment();
+ ZEN_ASSERT(!HashField.HasError(), "Object must be followed by a CompactBinaryAttachment with the object hash.");
+ }
+ else
+ {
+ Object.Reset();
+ }
+ }
+ }
+}
+
+void
+CbPackage::Save(CbWriter& Writer) const
+{
+ if (Object.CreateIterator())
+ {
+ Writer.AddObject(Object);
+ Writer.AddCompactBinaryAttachment(ObjectHash);
+ }
+ for (const CbAttachment& Attachment : Attachments)
+ {
+ Attachment.Save(Writer);
+ }
+ Writer.AddNull();
+}
+
+void
+CbPackage::Save(BinaryWriter& StreamWriter) const
+{
+ CbWriter Writer;
+ Save(Writer);
+ Writer.Save(StreamWriter);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void
+usonpackage_forcelink()
+{
+}
+
+TEST_CASE("usonpackage")
+{
+ using namespace std::literals;
+
+ const auto TestSaveLoadValidate = [&](const char* Test, const CbAttachment& Attachment) {
+ ZEN_UNUSED(Test);
+
+ CbWriter Writer;
+ Attachment.Save(Writer);
+ CbFieldIterator Fields = Writer.Save();
+
+ MemoryOutStream WriteStream;
+ BinaryWriter StreamWriter{WriteStream};
+ Attachment.Save(StreamWriter);
+
+ CHECK(MakeMemoryView(WriteStream).EqualBytes(Fields.GetRangeBuffer().GetView()));
+ CHECK(ValidateCompactBinaryRange(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None);
+ CHECK(ValidateCompactBinaryAttachment(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None);
+
+ CbAttachment FromFields;
+ FromFields.Load(Fields);
+ CHECK(!bool(Fields));
+ CHECK(FromFields == Attachment);
+
+ CbAttachment FromArchive;
+ MemoryInStream InStream(MakeMemoryView(WriteStream));
+ BinaryReader Reader(InStream);
+ FromArchive.Load(Reader);
+ CHECK(Reader.CurrentOffset() == InStream.Size());
+ CHECK(FromArchive == Attachment);
+ };
+
+ SUBCASE("Empty Attachment")
+ {
+ CbAttachment Attachment;
+ CHECK(Attachment.IsNull());
+ CHECK_FALSE(bool(Attachment));
+ CHECK_FALSE(bool(Attachment.AsBinaryView()));
+ CHECK_FALSE(Attachment.AsCompactBinary().HasValue());
+ CHECK_FALSE(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::Zero);
+ TestSaveLoadValidate("Null", Attachment);
+ }
+
+ SUBCASE("Binary Attachment")
+ {
+ const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView<uint8_t>({0, 1, 2, 3}));
+ CbAttachment Attachment(Buffer);
+ CHECK_FALSE(Attachment.IsNull());
+ CHECK(bool(Attachment));
+ CHECK(Attachment.AsBinaryView() == Buffer);
+ CHECK_FALSE(Attachment.AsCompactBinary().HasValue());
+ CHECK(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::HashMemory(Buffer));
+ TestSaveLoadValidate("Binary", Attachment);
+ }
+
+ SUBCASE("Compact Binary Attachment")
+ {
+ CbWriter Writer;
+ Writer << "Name"sv << 42;
+ CbFieldIterator Fields = Writer.Save();
+ CbAttachment Attachment(Fields);
+
+ CHECK_FALSE(Attachment.IsNull());
+ CHECK(bool(Attachment));
+ CHECK(Attachment.AsBinaryView() == Fields.GetRangeBuffer());
+ CHECK(Attachment.AsCompactBinary() == Fields);
+ CHECK(Attachment.IsBinary());
+ CHECK(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == Fields.GetRangeHash());
+ TestSaveLoadValidate("CompactBinary", Attachment);
+ }
+
+ SUBCASE("Binary View")
+ {
+ const uint8_t Value[]{0, 1, 2, 3};
+ SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value));
+ CbAttachment Attachment(Buffer);
+ CHECK_FALSE(Attachment.IsNull());
+ CHECK(bool(Attachment));
+ CHECK(Attachment.AsBinaryView().GetView().EqualBytes(Buffer.GetView()));
+ CHECK_FALSE(Attachment.AsCompactBinary().HasValue());
+ CHECK(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::HashMemory(Buffer));
+ }
+
+ SUBCASE("Compact Binary View")
+ {
+ CbWriter Writer;
+ Writer << "Name"sv << 42;
+ CbFieldIterator Fields = Writer.Save();
+ CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields));
+ CbAttachment Attachment(FieldsView);
+
+ CHECK_FALSE(Attachment.IsNull());
+ CHECK(bool(Attachment));
+
+ CHECK(Attachment.AsBinaryView() != FieldsView.GetRangeBuffer());
+
+ CHECK(Attachment.AsCompactBinary().GetRangeView().EqualBytes(Fields.GetRangeView()));
+ CHECK(Attachment.IsBinary());
+ CHECK(Attachment.GetHash() == Fields.GetRangeHash());
+ }
+
+ SUBCASE("Binary Load from View")
+ {
+ const uint8_t Value[]{0, 1, 2, 3};
+ const SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value));
+ CbAttachment Attachment(Buffer);
+
+ CbWriter Writer;
+ Attachment.Save(Writer);
+ CbFieldIterator Fields = Writer.Save();
+ CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields));
+ Attachment.Load(FieldsView);
+
+ CHECK_FALSE(Attachment.IsNull());
+ CHECK(bool(Attachment));
+ CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsBinaryView().GetView()));
+ CHECK(Attachment.AsBinaryView().GetView().EqualBytes(Buffer.GetView()));
+ CHECK_FALSE(Attachment.AsCompactBinary().HasValue());
+ CHECK(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::HashMemory(MakeMemoryView(Value)));
+ }
+
+ SUBCASE("Compact Binary Load from View")
+ {
+ CbWriter ValueWriter;
+ ValueWriter << "Name"sv << 42;
+ const CbFieldIterator Value = ValueWriter.Save();
+
+ CHECK(ValidateCompactBinaryRange(Value.GetRangeView(), CbValidateMode::All) == CbValidateError::None);
+ CbAttachment Attachment(Value);
+
+ CbWriter Writer;
+ Attachment.Save(Writer);
+ CbFieldIterator Fields = Writer.Save();
+ CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields));
+
+ Attachment.Load(FieldsView);
+
+ CHECK_FALSE(Attachment.IsNull());
+ CHECK(bool(Attachment));
+
+ CHECK(Attachment.AsBinaryView().GetView().EqualBytes(Value.GetRangeView()));
+ CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsCompactBinary().GetRangeBuffer().GetView()));
+ CHECK(Attachment.IsBinary());
+ CHECK(Attachment.IsCompactBinary());
+
+ CHECK(Attachment.GetHash() == Value.GetRangeHash());
+ }
+
+ SUBCASE("Compact Binary Uniform Sub-View")
+ {
+ const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView<uint8_t>({0, 1, 2, 3}));
+ const CbFieldViewIterator FieldViews = CbFieldViewIterator::MakeRange(Buffer.GetView().RightChop(2), CbFieldType::IntegerPositive);
+ const CbFieldIterator SavedFields = CbFieldIterator::CloneRange(FieldViews);
+ CbFieldIterator Fields = CbFieldIterator::MakeRangeView(FieldViews, Buffer);
+ CbAttachment Attachment(Fields);
+ const SharedBuffer Binary = Attachment.AsBinaryView();
+ CHECK(Attachment.AsCompactBinary() == Fields);
+ CHECK(Binary.GetSize() == SavedFields.GetRangeSize());
+ CHECK(Binary.GetView().EqualBytes(SavedFields.GetRangeView()));
+ CHECK(Attachment.GetHash() == SavedFields.GetRangeHash());
+ TestSaveLoadValidate("CompactBinaryUniformSubView", Attachment);
+ }
+
+ SUBCASE("Binary Null")
+ {
+ const CbAttachment Attachment(SharedBuffer{});
+
+ CHECK(Attachment.IsNull());
+ CHECK_FALSE(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::Zero);
+ }
+
+ SUBCASE("Binary Empty")
+ {
+ const CbAttachment Attachment(SharedBuffer(UniqueBuffer::Alloc(0)));
+
+ CHECK(Attachment.IsNull());
+ CHECK_FALSE(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::Zero);
+ }
+
+ SUBCASE("Compact Binary Empty")
+ {
+ const CbAttachment Attachment(CbFieldIterator{});
+
+ CHECK(Attachment.IsNull());
+ CHECK_FALSE(Attachment.IsBinary());
+ CHECK_FALSE(Attachment.IsCompactBinary());
+ CHECK(Attachment.GetHash() == IoHash::Zero);
+ }
+}
+
+TEST_CASE("usonpackage.serialization")
+{
+ using namespace std::literals;
+
+ const auto TestSaveLoadValidate = [&](const char* Test, const CbPackage& Package) {
+ ZEN_UNUSED(Test);
+
+ CbWriter Writer;
+ Package.Save(Writer);
+ CbFieldIterator Fields = Writer.Save();
+
+ MemoryOutStream MemStream;
+ BinaryWriter WriteAr(MemStream);
+ Package.Save(WriteAr);
+
+ CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView()));
+ CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None);
+ CHECK(ValidateCompactBinaryPackage(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None);
+
+ CbPackage FromFields;
+ FromFields.Load(Fields);
+ CHECK_FALSE(bool(Fields));
+ CHECK(FromFields == Package);
+
+ CbPackage FromArchive;
+ MemoryInStream ReadMemStream(MakeMemoryView(MemStream));
+ BinaryReader ReadAr(ReadMemStream);
+ FromArchive.Load(ReadAr);
+ CHECK(ReadAr.CurrentOffset() == ReadMemStream.Size());
+ CHECK(FromArchive == Package);
+ };
+
+ SUBCASE("Empty")
+ {
+ CbPackage Package;
+ CHECK(Package.IsNull());
+ CHECK_FALSE(bool(Package));
+ CHECK(Package.GetAttachments().size() == 0);
+ TestSaveLoadValidate("Empty", Package);
+ }
+
+ SUBCASE("Object Only")
+ {
+ CbWriter Writer;
+ Writer.BeginObject();
+ Writer << "Field" << 42;
+ Writer.EndObject();
+
+ const CbObject Object = Writer.Save().AsObject();
+ CbPackage Package(Object);
+ CHECK_FALSE(Package.IsNull());
+ CHECK(bool(Package));
+ CHECK(Package.GetAttachments().size() == 0);
+ CHECK(Package.GetObject().GetOuterBuffer() == Object.GetOuterBuffer());
+ CHECK(Package.GetObject()["Field"].AsInt32() == 42);
+ CHECK(Package.GetObjectHash() == Package.GetObject().GetHash());
+ TestSaveLoadValidate("Object", Package);
+ }
+
+ // Object View Only
+ {
+ CbWriter Writer;
+ Writer.BeginObject();
+ Writer << "Field" << 42;
+ Writer.EndObject();
+
+ const CbObject Object = Writer.Save().AsObject();
+ CbPackage Package(CbObject::MakeView(Object));
+ CHECK_FALSE(Package.IsNull());
+ CHECK(bool(Package));
+ CHECK(Package.GetAttachments().size() == 0);
+ CHECK(Package.GetObject().GetOuterBuffer() != Object.GetOuterBuffer());
+ CHECK(Package.GetObject()["Field"].AsInt32() == 42);
+ CHECK(Package.GetObjectHash() == Package.GetObject().GetHash());
+ TestSaveLoadValidate("Object", Package);
+ }
+
+ // Attachment Only
+ {
+ CbObject Object;
+ {
+ CbWriter Writer;
+ Writer.BeginObject();
+ Writer << "Field" << 42;
+ Writer.EndObject();
+ Object = Writer.Save().AsObject();
+ }
+ CbField Field = CbField::Clone(Object["Field"]);
+
+ CbPackage Package;
+ Package.AddAttachment(CbAttachment(CbFieldIterator::MakeSingle(Object.AsField())));
+ Package.AddAttachment(CbAttachment(Field.GetBuffer()));
+
+ CHECK_FALSE(Package.IsNull());
+ CHECK(bool(Package));
+ CHECK(Package.GetAttachments().size() == 2);
+ CHECK(Package.GetObject().Equals(CbObject()));
+ CHECK(Package.GetObjectHash() == IoHash());
+ TestSaveLoadValidate("Attachments", Package);
+
+ const CbAttachment* const ObjectAttachment = Package.FindAttachment(Object.GetHash());
+ REQUIRE(ObjectAttachment);
+
+ const CbAttachment* const FieldAttachment = Package.FindAttachment(Field.GetHash());
+ REQUIRE(FieldAttachment);
+
+ CHECK(ObjectAttachment->AsCompactBinary().AsObject().Equals(Object));
+ CHECK(FieldAttachment->AsBinaryView() == Field.GetBuffer());
+
+ Package.AddAttachment(CbAttachment(SharedBuffer::Clone(Object.GetView())));
+ Package.AddAttachment(CbAttachment(CbFieldIterator::CloneRange(CbFieldViewIterator::MakeSingle(Field))));
+
+ CHECK(Package.GetAttachments().size() == 2);
+ CHECK(Package.FindAttachment(Object.GetHash()) == ObjectAttachment);
+ CHECK(Package.FindAttachment(Field.GetHash()) == FieldAttachment);
+
+ CHECK(ObjectAttachment->AsCompactBinary().AsObject().Equals(Object));
+ CHECK(ObjectAttachment->AsBinaryView() == Object.GetBuffer());
+ CHECK(FieldAttachment->AsCompactBinary().Equals(Field));
+ CHECK(FieldAttachment->AsBinaryView() == Field.GetBuffer());
+
+ CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments())));
+ }
+
+ // Shared Values
+ const uint8_t Level4Values[]{0, 1, 2, 3};
+ SharedBuffer Level4 = SharedBuffer::MakeView(MakeMemoryView(Level4Values));
+ const IoHash Level4Hash = IoHash::HashMemory(Level4);
+
+ CbField Level3;
+ {
+ CbWriter Writer;
+ Writer.SetName("Level4").AddBinaryAttachment(Level4Hash);
+ Level3 = Writer.Save();
+ }
+ const IoHash Level3Hash = Level3.GetHash();
+
+ CbArray Level2;
+ {
+ CbWriter Writer;
+ Writer.SetName("Level3");
+ Writer.BeginArray();
+ Writer.AddCompactBinaryAttachment(Level3Hash);
+ Writer.EndArray();
+ Level2 = Writer.Save().AsArray();
+ }
+ const IoHash Level2Hash = Level2.AsFieldView().GetHash();
+
+ CbObject Level1;
+ {
+ CbWriter Writer;
+ Writer.BeginObject();
+ Writer.SetName("Level2").AddCompactBinaryAttachment(Level2Hash);
+ Writer.EndObject();
+ Level1 = Writer.Save().AsObject();
+ }
+ const IoHash Level1Hash = Level1.AsFieldView().GetHash();
+
+ const auto Resolver = [&Level2, &Level2Hash, &Level3, &Level3Hash, &Level4, &Level4Hash](const IoHash& Hash) -> SharedBuffer {
+ return Hash == Level2Hash ? Level2.GetBuffer()
+ : Hash == Level3Hash ? Level3.GetBuffer()
+ : Hash == Level4Hash ? Level4
+ : SharedBuffer();
+ };
+
+ // Object + Attachments
+ {
+ CbPackage Package;
+ Package.SetObject(Level1, Level1Hash, Resolver);
+
+ CHECK_FALSE(Package.IsNull());
+ CHECK(bool(Package));
+ CHECK(Package.GetAttachments().size() == 3);
+ CHECK(Package.GetObject().GetBuffer() == Level1.GetBuffer());
+ CHECK(Package.GetObjectHash() == Level1Hash);
+ TestSaveLoadValidate("Object+Attachments", Package);
+
+ const CbAttachment* const Level2Attachment = Package.FindAttachment(Level2Hash);
+ const CbAttachment* const Level3Attachment = Package.FindAttachment(Level3Hash);
+ const CbAttachment* const Level4Attachment = Package.FindAttachment(Level4Hash);
+ CHECK((Level2Attachment && Level2Attachment->AsCompactBinary().AsArray().Equals(Level2)));
+ CHECK((Level3Attachment && Level3Attachment->AsCompactBinary().Equals(Level3)));
+ CHECK((Level4Attachment && Level4Attachment->AsBinaryView() != Level4 &&
+ Level4Attachment->AsBinaryView().GetView().EqualBytes(Level4.GetView())));
+
+ CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments())));
+
+ const CbPackage PackageCopy = Package;
+ CHECK(PackageCopy == Package);
+
+ CHECK(Package.RemoveAttachment(Level1Hash) == 0);
+ CHECK(Package.RemoveAttachment(Level2Hash) == 1);
+ CHECK(Package.RemoveAttachment(Level3Hash) == 1);
+ CHECK(Package.RemoveAttachment(Level4Hash) == 1);
+ CHECK(Package.RemoveAttachment(Level4Hash) == 0);
+ CHECK(Package.GetAttachments().size() == 0);
+
+ CHECK(PackageCopy != Package);
+ Package = PackageCopy;
+ CHECK(PackageCopy == Package);
+ Package.SetObject(CbObject());
+ CHECK(PackageCopy != Package);
+ CHECK(Package.GetObjectHash() == IoHash());
+ }
+
+ // Out of Order
+ {
+ CbWriter Writer;
+ Writer.AddBinary(Level2.GetBuffer());
+ Writer.AddCompactBinaryAttachment(Level2Hash);
+ Writer.AddBinary(Level4);
+ Writer.AddBinaryAttachment(Level4Hash);
+ Writer.AddObject(Level1);
+ Writer.AddCompactBinaryAttachment(Level1Hash);
+ Writer.AddBinary(Level3.GetBuffer());
+ Writer.AddCompactBinaryAttachment(Level3Hash);
+ Writer.AddNull();
+
+ CbFieldIterator Fields = Writer.Save();
+ CbPackage FromFields;
+ FromFields.Load(Fields);
+
+ const CbAttachment* const Level2Attachment = FromFields.FindAttachment(Level2Hash);
+ REQUIRE(Level2Attachment);
+ const CbAttachment* const Level3Attachment = FromFields.FindAttachment(Level3Hash);
+ REQUIRE(Level3Attachment);
+ const CbAttachment* const Level4Attachment = FromFields.FindAttachment(Level4Hash);
+ REQUIRE(Level4Attachment);
+
+ CHECK(FromFields.GetObject().Equals(Level1));
+ CHECK(FromFields.GetObject().GetOuterBuffer() == Fields.GetOuterBuffer());
+ CHECK(FromFields.GetObjectHash() == Level1Hash);
+
+ const MemoryView FieldsOuterBufferView = Fields.GetOuterBuffer().GetView();
+
+ CHECK(Level2Attachment->AsCompactBinary().AsArray().Equals(Level2));
+ CHECK(FieldsOuterBufferView.Contains(Level2Attachment->AsBinaryView().GetView()));
+ CHECK(Level2Attachment->GetHash() == Level2Hash);
+
+ CHECK(Level3Attachment->AsCompactBinary().Equals(Level3));
+ CHECK(FieldsOuterBufferView.Contains(Level3Attachment->AsBinaryView().GetView()));
+ CHECK(Level3Attachment->GetHash() == Level3Hash);
+
+ CHECK(Level4Attachment->AsBinaryView().GetView().EqualBytes(Level4.GetView()));
+ CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinaryView().GetView()));
+ CHECK(Level4Attachment->GetHash() == Level4Hash);
+
+ MemoryOutStream WriteStream;
+ BinaryWriter WriteAr(WriteStream);
+ Writer.Save(WriteAr);
+ CbPackage FromArchive;
+ MemoryInStream ReadStream(MakeMemoryView(WriteStream));
+ BinaryReader ReadAr(ReadStream);
+ FromArchive.Load(ReadAr);
+
+ Writer.Reset();
+ FromArchive.Save(Writer);
+ CbFieldIterator Saved = Writer.Save();
+ CHECK(Saved.AsObject().Equals(Level1));
+ ++Saved;
+ CHECK(Saved.AsCompactBinaryAttachment() == Level1Hash);
+ ++Saved;
+ CHECK(Saved.AsBinaryView().EqualBytes(Level2.GetView()));
+ ++Saved;
+ CHECK(Saved.AsCompactBinaryAttachment() == Level2Hash);
+ ++Saved;
+ CHECK(Saved.AsBinaryView().EqualBytes(Level3.GetView()));
+ ++Saved;
+ CHECK(Saved.AsCompactBinaryAttachment() == Level3Hash);
+ ++Saved;
+ CHECK(Saved.AsBinaryView().EqualBytes(Level4.GetView()));
+ ++Saved;
+ CHECK(Saved.AsBinaryAttachment() == Level4Hash);
+ ++Saved;
+ CHECK(Saved.IsNull());
+ ++Saved;
+ CHECK(!Saved);
+ }
+
+ // Null Attachment
+ {
+ const CbAttachment NullAttachment;
+ CbPackage Package;
+ Package.AddAttachment(NullAttachment);
+ CHECK(Package.IsNull());
+ CHECK_FALSE(bool(Package));
+ CHECK(Package.GetAttachments().size() == 0);
+ CHECK_FALSE(Package.FindAttachment(NullAttachment));
+ }
+
+ // Resolve After Merge
+ {
+ bool bResolved = false;
+ CbPackage Package;
+ Package.AddAttachment(CbAttachment(Level3.GetBuffer()));
+ Package.AddAttachment(CbAttachment(CbFieldIterator::MakeSingle(Level3)), [&bResolved](const IoHash& Hash) -> SharedBuffer {
+ ZEN_UNUSED(Hash);
+ bResolved = true;
+ return SharedBuffer();
+ });
+ CHECK(bResolved);
+ }
+}
+
+} // namespace zen
diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp
new file mode 100644
index 000000000..51ed31e95
--- /dev/null
+++ b/zencore/compactbinaryvalidation.cpp
@@ -0,0 +1,607 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/compactbinaryvalidation.h"
+
+#include <zencore/compactbinarypackage.h>
+#include <zencore/endian.h>
+#include <zencore/memory.h>
+#include <zencore/string.h>
+
+#include <algorithm>
+
+#include <doctest/doctest.h>
+
+namespace zen {
+
+namespace CbValidationPrivate {
+
+ template<typename T>
+ static constexpr inline T ReadUnaligned(const void* const Memory)
+ {
+#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
+ return *static_cast<const T*>(Memory);
+#else
+ T Value;
+ memcpy(&Value, Memory, sizeof(Value));
+ return Value;
+#endif
+ }
+
+} // namespace CbValidationPrivate
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Adds the given error(s) to the error mask.
+ *
+ * This function exists to make validation errors easier to debug by providing one location to set a breakpoint.
+ */
+ZEN_NOINLINE static void
+AddError(CbValidateError& OutError, const CbValidateError InError)
+{
+ OutError |= InError;
+}
+
+/**
+ * Validate and read a field type from the view.
+ *
+ * A type argument with the HasFieldType flag indicates that the type will not be read from the view.
+ */
+static CbFieldType
+ValidateCbFieldType(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType Type = CbFieldType::HasFieldType)
+{
+ ZEN_UNUSED(Mode);
+ if (CbFieldTypeOps::HasFieldType(Type))
+ {
+ if (View.GetSize() >= 1)
+ {
+ Type = *static_cast<const CbFieldType*>(View.GetData());
+ View += 1;
+ if (CbFieldTypeOps::HasFieldType(Type))
+ {
+ AddError(Error, CbValidateError::InvalidType);
+ }
+ }
+ else
+ {
+ AddError(Error, CbValidateError::OutOfBounds);
+ View.Reset();
+ return CbFieldType::None;
+ }
+ }
+
+ if (CbFieldTypeOps::GetSerializedType(Type) != Type)
+ {
+ AddError(Error, CbValidateError::InvalidType);
+ View.Reset();
+ }
+
+ return Type;
+}
+
+/**
+ * Validate and read an unsigned integer from the view.
+ *
+ * Modifies the view to start at the end of the value, and adds error flags if applicable.
+ */
+static uint64_t
+ValidateCbUInt(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+ if (View.GetSize() > 0 && View.GetSize() >= MeasureVarUInt(View.GetData()))
+ {
+ uint32_t ValueByteCount;
+ const uint64_t Value = ReadVarUInt(View.GetData(), ValueByteCount);
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Format) && ValueByteCount > MeasureVarUInt(Value))
+ {
+ AddError(Error, CbValidateError::InvalidInteger);
+ }
+ View += ValueByteCount;
+ return Value;
+ }
+ else
+ {
+ AddError(Error, CbValidateError::OutOfBounds);
+ View.Reset();
+ return 0;
+ }
+}
+
+/**
+ * Validate a 64-bit floating point value from the view.
+ *
+ * Modifies the view to start at the end of the value, and adds error flags if applicable.
+ */
+static void
+ValidateCbFloat64(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+ if (View.GetSize() >= sizeof(double))
+ {
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Format))
+ {
+ const uint64_t RawValue = FromNetworkOrder(CbValidationPrivate::ReadUnaligned<uint64_t>(View.GetData()));
+ const double Value = reinterpret_cast<const double&>(RawValue);
+ if (Value == double(float(Value)))
+ {
+ AddError(Error, CbValidateError::InvalidFloat);
+ }
+ }
+ View += sizeof(double);
+ }
+ else
+ {
+ AddError(Error, CbValidateError::OutOfBounds);
+ View.Reset();
+ }
+}
+
+/**
+ * Validate and read a string from the view.
+ *
+ * Modifies the view to start at the end of the string, and adds error flags if applicable.
+ */
+static std::string_view
+ValidateCbString(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+ const uint64_t NameSize = ValidateCbUInt(View, Mode, Error);
+ if (View.GetSize() >= NameSize)
+ {
+ const std::string_view Name(static_cast<const char*>(View.GetData()), static_cast<int32_t>(NameSize));
+ View += NameSize;
+ return Name;
+ }
+ else
+ {
+ AddError(Error, CbValidateError::OutOfBounds);
+ View.Reset();
+ return std::string_view();
+ }
+}
+
+static CbFieldView ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ExternalType);
+
+/** A type that checks whether all validated fields are of the same type. */
+class CbUniformFieldsValidator
+{
+public:
+ inline explicit CbUniformFieldsValidator(CbFieldType InExternalType) : ExternalType(InExternalType) {}
+
+ inline CbFieldView ValidateField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+ {
+ const void* const FieldData = View.GetData();
+ if (CbFieldView Field = ValidateCbField(View, Mode, Error, ExternalType))
+ {
+ ++FieldCount;
+ if (CbFieldTypeOps::HasFieldType(ExternalType))
+ {
+ const CbFieldType FieldType = *static_cast<const CbFieldType*>(FieldData);
+ if (FieldCount == 1)
+ {
+ FirstType = FieldType;
+ }
+ else if (FieldType != FirstType)
+ {
+ bUniform = false;
+ }
+ }
+ return Field;
+ }
+
+ // It may not safe to check for uniformity if the field was invalid.
+ bUniform = false;
+ return CbFieldView();
+ }
+
+ inline bool IsUniform() const { return FieldCount > 0 && bUniform; }
+
+private:
+ uint32_t FieldCount = 0;
+ bool bUniform = true;
+ CbFieldType FirstType = CbFieldType::None;
+ CbFieldType ExternalType;
+};
+
+static void
+ValidateCbObject(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ObjectType)
+{
+ const uint64_t Size = ValidateCbUInt(View, Mode, Error);
+ MemoryView ObjectView = View.Left(Size);
+ View += Size;
+
+ if (Size > 0)
+ {
+ std::vector<std::string_view> Names;
+
+ const bool bUniformObject = CbFieldTypeOps::GetType(ObjectType) == CbFieldType::UniformObject;
+ const CbFieldType ExternalType = bUniformObject ? ValidateCbFieldType(ObjectView, Mode, Error) : CbFieldType::HasFieldType;
+ CbUniformFieldsValidator UniformValidator(ExternalType);
+ do
+ {
+ if (CbFieldView Field = UniformValidator.ValidateField(ObjectView, Mode, Error))
+ {
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Names))
+ {
+ if (Field.HasName())
+ {
+ Names.push_back(Field.GetName());
+ }
+ else
+ {
+ AddError(Error, CbValidateError::MissingName);
+ }
+ }
+ }
+ } while (!ObjectView.IsEmpty());
+
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Names) && Names.size() > 1)
+ {
+ std::sort(begin(Names), end(Names), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; });
+
+ for (const std::string_view *NamesIt = Names.data(), *NamesEnd = NamesIt + Names.size() - 1; NamesIt != NamesEnd; ++NamesIt)
+ {
+ if (NamesIt[0] == NamesIt[1])
+ {
+ AddError(Error, CbValidateError::DuplicateName);
+ break;
+ }
+ }
+ }
+
+ if (!bUniformObject && EnumHasAnyFlags(Mode, CbValidateMode::Format) && UniformValidator.IsUniform())
+ {
+ AddError(Error, CbValidateError::NonUniformObject);
+ }
+ }
+}
+
+static void
+ValidateCbArray(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, CbFieldType ArrayType)
+{
+ const uint64_t Size = ValidateCbUInt(View, Mode, Error);
+ MemoryView ArrayView = View.Left(Size);
+ View += Size;
+
+ const uint64_t Count = ValidateCbUInt(ArrayView, Mode, Error);
+ const uint64_t FieldsSize = ArrayView.GetSize();
+ const bool bUniformArray = CbFieldTypeOps::GetType(ArrayType) == CbFieldType::UniformArray;
+ const CbFieldType ExternalType = bUniformArray ? ValidateCbFieldType(ArrayView, Mode, Error) : CbFieldType::HasFieldType;
+ CbUniformFieldsValidator UniformValidator(ExternalType);
+
+ for (uint64_t Index = 0; Index < Count; ++Index)
+ {
+ if (CbFieldView Field = UniformValidator.ValidateField(ArrayView, Mode, Error))
+ {
+ if (Field.HasName() && EnumHasAnyFlags(Mode, CbValidateMode::Names))
+ {
+ AddError(Error, CbValidateError::ArrayName);
+ }
+ }
+ }
+
+ if (!bUniformArray && EnumHasAnyFlags(Mode, CbValidateMode::Format) && UniformValidator.IsUniform() && FieldsSize > Count)
+ {
+ AddError(Error, CbValidateError::NonUniformArray);
+ }
+}
+
+static CbFieldView
+ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, const CbFieldType ExternalType = CbFieldType::HasFieldType)
+{
+ const MemoryView FieldView = View;
+ const CbFieldType Type = ValidateCbFieldType(View, Mode, Error, ExternalType);
+ const std::string_view Name = CbFieldTypeOps::HasFieldName(Type) ? ValidateCbString(View, Mode, Error) : std::string_view();
+
+ auto ValidateFixedPayload = [&View, &Error](uint32_t PayloadSize) {
+ if (View.GetSize() >= PayloadSize)
+ {
+ View += PayloadSize;
+ }
+ else
+ {
+ AddError(Error, CbValidateError::OutOfBounds);
+ View.Reset();
+ }
+ };
+
+ if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds | CbValidateError::InvalidType))
+ {
+ return CbFieldView();
+ }
+
+ switch (CbFieldType FieldType = CbFieldTypeOps::GetType(Type))
+ {
+ default:
+ case CbFieldType::None:
+ AddError(Error, CbValidateError::InvalidType);
+ View.Reset();
+ break;
+ case CbFieldType::Null:
+ case CbFieldType::BoolFalse:
+ case CbFieldType::BoolTrue:
+ if (FieldView == View)
+ {
+ // Reset the view because a zero-sized field can cause infinite field iteration.
+ AddError(Error, CbValidateError::InvalidType);
+ View.Reset();
+ }
+ break;
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ ValidateCbObject(View, Mode, Error, FieldType);
+ break;
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ ValidateCbArray(View, Mode, Error, FieldType);
+ break;
+ case CbFieldType::Binary:
+ {
+ const uint64_t ValueSize = ValidateCbUInt(View, Mode, Error);
+ if (View.GetSize() < ValueSize)
+ {
+ AddError(Error, CbValidateError::OutOfBounds);
+ View.Reset();
+ }
+ else
+ {
+ View += ValueSize;
+ }
+ break;
+ }
+ case CbFieldType::String:
+ ValidateCbString(View, Mode, Error);
+ break;
+ case CbFieldType::IntegerPositive:
+ ValidateCbUInt(View, Mode, Error);
+ break;
+ case CbFieldType::IntegerNegative:
+ ValidateCbUInt(View, Mode, Error);
+ break;
+ case CbFieldType::Float32:
+ ValidateFixedPayload(4);
+ break;
+ case CbFieldType::Float64:
+ ValidateCbFloat64(View, Mode, Error);
+ break;
+ case CbFieldType::CompactBinaryAttachment:
+ case CbFieldType::BinaryAttachment:
+ case CbFieldType::Hash:
+ ValidateFixedPayload(20);
+ break;
+ case CbFieldType::Uuid:
+ ValidateFixedPayload(16);
+ break;
+ case CbFieldType::DateTime:
+ case CbFieldType::TimeSpan:
+ ValidateFixedPayload(8);
+ break;
+ case CbFieldType::ObjectId:
+ ValidateFixedPayload(12);
+ break;
+ case CbFieldType::CustomById:
+ case CbFieldType::CustomByName:
+ ZEN_NOT_IMPLEMENTED(); // TODO: FIX!
+ break;
+ }
+
+ if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds | CbValidateError::InvalidType))
+ {
+ return CbFieldView();
+ }
+
+ return CbFieldView(FieldView.GetData(), ExternalType);
+}
+
+static CbFieldView
+ValidateCbPackageField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+ if (View.IsEmpty())
+ {
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+ return CbFieldView();
+ }
+ if (CbFieldView Field = ValidateCbField(View, Mode, Error))
+ {
+ if (Field.HasName() && EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+ return Field;
+ }
+ return CbFieldView();
+}
+
+static IoHash
+ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+ const MemoryView ValueView = Value.AsBinaryView();
+ if (Value.HasError() && EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+ }
+ else if (ValueView.GetSize())
+ {
+ if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error))
+ {
+ const IoHash Hash = HashField.AsAttachment();
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ if (HashField.HasError())
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+ else if (Hash != IoHash::HashMemory(ValueView.GetData(), ValueView.GetSize()))
+ {
+ AddError(Error, CbValidateError::InvalidPackageHash);
+ }
+ }
+ return Hash;
+ }
+ }
+ return IoHash();
+}
+
+static IoHash
+ValidateCbPackageObject(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
+{
+ CbObjectView Object = Value.AsObjectView();
+ if (Value.HasError())
+ {
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+ }
+ else if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error))
+ {
+ const IoHash Hash = HashField.AsAttachment();
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ if (!Object.CreateViewIterator())
+ {
+ AddError(Error, CbValidateError::NullPackageObject);
+ }
+ if (HashField.HasError())
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+ else if (Hash != Value.GetHash())
+ {
+ AddError(Error, CbValidateError::InvalidPackageHash);
+ }
+ }
+ return Hash;
+ }
+ return IoHash();
+}
+
+CbValidateError
+ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type)
+{
+ CbValidateError Error = CbValidateError::None;
+ if (EnumHasAnyFlags(Mode, CbValidateMode::All))
+ {
+ ValidateCbField(View, Mode, Error, Type);
+ if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding))
+ {
+ AddError(Error, CbValidateError::Padding);
+ }
+ }
+ return Error;
+}
+
+CbValidateError
+ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode)
+{
+ CbValidateError Error = CbValidateError::None;
+ if (EnumHasAnyFlags(Mode, CbValidateMode::All))
+ {
+ while (!View.IsEmpty())
+ {
+ ValidateCbField(View, Mode, Error);
+ }
+ }
+ return Error;
+}
+
+CbValidateError
+ValidateCompactBinaryAttachment(MemoryView View, CbValidateMode Mode)
+{
+ CbValidateError Error = CbValidateError::None;
+ if (EnumHasAnyFlags(Mode, CbValidateMode::All))
+ {
+ if (CbFieldView Value = ValidateCbPackageField(View, Mode, Error))
+ {
+ ValidateCbPackageAttachment(Value, View, Mode, Error);
+ }
+ if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding))
+ {
+ AddError(Error, CbValidateError::Padding);
+ }
+ }
+ return Error;
+}
+
+CbValidateError
+ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode)
+{
+ std::vector<IoHash> Attachments;
+ CbValidateError Error = CbValidateError::None;
+ if (EnumHasAnyFlags(Mode, CbValidateMode::All))
+ {
+ uint32_t ObjectCount = 0;
+ while (CbFieldView Value = ValidateCbPackageField(View, Mode, Error))
+ {
+ if (Value.IsBinary())
+ {
+ const IoHash Hash = ValidateCbPackageAttachment(Value, View, Mode, Error);
+ if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ Attachments.push_back(Hash);
+ if (Value.AsBinaryView().IsEmpty())
+ {
+ AddError(Error, CbValidateError::NullPackageAttachment);
+ }
+ }
+ }
+ else if (Value.IsObject())
+ {
+ ValidateCbPackageObject(Value, View, Mode, Error);
+ if (++ObjectCount > 1 && EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ AddError(Error, CbValidateError::MultiplePackageObjects);
+ }
+ }
+ else if (Value.IsNull())
+ {
+ break;
+ }
+ else if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ AddError(Error, CbValidateError::InvalidPackageFormat);
+ }
+
+ if (EnumHasAnyFlags(Error, CbValidateError::OutOfBounds))
+ {
+ break;
+ }
+ }
+
+ if (!View.IsEmpty() && EnumHasAnyFlags(Mode, CbValidateMode::Padding))
+ {
+ AddError(Error, CbValidateError::Padding);
+ }
+
+ if (Attachments.size() && EnumHasAnyFlags(Mode, CbValidateMode::Package))
+ {
+ std::sort(begin(Attachments), end(Attachments));
+ for (const IoHash *It = Attachments.data(), *End = It + Attachments.size() - 1; It != End; ++It)
+ {
+ if (It[0] == It[1])
+ {
+ AddError(Error, CbValidateError::DuplicateAttachments);
+ break;
+ }
+ }
+ }
+ }
+ return Error;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void
+usonvalidation_forcelink()
+{
+}
+
+TEST_CASE("usonvalidation")
+{
+ SUBCASE("Basic") {}
+}
+
+} // namespace zen
diff --git a/zencore/compress.cpp b/zencore/compress.cpp
new file mode 100644
index 000000000..0a9c43949
--- /dev/null
+++ b/zencore/compress.cpp
@@ -0,0 +1,11 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/compress.h>
+
+#include <doctest/doctest.h>
+#include <lz4.h>
+#include <functional>
+
+namespace zen {
+
+} // namespace zen
diff --git a/zencore/except.cpp b/zencore/except.cpp
new file mode 100644
index 000000000..b02122f58
--- /dev/null
+++ b/zencore/except.cpp
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/except.h>
+
+namespace zen {
+
+void
+ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] const char* Message)
+{
+ // TODO
+
+ int ErrValue = hRes;
+
+ throw std::system_error(ErrValue, std::system_category(), Message);
+}
+
+} // namespace zen
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
new file mode 100644
index 000000000..663a626a1
--- /dev/null
+++ b/zencore/filesystem.cpp
@@ -0,0 +1,592 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/filesystem.h>
+
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/iobuffer.h>
+#include <zencore/string.h>
+#include <zencore/windows.h>
+
+#include <atlbase.h>
+#include <atlfile.h>
+#include <winioctl.h>
+#include <winnt.h>
+#include <filesystem>
+
+#include <spdlog/spdlog.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+using namespace std::literals;
+
+static bool
+DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
+{
+ CHandle hDir(CreateFileW(Path,
+ GENERIC_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ nullptr,
+ OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT,
+ nullptr));
+
+ if (hDir != INVALID_HANDLE_VALUE)
+ {
+ REPARSE_GUID_DATA_BUFFER Rgdb = {};
+ Rgdb.ReparseTag = dwReparseTag;
+
+ DWORD dwBytes;
+ const BOOL bOK =
+ DeviceIoControl(hDir, FSCTL_DELETE_REPARSE_POINT, &Rgdb, REPARSE_GUID_DATA_BUFFER_HEADER_SIZE, nullptr, 0, &dwBytes, nullptr);
+
+ return bOK == TRUE;
+ }
+
+ return false;
+}
+
+bool
+CreateDirectories(const wchar_t* Dir)
+{
+ return std::filesystem::create_directories(Dir);
+}
+
+bool
+CreateDirectories(const std::filesystem::path& Dir)
+{
+ return std::filesystem::create_directories(Dir);
+}
+
+// Erase all files and directories in a given directory, leaving an empty directory
+// behind
+
+static bool
+WipeDirectory(const wchar_t* DirPath)
+{
+ ExtendableWideStringBuilder<128> Pattern;
+ Pattern.Append(DirPath);
+ Pattern.Append(L"\\*");
+
+ WIN32_FIND_DATAW FindData;
+ HANDLE hFind = FindFirstFileW(Pattern.c_str(), &FindData);
+
+ bool AllOk = true;
+
+ if (hFind != nullptr)
+ {
+ do
+ {
+ bool IsRegular = true;
+
+ if (FindData.cFileName[0] == L'.')
+ {
+ if (FindData.cFileName[1] == L'.')
+ {
+ if (FindData.cFileName[2] == L'\0')
+ {
+ IsRegular = false;
+ }
+ }
+ else if (FindData.cFileName[1] == L'\0')
+ {
+ IsRegular = false;
+ }
+ }
+
+ if (IsRegular)
+ {
+ ExtendableWideStringBuilder<128> Path;
+ Path.Append(DirPath);
+ Path.Append(L'\\');
+ Path.Append(FindData.cFileName);
+
+ // if (fd.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN)
+ // deleteReparsePoint(path.c_str(), fd.dwReserved0);
+
+ if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ {
+ if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_OPEN)
+ {
+ DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
+ }
+
+ if (FindData.dwFileAttributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS)
+ {
+ DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
+ }
+
+ bool Success = DeleteDirectories(Path.c_str());
+
+ if (!Success)
+ {
+ if (FindData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+ {
+ DeleteReparsePoint(Path.c_str(), FindData.dwReserved0);
+ }
+ }
+ }
+ else
+ {
+ DeleteFileW(Path.c_str());
+ }
+ }
+ } while (FindNextFileW(hFind, &FindData) == TRUE);
+ }
+
+ FindClose(hFind);
+
+ return true;
+}
+
+bool
+DeleteDirectories(const wchar_t* DirPath)
+{
+ return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
+}
+
+bool
+CleanDirectory(const wchar_t* DirPath)
+{
+ if (std::filesystem::exists(DirPath))
+ {
+ return WipeDirectory(DirPath);
+ }
+ else
+ {
+ return CreateDirectories(DirPath);
+ }
+}
+
+bool
+DeleteDirectories(const std::filesystem::path& Dir)
+{
+ return DeleteDirectories(Dir.c_str());
+}
+
+bool
+CleanDirectory(const std::filesystem::path& Dir)
+{
+ return CleanDirectory(Dir.c_str());
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+bool
+SupportsBlockRefCounting(std::filesystem::path Path)
+{
+ ATL::CHandle Handle(CreateFileW(Path.c_str(),
+ GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ nullptr,
+ OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS,
+ nullptr));
+
+ if (Handle == INVALID_HANDLE_VALUE)
+ {
+ Handle.Detach();
+ return false;
+ }
+
+ ULONG FileSystemFlags = 0;
+ if (!GetVolumeInformationByHandleW(Handle, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0))
+ {
+ return false;
+ }
+
+ if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING))
+ {
+ return false;
+ }
+
+ return true;
+}
+
+bool
+CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
+{
+ ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+ if (FromFile == INVALID_HANDLE_VALUE)
+ {
+ FromFile.Detach();
+ return false;
+ }
+
+ ULONG FileSystemFlags;
+ if (!GetVolumeInformationByHandleW(FromFile, nullptr, 0, nullptr, nullptr, /* lpFileSystemFlags */ &FileSystemFlags, nullptr, 0))
+ {
+ return false;
+ }
+ if (!(FileSystemFlags & FILE_SUPPORTS_BLOCK_REFCOUNTING))
+ {
+ SetLastError(ERROR_NOT_CAPABLE);
+ return false;
+ }
+
+ FILE_END_OF_FILE_INFO FileSize;
+ if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile))
+ {
+ return false;
+ }
+
+ FILE_BASIC_INFO BasicInfo;
+ if (!GetFileInformationByHandleEx(FromFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo))
+ {
+ return false;
+ }
+
+ DWORD dwBytesReturned = 0;
+ FSCTL_GET_INTEGRITY_INFORMATION_BUFFER GetIntegrityInfoBuffer;
+ if (!DeviceIoControl(FromFile,
+ FSCTL_GET_INTEGRITY_INFORMATION,
+ nullptr,
+ 0,
+ &GetIntegrityInfoBuffer,
+ sizeof GetIntegrityInfoBuffer,
+ &dwBytesReturned,
+ nullptr))
+ {
+ return false;
+ }
+
+ SetFileAttributesW(ToPath.c_str(), FILE_ATTRIBUTE_NORMAL);
+
+ ATL::CHandle TargetFile(CreateFileW(ToPath.c_str(),
+ GENERIC_READ | GENERIC_WRITE | DELETE,
+ /* no sharing */ FILE_SHARE_READ,
+ nullptr,
+ OPEN_ALWAYS,
+ 0,
+ /* hTemplateFile */ FromFile));
+
+ if (TargetFile == INVALID_HANDLE_VALUE)
+ {
+ TargetFile.Detach();
+ return false;
+ }
+
+ // Delete target file when handle is closed (we only reset this if the copy succeeds)
+ FILE_DISPOSITION_INFO FileDisposition = {TRUE};
+ if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
+ {
+ TargetFile.Close();
+ DeleteFileW(ToPath.c_str());
+ return false;
+ }
+
+ // Make file sparse so we don't end up allocating space when we change the file size
+ if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &dwBytesReturned, nullptr))
+ {
+ return false;
+ }
+
+ // Copy integrity checking information
+ FSCTL_SET_INTEGRITY_INFORMATION_BUFFER IntegritySet = {GetIntegrityInfoBuffer.ChecksumAlgorithm,
+ GetIntegrityInfoBuffer.Reserved,
+ GetIntegrityInfoBuffer.Flags};
+ if (!DeviceIoControl(TargetFile, FSCTL_SET_INTEGRITY_INFORMATION, &IntegritySet, sizeof IntegritySet, nullptr, 0, nullptr, nullptr))
+ {
+ return false;
+ }
+
+ // Resize file - note that the file is sparse at this point so no additional data will be written
+ if (!SetFileInformationByHandle(TargetFile, FileEndOfFileInfo, &FileSize, sizeof FileSize))
+ {
+ return false;
+ }
+
+ constexpr auto RoundToClusterSize = [](LONG64 FileSize, ULONG ClusterSize) -> LONG64 {
+ return (FileSize + ClusterSize - 1) / ClusterSize * ClusterSize;
+ };
+ static_assert(RoundToClusterSize(5678, 4 * 1024) == 8 * 1024);
+
+ // Loop for cloning file contents. This is necessary as the API has a 32-bit size
+ // limit for some reason
+
+ const LONG64 SplitThreshold = (1LL << 32) - GetIntegrityInfoBuffer.ClusterSizeInBytes;
+
+ DUPLICATE_EXTENTS_DATA DuplicateExtentsData{.FileHandle = FromFile};
+
+ for (LONG64 CurrentByteOffset = 0,
+ RemainingBytes = RoundToClusterSize(FileSize.EndOfFile.QuadPart, GetIntegrityInfoBuffer.ClusterSizeInBytes);
+ RemainingBytes > 0;
+ CurrentByteOffset += SplitThreshold, RemainingBytes -= SplitThreshold)
+ {
+ DuplicateExtentsData.SourceFileOffset.QuadPart = CurrentByteOffset;
+ DuplicateExtentsData.TargetFileOffset.QuadPart = CurrentByteOffset;
+ DuplicateExtentsData.ByteCount.QuadPart = std::min(SplitThreshold, RemainingBytes);
+
+ if (!DeviceIoControl(TargetFile,
+ FSCTL_DUPLICATE_EXTENTS_TO_FILE,
+ &DuplicateExtentsData,
+ sizeof DuplicateExtentsData,
+ nullptr,
+ 0,
+ &dwBytesReturned,
+ nullptr))
+ {
+ return false;
+ }
+ }
+
+ // Make the file not sparse again now that we have populated the contents
+ if (!(BasicInfo.FileAttributes & FILE_ATTRIBUTE_SPARSE_FILE))
+ {
+ FILE_SET_SPARSE_BUFFER SetSparse = {FALSE};
+
+ if (!DeviceIoControl(TargetFile, FSCTL_SET_SPARSE, &SetSparse, sizeof SetSparse, nullptr, 0, &dwBytesReturned, nullptr))
+ {
+ return false;
+ }
+ }
+
+ // Update timestamps (but don't lie about the creation time)
+ BasicInfo.CreationTime.QuadPart = 0;
+ if (!SetFileInformationByHandle(TargetFile, FileBasicInfo, &BasicInfo, sizeof BasicInfo))
+ {
+ return false;
+ }
+
+ if (!FlushFileBuffers(TargetFile))
+ {
+ return false;
+ }
+
+ // Finally now everything is done - make sure the file is not deleted on close!
+
+ FileDisposition = {FALSE};
+
+ const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition));
+
+ return AllOk;
+}
+
+bool
+CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
+{
+ bool Success = false;
+
+ if (Options.EnableClone)
+ {
+ Success = CloneFile(FromPath.native(), ToPath.native());
+
+ if (Success)
+ {
+ return true;
+ }
+ }
+
+ if (Options.MustClone)
+ {
+ return false;
+ }
+
+ BOOL CancelFlag = FALSE;
+ Success = !!::CopyFileExW(FromPath.c_str(),
+ ToPath.c_str(),
+ /* lpProgressRoutine */ nullptr,
+ /* lpData */ nullptr,
+ &CancelFlag,
+ /* dwCopyFlags */ 0);
+
+ if (!Success)
+ {
+ throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file copy failed");
+ }
+
+ return Success;
+}
+
+bool
+WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount)
+{
+ using namespace fmt::literals;
+
+ CAtlFile Outfile;
+ HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
+ if (FAILED(hRes))
+ {
+ zen::ThrowIfFailed(hRes, "File open failed for '{}'"_format(Path).c_str());
+ }
+
+ // TODO: this could be block-enlightened
+
+ for (size_t i = 0; i < BufferCount; ++i)
+ {
+ uint64_t WriteSize = Data[i]->Size();
+ const void* DataPtr = Data[i]->Data();
+
+ while (WriteSize)
+ {
+ uint64_t ChunkSize = zen::Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024);
+
+ hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize));
+
+ if (FAILED(hRes))
+ {
+ zen::ThrowIfFailed(hRes, "File write failed for '{}'"_format(Path).c_str());
+ }
+
+ WriteSize -= ChunkSize;
+ DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize;
+ }
+ }
+
+ return true;
+}
+
+FileContents
+ReadFile(std::filesystem::path Path)
+{
+ ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+ if (FromFile == INVALID_HANDLE_VALUE)
+ {
+ FromFile.Detach();
+ return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
+ }
+
+ FILE_END_OF_FILE_INFO FileSize;
+ if (!GetFileSizeEx(FromFile, &FileSize.EndOfFile))
+ {
+ return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
+ }
+
+ const uint64_t FileSizeBytes = FileSize.EndOfFile.QuadPart;
+
+ FileContents Contents;
+
+ Contents.Data.emplace_back(IoBuffer(IoBuffer::File, FromFile.Detach(), 0, FileSizeBytes));
+
+ return Contents;
+}
+
+bool
+ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
+{
+ ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+ if (FromFile == INVALID_HANDLE_VALUE)
+ {
+ FromFile.Detach();
+ return false;
+ }
+
+ std::vector<uint8_t> ReadBuffer(ChunkSize);
+
+ for (;;)
+ {
+ DWORD dwBytesRead = 0;
+ BOOL Success = ::ReadFile(FromFile, ReadBuffer.data(), (DWORD)ReadBuffer.size(), &dwBytesRead, nullptr);
+
+ if (!Success)
+ {
+ throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed");
+ }
+
+ if (dwBytesRead == 0)
+ break;
+
+ ProcessFunc(ReadBuffer.data(), dwBytesRead);
+ }
+
+ return true;
+}
+
+std::string
+ToUtf8(const std::filesystem::path& Path)
+{
+ return WideToUtf8(Path.native().c_str());
+}
+
+void
+FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor)
+{
+ uint64_t FileInfoBuffer[8 * 1024];
+
+ FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo;
+ bool Continue = true;
+
+ std::wstring RootDirPath = RootDir.native();
+
+ CAtlFile RootDirHandle;
+ HRESULT hRes = RootDirHandle.Create(RootDirPath.c_str(),
+ GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE,
+ OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS);
+
+ zen::ThrowIfFailed(hRes, "Failed to open handle to volume root");
+
+ while (Continue)
+ {
+ BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer);
+ FibClass = FileIdBothDirectoryInfo; // Set up for next iteration
+
+ uint64_t EntryOffset = 0;
+
+ if (!Success)
+ {
+ DWORD LastError = GetLastError();
+
+ if (LastError == ERROR_NO_MORE_FILES)
+ {
+ break;
+ }
+
+ throw std::system_error(std::error_code(LastError, std::system_category()), "file system traversal error");
+ }
+
+ for (;;)
+ {
+ const FILE_ID_BOTH_DIR_INFO* DirInfo =
+ reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset);
+
+ std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t));
+
+ if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+ {
+ if (FileName == L"."sv || FileName == L".."sv)
+ {
+ // Not very interesting
+ }
+ else
+ {
+ const bool ShouldDescend = Visitor.VisitDirectory(RootDir, FileName);
+
+ if (ShouldDescend)
+ {
+ // Note that this recursion combined with the buffer could
+ // blow the stack, we should consider a different strategy
+
+ std::filesystem::path FullPath = RootDir / FileName;
+
+ TraverseFileSystem(FullPath, Visitor);
+ }
+ }
+ }
+ else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE)
+ {
+ spdlog::warn("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir);
+ }
+ else
+ {
+ std::filesystem::path FullPath = RootDir / FileName;
+
+ Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
+ }
+
+ const uint64_t NextOffset = DirInfo->NextEntryOffset;
+
+ if (NextOffset == 0)
+ {
+ break;
+ }
+
+ EntryOffset += NextOffset;
+ }
+ }
+}
+
+} // namespace zen
diff --git a/zencore/httpclient.cpp b/zencore/httpclient.cpp
new file mode 100644
index 000000000..268483403
--- /dev/null
+++ b/zencore/httpclient.cpp
@@ -0,0 +1,23 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/httpclient.h>
+
+#include <spdlog/spdlog.h>
+
+#include <doctest/doctest.h>
+
+namespace zen {
+
+TEST_CASE("httpclient")
+{
+ using namespace std::literals;
+
+ SUBCASE("client") {}
+}
+
+void
+httpclient_forcelink()
+{
+}
+
+} // namespace zen
diff --git a/zencore/httpserver.cpp b/zencore/httpserver.cpp
new file mode 100644
index 000000000..52389a11b
--- /dev/null
+++ b/zencore/httpserver.cpp
@@ -0,0 +1,1459 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/httpserver.h>
+
+#define _WINSOCKAPI_
+#include <zencore/windows.h>
+#include "iothreadpool.h"
+
+#include <atlbase.h>
+#include <conio.h>
+#include <http.h>
+#include <new.h>
+#include <zencore/compactbinary.h>
+#include <zencore/iobuffer.h>
+#include <zencore/refcount.h>
+#include <zencore/string.h>
+#include <zencore/thread.h>
+#include <charconv>
+#include <span>
+#include <string_view>
+
+#include <spdlog/spdlog.h>
+
+#include <doctest/doctest.h>
+
+#pragma comment(lib, "httpapi.lib")
+
+//////////////////////////////////////////////////////////////////////////
+
+std::wstring
+UTF8_to_wstring(const char* in)
+{
+ std::wstring out;
+ unsigned int codepoint;
+
+ while (*in != 0)
+ {
+ unsigned char ch = static_cast<unsigned char>(*in);
+
+ if (ch <= 0x7f)
+ codepoint = ch;
+ else if (ch <= 0xbf)
+ codepoint = (codepoint << 6) | (ch & 0x3f);
+ else if (ch <= 0xdf)
+ codepoint = ch & 0x1f;
+ else if (ch <= 0xef)
+ codepoint = ch & 0x0f;
+ else
+ codepoint = ch & 0x07;
+
+ ++in;
+
+ if (((*in & 0xc0) != 0x80) && (codepoint <= 0x10ffff))
+ {
+ if (sizeof(wchar_t) > 2)
+ {
+ out.append(1, static_cast<wchar_t>(codepoint));
+ }
+ else if (codepoint > 0xffff)
+ {
+ out.append(1, static_cast<wchar_t>(0xd800 + (codepoint >> 10)));
+ out.append(1, static_cast<wchar_t>(0xdc00 + (codepoint & 0x03ff)));
+ }
+ else if (codepoint < 0xd800 || codepoint >= 0xe000)
+ {
+ out.append(1, static_cast<wchar_t>(codepoint));
+ }
+ }
+ }
+
+ return out;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+const char*
+ReasonStringForHttpResultCode(int HttpCode)
+{
+ switch (HttpCode)
+ {
+ // 1xx Informational
+
+ case 100:
+ return "Continue";
+ case 101:
+ return "Switching Protocols";
+
+ // 2xx Success
+
+ case 200:
+ return "OK";
+ case 201:
+ return "Created";
+ case 202:
+ return "Accepted";
+ case 204:
+ return "No Content";
+ case 205:
+ return "Reset Content";
+ case 206:
+ return "Partial Content";
+
+ // 3xx Redirection
+
+ case 300:
+ return "Multiple Choices";
+ case 301:
+ return "Moved Permanently";
+ case 302:
+ return "Found";
+ case 303:
+ return "See Other";
+ case 304:
+ return "Not Modified";
+ case 305:
+ return "Use Proxy";
+ case 306:
+ return "Switch Proxy";
+ case 307:
+ return "Temporary Redirect";
+ case 308:
+ return "Permanent Redirect";
+
+ // 4xx Client errors
+
+ case 400:
+ return "Bad Request";
+ case 401:
+ return "Unauthorized";
+ case 402:
+ return "Payment Required";
+ case 403:
+ return "Forbidden";
+ case 404:
+ return "Not Found";
+ case 405:
+ return "Method Not Allowed";
+ case 406:
+ return "Not Acceptable";
+ case 407:
+ return "Proxy Authentication Required";
+ case 408:
+ return "Request Timeout";
+ case 409:
+ return "Conflict";
+ case 410:
+ return "Gone";
+ case 411:
+ return "Length Required";
+ case 412:
+ return "Precondition Failed";
+ case 413:
+ return "Payload Too Large";
+ case 414:
+ return "URI Too Long";
+ case 415:
+ return "Unsupported Media Type";
+ case 416:
+ return "Range Not Satisifiable";
+ case 417:
+ return "Expectation Failed";
+ case 418:
+ return "I'm a teapot";
+ case 421:
+ return "Misdirected Request";
+ case 422:
+ return "Unprocessable Entity";
+ case 423:
+ return "Locked";
+ case 424:
+ return "Failed Dependency";
+ case 425:
+ return "Too Early";
+ case 426:
+ return "Upgrade Required";
+ case 428:
+ return "Precondition Required";
+ case 429:
+ return "Too Many Requests";
+ case 431:
+ return "Request Header Fields Too Large";
+
+ // 5xx Server errors
+
+ case 500:
+ return "Internal Server Error";
+ case 501:
+ return "Not Implemented";
+ case 502:
+ return "Bad Gateway";
+ case 503:
+ return "Service Unavailable";
+ case 504:
+ return "Gateway Timeout";
+ case 505:
+ return "HTTP Version Not Supported";
+ case 506:
+ return "Variant Also Negotiates";
+ case 507:
+ return "Insufficient Storage";
+ case 508:
+ return "Loop Detected";
+ case 510:
+ return "Not Extended";
+ case 511:
+ return "Network Authentication Required";
+
+ default:
+ return "Unknown Result";
+ }
+}
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+HttpServerRequest::HttpServerRequest()
+{
+}
+
+HttpServerRequest::~HttpServerRequest()
+{
+}
+
+void
+HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, CbObject Data)
+{
+#if 0
+ struct Visitor : public ICbVisitor
+ {
+ virtual void SetName(std::string_view Name) override { OutText << '\'' << Name << "': "; }
+ virtual void BeginObject() override { OutText << "{ "; }
+ virtual void EndObject() override { OutText << "}"; }
+ virtual void BeginArray() override { OutText << "[ "; }
+ virtual void EndArray() override { OutText << " ]"; }
+
+ virtual void VisitNull() override { OutText << "null"; }
+ virtual void VisitBinary(SharedBuffer Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitString(std::string_view Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitInteger(int64_t Value) override { OutText << Value; }
+ virtual void VisitInteger(uint64_t Value) override { OutText << Value; }
+ virtual void VisitFloat(float Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitDouble(double Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitBool(bool Value) override { OutText << Value; }
+ virtual void VisitCbAttachment(const IoHash& Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitBinaryAttachment(const IoHash& Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitHash(const IoHash& Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitUuid(const Guid& Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitObjectId(const Oid& Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitDateTime(DateTime Value) override { ZEN_UNUSED(Value); }
+ virtual void VisitTimeSpan(TimeSpan Value) override { ZEN_UNUSED(Value); }
+
+ ExtendableStringBuilder<256> OutText;
+ } _;
+ // Data.CreateRefIterator().VisitFields(_);
+ return WriteResponse(HttpResponseCode, HttpContentType::kJSON, _.OutText);
+#else
+ SharedBuffer Buf = Data.GetBuffer();
+ std::array<IoBuffer, 1> buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())};
+ return WriteResponse(HttpResponseCode, HttpContentType::kCbObject, buffers);
+#endif
+}
+
+void
+HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString)
+{
+ return WriteResponse(HttpResponseCode, ContentType, std::u8string_view{(char8_t*)ResponseString.data(), ResponseString.size()});
+}
+
+void
+HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob)
+{
+ std::array<IoBuffer, 1> buffers{Blob};
+ return WriteResponse(HttpResponseCode, ContentType, buffers);
+}
+
+HttpServerRequest::QueryParams
+HttpServerRequest::GetQueryParams()
+{
+ QueryParams Params;
+
+ const std::string_view QStr = QueryString();
+
+ const char* QueryIt = QStr.data();
+ const char* QueryEnd = QueryIt + QStr.size();
+
+ while (QueryIt != QueryEnd)
+ {
+ if (*QueryIt == '&')
+ {
+ ++QueryIt;
+ continue;
+ }
+
+ const std::string_view Query{QueryIt, QueryEnd};
+
+ size_t DelimIndex = Query.find('&', 0);
+
+ if (DelimIndex == std::string_view::npos)
+ {
+ DelimIndex = Query.size();
+ }
+
+ std::string_view ThisQuery{QueryIt, DelimIndex};
+
+ size_t EqIndex = ThisQuery.find('=', 0);
+
+ if (EqIndex != std::string_view::npos)
+ {
+ std::string_view Parm{ThisQuery.data(), EqIndex};
+ ThisQuery.remove_prefix(EqIndex + 1);
+
+ Params.KvPairs.emplace_back(Parm, ThisQuery);
+ }
+
+ QueryIt += DelimIndex;
+ }
+
+ return std::move(Params);
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// HTTP
+//
+
+class HttpSysServer;
+class HttpTransaction;
+
+class HttpSysRequestHandler
+{
+public:
+ HttpSysRequestHandler(HttpTransaction& InRequest) : m_Request(InRequest) {}
+ virtual ~HttpSysRequestHandler() = default;
+
+ virtual void IssueRequest() = 0;
+ virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) = 0;
+
+ HttpTransaction& Transaction() { return m_Request; }
+
+private:
+ HttpTransaction& m_Request; // Outermost HTTP transaction object
+};
+
+/** HTTP transaction
+
+ There will be an instance of this per pending and in-flight HTTP transaction
+
+ */
+class HttpTransaction
+{
+public:
+ HttpTransaction(HttpSysServer& Server) : m_HttpServer(Server), m_HttpHandler(&m_InitialHttpHandler) {}
+
+ virtual ~HttpTransaction() {}
+
+ enum class Status
+ {
+ kDone,
+ kRequestPending
+ };
+
+ Status HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred);
+
+ static void __stdcall IoCompletionCallback(PTP_CALLBACK_INSTANCE Instance,
+ PVOID pContext /* HttpSysServer */,
+ PVOID pOverlapped,
+ ULONG IoResult,
+ ULONG_PTR NumberOfBytesTransferred,
+ PTP_IO Io)
+ {
+ UNREFERENCED_PARAMETER(Io);
+ UNREFERENCED_PARAMETER(Instance);
+ UNREFERENCED_PARAMETER(pContext);
+
+ // Note that for a given transaction we may be in this completion function on more
+ // than one thread at any given moment. This means we need to be careful about what
+ // happens in here
+
+ HttpTransaction* Transaction = CONTAINING_RECORD(pOverlapped, HttpTransaction, m_HttpOverlapped);
+
+ if (Transaction->HandleCompletion(IoResult, NumberOfBytesTransferred) == HttpTransaction::Status::kDone)
+ {
+ delete Transaction;
+ }
+ }
+
+ void IssueInitialRequest();
+
+ PTP_IO Iocp();
+ HANDLE RequestQueueHandle();
+ inline OVERLAPPED* Overlapped() { return &m_HttpOverlapped; }
+ inline HttpSysServer& Server() { return m_HttpServer; }
+
+ inline PHTTP_REQUEST HttpRequest() { return m_InitialHttpHandler.HttpRequest(); }
+
+protected:
+ OVERLAPPED m_HttpOverlapped{};
+ HttpSysServer& m_HttpServer;
+ HttpSysRequestHandler* m_HttpHandler{nullptr};
+ RwLock m_Lock;
+
+private:
+ struct InitialRequestHandler : public HttpSysRequestHandler
+ {
+ inline PHTTP_REQUEST HttpRequest() { return (PHTTP_REQUEST)m_RequestBuffer; }
+ inline uint32_t RequestBufferSize() const { return sizeof m_RequestBuffer; }
+
+ InitialRequestHandler(HttpTransaction& InRequest) : HttpSysRequestHandler(InRequest) {}
+ ~InitialRequestHandler() {}
+
+ virtual void IssueRequest() override;
+ virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override;
+
+ PHTTP_REQUEST m_HttpRequestPtr = (HTTP_REQUEST*)(m_RequestBuffer);
+ UCHAR m_RequestBuffer[16384 + sizeof(HTTP_REQUEST)];
+ } m_InitialHttpHandler{*this};
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpMessageResponseRequest : public HttpSysRequestHandler
+{
+public:
+ HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode);
+ HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const char* Message);
+ HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const void* Payload, size_t PayloadSize);
+ HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, std::span<IoBuffer> Blobs);
+ ~HttpMessageResponseRequest();
+
+ virtual void IssueRequest() override;
+ virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override;
+
+ void SuppressResponseBody();
+
+private:
+ std::vector<HTTP_DATA_CHUNK> m_HttpDataChunks;
+ uint64_t m_TotalDataSize = 0; // Sum of all chunk sizes
+
+ uint16_t m_HttpResponseCode = 0;
+ uint32_t m_NextDataChunkOffset = 0; // This is used for responses where the number of chunks exceed the maximum number for one API call
+ uint32_t m_RemainingChunkCount = 0;
+ bool m_IsInitialResponse = true;
+
+ void Initialize(uint16_t ResponseCode, std::span<IoBuffer> Blobs);
+
+ std::vector<IoBuffer> m_DataBuffers;
+};
+
+HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode) : HttpSysRequestHandler(InRequest)
+{
+ std::array<IoBuffer, 0> buffers;
+
+ Initialize(ResponseCode, buffers);
+}
+
+HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const char* Message)
+: HttpSysRequestHandler(InRequest)
+{
+ IoBuffer MessageBuffer(IoBuffer::Wrap, Message, strlen(Message));
+ std::array<IoBuffer, 1> buffers({MessageBuffer});
+
+ Initialize(ResponseCode, buffers);
+}
+
+HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest,
+ uint16_t ResponseCode,
+ const void* Payload,
+ size_t PayloadSize)
+: HttpSysRequestHandler(InRequest)
+{
+ IoBuffer MessageBuffer(IoBuffer::Wrap, Payload, PayloadSize);
+ std::array<IoBuffer, 1> buffers({MessageBuffer});
+
+ Initialize(ResponseCode, buffers);
+}
+
+HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, std::span<IoBuffer> Blobs)
+: HttpSysRequestHandler(InRequest)
+{
+ Initialize(ResponseCode, Blobs);
+}
+
+HttpMessageResponseRequest::~HttpMessageResponseRequest()
+{
+}
+
+void
+HttpMessageResponseRequest::Initialize(uint16_t ResponseCode, std::span<IoBuffer> Blobs)
+{
+ m_HttpResponseCode = ResponseCode;
+
+ const uint32_t ChunkCount = (uint32_t)Blobs.size();
+
+ m_HttpDataChunks.resize(ChunkCount);
+ m_DataBuffers.reserve(ChunkCount);
+
+ for (IoBuffer& Buffer : Blobs)
+ {
+ m_DataBuffers.emplace_back(std::move(Buffer)).MakeOwned();
+ }
+
+ // Initialize the full array up front
+
+ uint64_t LocalDataSize = 0;
+
+ {
+ PHTTP_DATA_CHUNK ChunkPtr = m_HttpDataChunks.data();
+
+ for (IoBuffer& Buffer : m_DataBuffers)
+ {
+ const ULONG BufferDataSize = (ULONG)Buffer.Size();
+
+ ZEN_ASSERT(BufferDataSize);
+
+ IoBufferFileReference FileRef;
+ if (Buffer.GetFileReference(/* out */ FileRef))
+ {
+ ChunkPtr->DataChunkType = HttpDataChunkFromFileHandle;
+ ChunkPtr->FromFileHandle.FileHandle = FileRef.FileHandle;
+ ChunkPtr->FromFileHandle.ByteRange.StartingOffset.QuadPart = FileRef.FileChunkOffset;
+ ChunkPtr->FromFileHandle.ByteRange.Length.QuadPart = BufferDataSize;
+ }
+ else
+ {
+ ChunkPtr->DataChunkType = HttpDataChunkFromMemory;
+ ChunkPtr->FromMemory.pBuffer = (void*)Buffer.Data();
+ ChunkPtr->FromMemory.BufferLength = BufferDataSize;
+ }
+ ++ChunkPtr;
+
+ LocalDataSize += BufferDataSize;
+ }
+ }
+
+ m_RemainingChunkCount = ChunkCount;
+ m_TotalDataSize = LocalDataSize;
+}
+
+void
+HttpMessageResponseRequest::SuppressResponseBody()
+{
+ m_RemainingChunkCount = 0;
+ m_HttpDataChunks.clear();
+ m_DataBuffers.clear();
+}
+
+HttpSysRequestHandler*
+HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred)
+{
+ ZEN_UNUSED(NumberOfBytesTransferred);
+ ZEN_UNUSED(IoResult);
+
+ if (m_RemainingChunkCount == 0)
+ return nullptr; // All done
+
+ return this;
+}
+
+void
+HttpMessageResponseRequest::IssueRequest()
+{
+ HttpTransaction& Tx = Transaction();
+ HTTP_REQUEST* const HttpReq = Tx.HttpRequest();
+ PTP_IO const Iocp = Tx.Iocp();
+
+ StartThreadpoolIo(Iocp);
+
+ // Split payload into batches to play well with the underlying API
+
+ const int MaxChunksPerCall = 9999;
+
+ const int ThisRequestChunkCount = std::min<int>(m_RemainingChunkCount, MaxChunksPerCall);
+ const int ThisRequestChunkOffset = m_NextDataChunkOffset;
+
+ m_RemainingChunkCount -= ThisRequestChunkCount;
+ m_NextDataChunkOffset += ThisRequestChunkCount;
+
+ ULONG SendFlags = 0;
+
+ if (m_RemainingChunkCount)
+ {
+ // We need to make more calls to send the full amount of data
+ SendFlags |= HTTP_SEND_RESPONSE_FLAG_MORE_DATA;
+ }
+
+ ULONG SendResult = 0;
+
+ if (m_IsInitialResponse)
+ {
+ // Populate response structure
+
+ HTTP_RESPONSE HttpResponse = {};
+
+ HttpResponse.EntityChunkCount = USHORT(ThisRequestChunkCount);
+ HttpResponse.pEntityChunks = m_HttpDataChunks.data() + ThisRequestChunkOffset;
+
+ // Content-length header
+
+ char ContentLengthString[32];
+ _ui64toa_s(m_TotalDataSize, ContentLengthString, sizeof ContentLengthString, 10);
+
+ PHTTP_KNOWN_HEADER ContentLengthHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentLength];
+ ContentLengthHeader->pRawValue = ContentLengthString;
+ ContentLengthHeader->RawValueLength = (USHORT)strlen(ContentLengthString);
+
+ // Content-type header
+
+ PHTTP_KNOWN_HEADER ContentTypeHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentType];
+
+ ContentTypeHeader->pRawValue = "application/octet-stream"; /* TODO! We must respect the content type specified */
+ ContentTypeHeader->RawValueLength = (USHORT)strlen(ContentTypeHeader->pRawValue);
+
+ HttpResponse.StatusCode = m_HttpResponseCode;
+ HttpResponse.pReason = ReasonStringForHttpResultCode(m_HttpResponseCode);
+ HttpResponse.ReasonLength = (USHORT)strlen(HttpResponse.pReason);
+
+ // Cache policy
+
+ HTTP_CACHE_POLICY CachePolicy;
+
+ CachePolicy.Policy = HttpCachePolicyNocache; // HttpCachePolicyUserInvalidates;
+ CachePolicy.SecondsToLive = 0;
+
+ // Initial response API call
+
+ SendResult = HttpSendHttpResponse(Tx.RequestQueueHandle(),
+ HttpReq->RequestId,
+ SendFlags,
+ &HttpResponse,
+ &CachePolicy,
+ NULL,
+ NULL,
+ 0,
+ Tx.Overlapped(),
+ NULL);
+
+ m_IsInitialResponse = false;
+ }
+ else
+ {
+ // Subsequent response API calls
+
+ SendResult = HttpSendResponseEntityBody(Tx.RequestQueueHandle(),
+ HttpReq->RequestId,
+ SendFlags,
+ (USHORT)ThisRequestChunkCount, // EntityChunkCount
+ &m_HttpDataChunks[ThisRequestChunkOffset], // EntityChunks
+ NULL, // BytesSent
+ NULL, // Reserved1
+ 0, // Reserved2
+ Tx.Overlapped(), // Overlapped
+ NULL // LogData
+ );
+ }
+
+ if ((SendResult != NO_ERROR) // Synchronous completion, but the completion event will still be posted to IOCP
+ && (SendResult != ERROR_IO_PENDING) // Asynchronous completion
+ )
+ {
+ // Some error occurred, no completion will be posted
+
+ CancelThreadpoolIo(Iocp);
+
+ spdlog::error("failed to send HTTP response (error: {}) URL: {}", SendResult, HttpReq->pRawUrl);
+
+ throw HttpServerException("Failed to send HTTP response", SendResult);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpSysServer
+{
+ friend class HttpTransaction;
+
+public:
+ HttpSysServer(WinIoThreadPool& InThreadPool);
+ ~HttpSysServer();
+
+ void Initialize(const wchar_t* UrlPath);
+ void Run(bool TestMode);
+
+ void RequestExit() { m_ShutdownEvent.Set(); }
+
+ void StartServer();
+ void StopServer();
+
+ void OnHandlingRequest();
+ void IssueNewRequestMaybe();
+
+ inline bool IsOk() const { return m_IsOk; }
+
+ void AddEndpoint(const char* Endpoint, HttpService& Service);
+ void RemoveEndpoint(const char* Endpoint, HttpService& Service);
+
+private:
+ bool m_IsOk = false;
+ bool m_IsHttpInitialized = false;
+ WinIoThreadPool& m_ThreadPool;
+
+ std::wstring m_BaseUri; // http://*:nnnn/
+ HTTP_SERVER_SESSION_ID m_HttpSessionId = 0;
+ HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0;
+ HANDLE m_RequestQueueHandle = 0;
+ std::atomic_int32_t m_PendingRequests{0};
+ int32_t m_MinPendingRequests = 4;
+ int32_t m_MaxPendingRequests = 32;
+ Event m_ShutdownEvent;
+};
+
+HttpSysServer::HttpSysServer(WinIoThreadPool& InThreadPool) : m_ThreadPool(InThreadPool)
+{
+ ULONG Result = HttpInitialize(HTTPAPI_VERSION_2, HTTP_INITIALIZE_SERVER, nullptr);
+
+ if (Result != NO_ERROR)
+ {
+ return;
+ }
+
+ m_IsHttpInitialized = true;
+ m_IsOk = true;
+}
+
+HttpSysServer::~HttpSysServer()
+{
+ if (m_IsHttpInitialized)
+ {
+ HttpTerminate(HTTP_INITIALIZE_SERVER, nullptr);
+ }
+}
+
+void
+HttpSysServer::Initialize(const wchar_t* UrlPath)
+{
+ // check(bIsOk);
+
+ ULONG Result = HttpCreateServerSession(HTTPAPI_VERSION_2, &m_HttpSessionId, 0);
+
+ if (Result != NO_ERROR)
+ {
+ // Flag error
+
+ return;
+ }
+
+ Result = HttpCreateUrlGroup(m_HttpSessionId, &m_HttpUrlGroupId, 0);
+
+ if (Result != NO_ERROR)
+ {
+ // Flag error
+
+ return;
+ }
+
+ m_BaseUri = UrlPath;
+
+ Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, UrlPath, /* #TODO UrlContext */ HTTP_URL_CONTEXT(0), 0);
+
+ if (Result != NO_ERROR)
+ {
+ // Flag error
+
+ return;
+ }
+
+ HTTP_BINDING_INFO HttpBindingInfo = {{0}, 0};
+
+ Result = HttpCreateRequestQueue(HTTPAPI_VERSION_2, NULL, NULL, 0, &m_RequestQueueHandle);
+
+ if (Result != NO_ERROR)
+ {
+ // Flag error!
+
+ return;
+ }
+
+ HttpBindingInfo.Flags.Present = 1;
+ HttpBindingInfo.RequestQueueHandle = m_RequestQueueHandle;
+
+ Result = HttpSetUrlGroupProperty(m_HttpUrlGroupId, HttpServerBindingProperty, &HttpBindingInfo, sizeof(HttpBindingInfo));
+
+ if (Result != NO_ERROR)
+ {
+ // Flag error!
+
+ return;
+ }
+
+ // Create I/O completion port
+
+ m_ThreadPool.CreateIocp(m_RequestQueueHandle, HttpTransaction::IoCompletionCallback, this);
+
+ // Check result!
+}
+
+void
+HttpSysServer::StartServer()
+{
+ int RequestCount = 32;
+
+ for (int i = 0; i < RequestCount; ++i)
+ {
+ IssueNewRequestMaybe();
+ }
+}
+
+void
+HttpSysServer::Run(bool TestMode)
+{
+ if (TestMode == false)
+ {
+ printf("Zen Server running. Press ESC or Q to quit\n");
+ }
+
+ bool KeepRunning = true;
+
+ do
+ {
+ int WaitTimeout = -1;
+
+ if (!TestMode)
+ {
+ WaitTimeout = 1000;
+ }
+
+ if (!TestMode && _kbhit() != 0)
+ {
+ char c = (char)_getch();
+
+ if (c == 27 || c == 'Q' || c == 'q')
+ {
+ RequestApplicationExit(0);
+ }
+ }
+
+ m_ShutdownEvent.Wait(WaitTimeout);
+ } while (!IsApplicationExitRequested());
+}
+
+void
+HttpSysServer::OnHandlingRequest()
+{
+ --m_PendingRequests;
+
+ if (m_PendingRequests > m_MinPendingRequests)
+ {
+ // We have more than the minimum number of requests pending, just let someone else
+ // enqueue new requests
+ return;
+ }
+
+ IssueNewRequestMaybe();
+}
+
+void
+HttpSysServer::IssueNewRequestMaybe()
+{
+ if (m_PendingRequests.load(std::memory_order::relaxed) >= m_MaxPendingRequests)
+ {
+ return;
+ }
+
+ std::unique_ptr<HttpTransaction> Request = std::make_unique<HttpTransaction>(*this);
+
+ Request->IssueInitialRequest();
+
+ // This may end up exceeding the MaxPendingRequests limit, but it's not
+ // really a problem. I'm doing it this way mostly to avoid dealing with
+ // exceptions here
+ ++m_PendingRequests;
+
+ Request.release();
+}
+
+void
+HttpSysServer::StopServer()
+{
+}
+
+void
+HttpSysServer::AddEndpoint(const char* UrlPath, HttpService& Service)
+{
+ if (UrlPath[0] == '/')
+ {
+ ++UrlPath;
+ }
+
+ const std::wstring Path16 = UTF8_to_wstring(UrlPath);
+ Service.SetUriPrefixLength(Path16.size() + 1 /* leading slash */);
+
+ // Convert to wide string
+
+ std::wstring Url16 = m_BaseUri + Path16;
+
+ ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */);
+
+ if (Result != NO_ERROR)
+ {
+ spdlog::error("HttpAddUrlToUrlGroup failed with result {}", Result);
+
+ return;
+ }
+}
+
+void
+HttpSysServer::RemoveEndpoint(const char* UrlPath, HttpService& Service)
+{
+ ZEN_UNUSED(Service);
+
+ if (UrlPath[0] == '/')
+ {
+ ++UrlPath;
+ }
+
+ const std::wstring Path16 = UTF8_to_wstring(UrlPath);
+
+ // Convert to wide string
+
+ std::wstring Url16 = m_BaseUri + Path16;
+
+ ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0);
+
+ if (Result != NO_ERROR)
+ {
+ spdlog::error("HttpRemoveUrlFromUrlGroup failed with result {}", Result);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpSysServerRequest : public HttpServerRequest
+{
+public:
+ HttpSysServerRequest(HttpTransaction& Tx, HttpService& Service) : m_HttpTx(Tx)
+ {
+ PHTTP_REQUEST HttpRequestPtr = Tx.HttpRequest();
+
+ const int PrefixLength = Service.UriPrefixLength();
+ const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t);
+
+ if (AbsPathLength >= PrefixLength)
+ {
+ // We convert the URI immediately because most of the code involved prefers to deal
+ // with utf8. This has some performance impact which I'd prefer to avoid but for now
+ // we just have to live with it
+
+ WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)},
+ m_Uri);
+ }
+ else
+ {
+ m_Uri.Reset();
+ }
+
+ if (auto QueryStringLength = HttpRequestPtr->CookedUrl.QueryStringLength)
+ {
+ --QueryStringLength;
+
+ WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryString);
+ }
+ else
+ {
+ m_QueryString.Reset();
+ }
+
+ switch (HttpRequestPtr->Verb)
+ {
+ case HttpVerbOPTIONS:
+ m_Verb = HttpVerb::kOptions;
+ break;
+
+ case HttpVerbGET:
+ m_Verb = HttpVerb::kGet;
+ break;
+
+ case HttpVerbHEAD:
+ m_Verb = HttpVerb::kHead;
+ break;
+
+ case HttpVerbPOST:
+ m_Verb = HttpVerb::kPost;
+ break;
+
+ case HttpVerbPUT:
+ m_Verb = HttpVerb::kPut;
+ break;
+
+ case HttpVerbDELETE:
+ m_Verb = HttpVerb::kDelete;
+ break;
+
+ case HttpVerbCOPY:
+ m_Verb = HttpVerb::kCopy;
+ break;
+
+ default:
+ // TODO: invalid request?
+ m_Verb = (HttpVerb)0;
+ break;
+ }
+
+ auto& clh = HttpRequestPtr->Headers.KnownHeaders[HttpHeaderContentLength];
+ std::string_view cl(clh.pRawValue, clh.RawValueLength);
+
+ std::from_chars(cl.data(), cl.data() + cl.size(), m_ContentLength);
+ }
+
+ ~HttpSysServerRequest() {}
+
+ virtual IoBuffer ReadPayload() override
+ {
+ // This is presently synchronous for simplicity, but we
+ // need to implement an asynchronous version also
+
+ HTTP_REQUEST* const HttpReq = m_HttpTx.HttpRequest();
+
+ IoBuffer buffer(m_ContentLength);
+
+ uint64_t BytesToRead = m_ContentLength;
+
+ uint8_t* ReadPointer = (uint8_t*)buffer.Data();
+
+ // First deal with any payload which has already been copied
+ // into our request buffer
+
+ const int EntityChunkCount = HttpReq->EntityChunkCount;
+
+ for (int i = 0; i < EntityChunkCount; ++i)
+ {
+ HTTP_DATA_CHUNK& EntityChunk = HttpReq->pEntityChunks[i];
+
+ ZEN_ASSERT(EntityChunk.DataChunkType == HttpDataChunkFromMemory);
+
+ const uint64_t BufferLength = EntityChunk.FromMemory.BufferLength;
+
+ ZEN_ASSERT(BufferLength <= BytesToRead);
+
+ memcpy(ReadPointer, EntityChunk.FromMemory.pBuffer, BufferLength);
+
+ ReadPointer += BufferLength;
+ BytesToRead -= BufferLength;
+ }
+
+ // Call http.sys API to receive the remaining data
+
+ while (BytesToRead)
+ {
+ ULONG BytesRead = 0;
+
+ ULONG ApiResult = HttpReceiveRequestEntityBody(m_HttpTx.RequestQueueHandle(),
+ HttpReq->RequestId,
+ 0, /* Flags */
+ ReadPointer,
+ (ULONG)BytesToRead,
+ &BytesRead,
+ NULL /* Overlapped */
+ );
+
+ if (ApiResult != NO_ERROR && ApiResult != ERROR_HANDLE_EOF)
+ {
+ throw HttpServerException("payload read failed", ApiResult);
+ }
+
+ BytesToRead -= BytesRead;
+ ReadPointer += BytesRead;
+ }
+
+ return buffer;
+ }
+
+ virtual void WriteResponse(HttpResponse HttpResponseCode) override
+ {
+ ZEN_ASSERT(m_IsHandled == false);
+
+ m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode);
+
+ if (m_SuppressBody)
+ {
+ m_Response->SuppressResponseBody();
+ }
+
+ m_IsHandled = true;
+ }
+
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) override
+ {
+ ZEN_ASSERT(m_IsHandled == false);
+ ZEN_UNUSED(ContentType);
+
+ m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode, Blobs);
+
+ if (m_SuppressBody)
+ {
+ m_Response->SuppressResponseBody();
+ }
+
+ m_IsHandled = true;
+ }
+
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) override
+ {
+ ZEN_ASSERT(m_IsHandled == false);
+ ZEN_UNUSED(ContentType);
+
+ m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode, ResponseString.data(), ResponseString.size());
+
+ if (m_SuppressBody)
+ {
+ m_Response->SuppressResponseBody();
+ }
+
+ m_IsHandled = true;
+ }
+
+ HttpTransaction& m_HttpTx;
+ HttpMessageResponseRequest* m_Response = nullptr;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+PTP_IO
+HttpTransaction::Iocp()
+{
+ return m_HttpServer.m_ThreadPool.Iocp();
+}
+
+HANDLE
+HttpTransaction::RequestQueueHandle()
+{
+ return m_HttpServer.m_RequestQueueHandle;
+}
+
+void
+HttpTransaction::IssueInitialRequest()
+{
+ m_InitialHttpHandler.IssueRequest();
+}
+
+HttpTransaction::Status
+HttpTransaction::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred)
+{
+ // We use this to ensure sequential execution of completion handlers
+ // for any given transaction.
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ bool RequestPending = false;
+
+ if (HttpSysRequestHandler* CurrentHandler = m_HttpHandler)
+ {
+ const bool IsInitialRequest = (CurrentHandler == &m_InitialHttpHandler);
+
+ if (IsInitialRequest)
+ {
+ // Ensure we have a sufficient number of pending requests outstanding
+ m_HttpServer.OnHandlingRequest();
+ }
+
+ m_HttpHandler = CurrentHandler->HandleCompletion(IoResult, NumberOfBytesTransferred);
+
+ if (m_HttpHandler)
+ {
+ try
+ {
+ m_HttpHandler->IssueRequest();
+
+ RequestPending = true;
+ }
+ catch (std::exception& Ex)
+ {
+ spdlog::error("exception caught from IssueRequest(): {}", Ex.what());
+
+ // something went wrong, no request is pending
+ }
+ }
+ else
+ {
+ if (IsInitialRequest == false)
+ {
+ delete CurrentHandler;
+ }
+ }
+ }
+
+ m_HttpServer.IssueNewRequestMaybe();
+
+ if (RequestPending)
+ {
+ return Status::kRequestPending;
+ }
+
+ return Status::kDone;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+HttpTransaction::InitialRequestHandler::IssueRequest()
+{
+ PTP_IO Iocp = Transaction().Iocp();
+
+ StartThreadpoolIo(Iocp);
+
+ HttpTransaction& Tx = Transaction();
+
+ HTTP_REQUEST* HttpReq = Tx.HttpRequest();
+
+ ULONG Result = HttpReceiveHttpRequest(Tx.RequestQueueHandle(),
+ HTTP_NULL_ID,
+ HTTP_RECEIVE_REQUEST_FLAG_COPY_BODY,
+ HttpReq,
+ RequestBufferSize(),
+ NULL,
+ Tx.Overlapped());
+
+ if (Result != ERROR_IO_PENDING && Result != NO_ERROR)
+ {
+ CancelThreadpoolIo(Iocp);
+
+ if (Result == ERROR_MORE_DATA)
+ {
+ // ProcessReceiveAndPostResponse(pIoRequest, pServerContext->Io, ERROR_MORE_DATA);
+ }
+
+ // CleanupHttpIoRequest(pIoRequest);
+
+ fprintf(stderr, "HttpReceiveHttpRequest failed, error 0x%lx\n", Result);
+
+ return;
+ }
+}
+
+HttpSysRequestHandler*
+HttpTransaction::InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred)
+{
+ ZEN_UNUSED(IoResult);
+ ZEN_UNUSED(NumberOfBytesTransferred);
+
+ // Route requests
+
+ try
+ {
+ if (HttpService* Service = reinterpret_cast<HttpService*>(m_HttpRequestPtr->UrlContext))
+ {
+ HttpSysServerRequest ThisRequest(Transaction(), *Service);
+
+ Service->HandleRequest(ThisRequest);
+
+ if (!ThisRequest.IsHandled())
+ {
+ return new HttpMessageResponseRequest(Transaction(), 404, "Not found");
+ }
+
+ if (ThisRequest.m_Response)
+ {
+ return ThisRequest.m_Response;
+ }
+ }
+
+ // Unable to route
+ return new HttpMessageResponseRequest(Transaction(), 404, "Item unknown");
+ }
+ catch (std::exception& ex)
+ {
+ // TODO provide more meaningful error output
+
+ return new HttpMessageResponseRequest(Transaction(), 500, ex.what());
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+struct HttpServer::Impl : public RefCounted
+{
+ WinIoThreadPool m_ThreadPool;
+ HttpSysServer m_HttpServer;
+
+ Impl(int ThreadCount) : m_ThreadPool(ThreadCount), m_HttpServer(m_ThreadPool) {}
+
+ void Initialize(int BasePort)
+ {
+ using namespace std::literals;
+
+ WideStringBuilder<64> BaseUri;
+ BaseUri << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv;
+
+ m_HttpServer.Initialize(BaseUri.c_str());
+ m_HttpServer.StartServer();
+ }
+
+ void Run(bool TestMode) { m_HttpServer.Run(TestMode); }
+
+ void RequestExit() { m_HttpServer.RequestExit(); }
+
+ void Cleanup() { m_HttpServer.StopServer(); }
+
+ void AddEndpoint(const char* Endpoint, HttpService& Service) { m_HttpServer.AddEndpoint(Endpoint, Service); }
+
+ void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler)
+ {
+ ZEN_UNUSED(endpoint, handler);
+
+ ZEN_NOT_IMPLEMENTED();
+ }
+};
+
+HttpServer::HttpServer()
+{
+ m_Impl = new Impl(32);
+}
+
+HttpServer::~HttpServer()
+{
+ m_Impl->Cleanup();
+}
+
+void
+HttpServer::AddEndpoint(HttpService& Service)
+{
+ m_Impl->AddEndpoint(Service.BaseUri(), Service);
+}
+
+void
+HttpServer::AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler)
+{
+ m_Impl->AddEndpoint(endpoint, handler);
+}
+
+void
+HttpServer::Initialize(int BasePort)
+{
+ m_Impl->Initialize(BasePort);
+}
+
+void
+HttpServer::Run(bool TestMode)
+{
+ m_Impl->Run(TestMode);
+}
+
+void
+HttpServer::RequestExit()
+{
+ m_Impl->RequestExit();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+HttpRequestRouter::AddPattern(const char* Id, const char* Regex)
+{
+ ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end());
+
+ m_PatternMap.insert({Id, Regex});
+}
+
+void
+HttpRequestRouter::RegisterRoute(const char* Regex, HttpRequestRouter::HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs)
+{
+ // Expand patterns
+
+ ExtendableStringBuilder<128> ExpandedRegex;
+
+ size_t RegexLen = strlen(Regex);
+
+ for (size_t i = 0; i < RegexLen;)
+ {
+ bool matched = false;
+
+ if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\')))
+ {
+ // Might have a pattern reference - find closing brace
+
+ for (size_t j = i + 1; j < RegexLen; ++j)
+ {
+ if (Regex[j] == '}')
+ {
+ std::string Pattern(&Regex[i + 1], j - i - 1);
+
+ if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end())
+ {
+ ExpandedRegex.Append(it->second.c_str());
+ }
+ else
+ {
+ // Default to anything goes (or should this just be an error?)
+
+ ExpandedRegex.Append("(.+?)");
+ }
+
+ // skip ahead
+ i = j + 1;
+
+ matched = true;
+
+ break;
+ }
+ }
+ }
+
+ if (!matched)
+ {
+ ExpandedRegex.Append(Regex[i++]);
+ }
+ }
+
+ m_Handlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), Regex);
+}
+
+bool
+HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request)
+{
+ const HttpVerb Verb = Request.RequestVerb();
+
+ std::string_view Uri = Request.RelativeUri();
+ HttpRouterRequest RouterRequest(Request);
+
+ for (const auto& Handler : m_Handlers)
+ {
+ if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx))
+ {
+ Handler.Handler(RouterRequest);
+
+ return true; // Route matched
+ }
+ }
+
+ return false; // No route matched
+}
+
+TEST_CASE("http")
+{
+ using namespace std::literals;
+
+ SUBCASE("router")
+ {
+ HttpRequestRouter r;
+ r.AddPattern("a", "[[:alpha:]]+");
+ r.RegisterRoute(
+ "{a}",
+ [&](auto) {},
+ HttpVerb::kGet);
+
+ // struct TestHttpServerRequest : public HttpServerRequest
+ //{
+ // TestHttpServerRequest(std::string_view Uri) : m_uri{Uri} {}
+ //};
+
+ // TestHttpServerRequest req{};
+ // r.HandleRequest(req);
+ }
+}
+
+void
+http_forcelink()
+{
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/atomic.h b/zencore/include/zencore/atomic.h
new file mode 100644
index 000000000..457128bd4
--- /dev/null
+++ b/zencore/include/zencore/atomic.h
@@ -0,0 +1,43 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <intrin.h>
+#include <cinttypes>
+
+namespace zen {
+
+inline uint32_t
+AtomicIncrement(volatile uint32_t& value)
+{
+ return _InterlockedIncrement((long volatile*)&value);
+}
+inline uint32_t
+AtomicDecrement(volatile uint32_t& value)
+{
+ return _InterlockedDecrement((long volatile*)&value);
+}
+
+inline uint64_t
+AtomicIncrement(volatile uint64_t& value)
+{
+ return _InterlockedIncrement64((__int64 volatile*)&value);
+}
+inline uint64_t
+AtomicDecrement(volatile uint64_t& value)
+{
+ return _InterlockedDecrement64((__int64 volatile*)&value);
+}
+
+inline uint32_t
+AtomicAdd(volatile uint32_t& value, uint32_t amount)
+{
+ return _InterlockedExchangeAdd((long volatile*)&value, amount);
+}
+inline uint64_t
+AtomicAdd(volatile uint64_t& value, uint64_t amount)
+{
+ return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount);
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/blake3.h b/zencore/include/zencore/blake3.h
new file mode 100644
index 000000000..1ef921c30
--- /dev/null
+++ b/zencore/include/zencore/blake3.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <cinttypes>
+#include <compare>
+#include <cstring>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * BLAKE3 hash - 256 bits
+ */
+struct BLAKE3
+{
+ uint8_t Hash[32];
+
+ inline auto operator<=>(const BLAKE3& rhs) const = default;
+
+ static BLAKE3 HashMemory(const void* data, size_t byteCount);
+ static BLAKE3 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static const int StringLength = 64;
+ typedef char String_t[StringLength + 1];
+
+ static BLAKE3 Zero; // Initialized to all zeroes
+
+ struct Hasher
+ {
+ size_t operator()(const BLAKE3& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct BLAKE3Stream
+{
+ BLAKE3Stream();
+
+ void Reset(); /// Begin streaming hash compute (not needed on freshly constructed instance)
+ BLAKE3Stream& Append(const void* data, size_t byteCount); /// Append another chunk
+ BLAKE3 GetHash(); /// Obtain final hash. If you wish to reuse the instance call reset()
+
+private:
+ alignas(16) uint8_t m_HashState[2048];
+};
+
+void blake3_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
new file mode 100644
index 000000000..c2d276c21
--- /dev/null
+++ b/zencore/include/zencore/compactbinary.h
@@ -0,0 +1,1335 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/enumflags.h>
+#include <zencore/intmath.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/memory.h>
+#include <zencore/meta.h>
+#include <zencore/sharedbuffer.h>
+#include <zencore/uid.h>
+#include <zencore/varint.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class CbObjectView;
+class CbArrayView;
+class BinaryReader;
+class BinaryWriter;
+
+class DateTime
+{
+public:
+ explicit DateTime(uint64_t InTicks) : Ticks(InTicks) {}
+ inline DateTime(int Year, int Month, int Day, int Hours = 0, int Minutes = 0, int Seconds = 0, int MilliSeconds = 0)
+ {
+ Set(Year, Month, Day, Hours, Minutes, Seconds, MilliSeconds);
+ }
+
+ inline uint64_t GetTicks() const { return Ticks; }
+ inline bool operator==(const DateTime& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const DateTime& Rhs) const { return Ticks - Rhs.Ticks; }
+
+private:
+ void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond);
+ uint64_t Ticks;
+};
+
+class TimeSpan
+{
+public:
+ explicit TimeSpan(uint64_t InTicks) : Ticks(InTicks) {}
+ inline TimeSpan(int Hours, int Minutes, int Seconds) { Set(0, Hours, Minutes, Seconds, 0); }
+ inline TimeSpan(int Days, int Hours, int Minutes, int Seconds) { Set(Days, Hours, Minutes, Seconds, 0); }
+ inline TimeSpan(int Days, int Hours, int Minutes, int Seconds, int Nanos) { Set(Days, Hours, Minutes, Seconds, Nanos); }
+
+ inline uint64_t GetTicks() const { return Ticks; }
+ inline bool operator==(const TimeSpan& Rhs) const { return Ticks == Rhs.Ticks; }
+ inline auto operator<=>(const TimeSpan& Rhs) const { return Ticks - Rhs.Ticks; }
+
+ /**
+ * Time span related constants.
+ */
+
+ /** The maximum number of ticks that can be represented in FTimespan. */
+ static constexpr int64_t MaxTicks = 9223372036854775807;
+
+ /** The minimum number of ticks that can be represented in FTimespan. */
+ static constexpr int64_t MinTicks = -9223372036854775807 - 1;
+
+ /** The number of nanoseconds per tick. */
+ static constexpr int64_t NanosecondsPerTick = 100;
+
+ /** The number of timespan ticks per day. */
+ static constexpr int64_t TicksPerDay = 864000000000;
+
+ /** The number of timespan ticks per hour. */
+ static constexpr int64_t TicksPerHour = 36000000000;
+
+ /** The number of timespan ticks per microsecond. */
+ static constexpr int64_t TicksPerMicrosecond = 10;
+
+ /** The number of timespan ticks per millisecond. */
+ static constexpr int64_t TicksPerMillisecond = 10000;
+
+ /** The number of timespan ticks per minute. */
+ static constexpr int64_t TicksPerMinute = 600000000;
+
+ /** The number of timespan ticks per second. */
+ static constexpr int64_t TicksPerSecond = 10000000;
+
+ /** The number of timespan ticks per week. */
+ static constexpr int64_t TicksPerWeek = 6048000000000;
+
+ /** The number of timespan ticks per year (365 days, not accounting for leap years). */
+ static constexpr int64_t TicksPerYear = 365 * TicksPerDay;
+
+private:
+ void Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano);
+
+ uint64_t Ticks;
+};
+
+struct Guid
+{
+ uint32_t A, B, C, D;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/**
+ * Field types and flags for CbField.
+ *
+ * This is a private type and is only declared here to enable inline use below.
+ *
+ * DO NOT CHANGE THE VALUE OF ANY MEMBERS OF THIS ENUM!
+ * BACKWARD COMPATIBILITY REQUIRES THAT THESE VALUES BE FIXED!
+ * SERIALIZATION USES HARD-CODED CONSTANTS BASED ON THESE VALUES!
+ */
+enum class CbFieldType : uint8_t
+{
+ /** A field type that does not occur in a valid object. */
+ None = 0x00,
+
+ /** Null. Payload is empty. */
+ Null = 0x01,
+
+ /**
+ * Object is an array of fields with unique non-empty names.
+ *
+ * Payload is a VarUInt byte count for the encoded fields followed by the fields.
+ */
+ Object = 0x02,
+ /**
+ * UniformObject is an array of fields with the same field types and unique non-empty names.
+ *
+ * Payload is a VarUInt byte count for the encoded fields followed by the fields.
+ */
+ UniformObject = 0x03,
+
+ /**
+ * Array is an array of fields with no name that may be of different types.
+ *
+ * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by the fields.
+ */
+ Array = 0x04,
+ /**
+ * UniformArray is an array of fields with no name and with the same field type.
+ *
+ * Payload is a VarUInt byte count, followed by a VarUInt item count, followed by field type,
+ * followed by the fields without their field type.
+ */
+ UniformArray = 0x05,
+
+ /** Binary. Payload is a VarUInt byte count followed by the data. */
+ Binary = 0x06,
+
+ /** String in UTF-8. Payload is a VarUInt byte count then an unterminated UTF-8 string. */
+ String = 0x07,
+
+ /**
+ * Non-negative integer with the range of a 64-bit unsigned integer.
+ *
+ * Payload is the value encoded as a VarUInt.
+ */
+ IntegerPositive = 0x08,
+ /**
+ * Negative integer with the range of a 64-bit signed integer.
+ *
+ * Payload is the ones' complement of the value encoded as a VarUInt.
+ */
+ IntegerNegative = 0x09,
+
+ /** Single precision float. Payload is one big endian IEEE 754 binary32 float. */
+ Float32 = 0x0a,
+ /** Double precision float. Payload is one big endian IEEE 754 binary64 float. */
+ Float64 = 0x0b,
+
+ /** Boolean false value. Payload is empty. */
+ BoolFalse = 0x0c,
+ /** Boolean true value. Payload is empty. */
+ BoolTrue = 0x0d,
+
+ /**
+ * CompactBinaryAttachment is a reference to a compact binary attachment stored externally.
+ *
+ * Payload is a 160-bit hash digest of the referenced compact binary data.
+ */
+ CompactBinaryAttachment = 0x0e,
+ /**
+ * BinaryAttachment is a reference to a binary attachment stored externally.
+ *
+ * Payload is a 160-bit hash digest of the referenced binary data.
+ */
+ BinaryAttachment = 0x0f,
+
+ /** Hash. Payload is a 160-bit hash digest. */
+ Hash = 0x10,
+ /** UUID/GUID. Payload is a 128-bit UUID as defined by RFC 4122. */
+ Uuid = 0x11,
+
+ /**
+ * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999.
+ *
+ * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000.
+ */
+ DateTime = 0x12,
+ /**
+ * Difference between two date/time values.
+ *
+ * Payload is a big endian int64 count of 100ns ticks in the span, and may be negative.
+ */
+ TimeSpan = 0x13,
+
+ /**
+ * Object ID
+ *
+ * Payload is a 12-byte opaque identifier
+ */
+ ObjectId = 0x14,
+
+ /**
+ * CustomById identifies the sub-type of its payload by an integer identifier.
+ *
+ * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed
+ * by a VarUInt of the sub-type identifier then the payload of the sub-type.
+ */
+ CustomById = 0x1e,
+ /**
+ * CustomByType identifies the sub-type of its payload by a string identifier.
+ *
+ * Payload is a VarUInt byte count of the sub-type identifier and the sub-type payload, followed
+ * by a VarUInt byte count of the unterminated sub-type identifier, then the sub-type identifier
+ * without termination, then the payload of the sub-type.
+ */
+ CustomByName = 0x1f,
+
+ /** Reserved for future use as a flag. Do not add types in this range. */
+ Reserved = 0x20,
+
+ /**
+ * A transient flag which indicates that the object or array containing this field has stored
+ * the field type before the payload and name. Non-uniform objects and fields will set this.
+ *
+ * Note: Since the flag must never be serialized, this bit may be repurposed in the future.
+ */
+ HasFieldType = 0x40,
+
+ /** A persisted flag which indicates that the field has a name stored before the payload. */
+ HasFieldName = 0x80,
+};
+
+ENUM_CLASS_FLAGS(CbFieldType);
+
+/** Functions that operate on CbFieldType. */
+class CbFieldTypeOps
+{
+ static constexpr CbFieldType SerializedTypeMask = CbFieldType(0b1011'1111);
+ static constexpr CbFieldType TypeMask = CbFieldType(0b0011'1111);
+ static constexpr CbFieldType ObjectMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType ObjectBase = CbFieldType(0b0000'0010);
+ static constexpr CbFieldType ArrayMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType ArrayBase = CbFieldType(0b0000'0100);
+ static constexpr CbFieldType IntegerMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType IntegerBase = CbFieldType(0b0000'1000);
+ static constexpr CbFieldType FloatMask = CbFieldType(0b0011'1100);
+ static constexpr CbFieldType FloatBase = CbFieldType(0b0000'1000);
+ static constexpr CbFieldType BoolMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType BoolBase = CbFieldType(0b0000'1100);
+ static constexpr CbFieldType AttachmentMask = CbFieldType(0b0011'1110);
+ static constexpr CbFieldType AttachmentBase = CbFieldType(0b0000'1110);
+
+ static void StaticAssertTypeConstants();
+
+public:
+ /** The type with flags removed. */
+ static constexpr inline CbFieldType GetType(CbFieldType Type) { return Type & TypeMask; }
+ /** The type with transient flags removed. */
+ static constexpr inline CbFieldType GetSerializedType(CbFieldType Type) { return Type & SerializedTypeMask; }
+
+ static constexpr inline bool HasFieldType(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldType); }
+ static constexpr inline bool HasFieldName(CbFieldType Type) { return EnumHasAnyFlags(Type, CbFieldType::HasFieldName); }
+
+ static constexpr inline bool IsNone(CbFieldType Type) { return GetType(Type) == CbFieldType::None; }
+ static constexpr inline bool IsNull(CbFieldType Type) { return GetType(Type) == CbFieldType::Null; }
+
+ static constexpr inline bool IsObject(CbFieldType Type) { return (Type & ObjectMask) == ObjectBase; }
+ static constexpr inline bool IsArray(CbFieldType Type) { return (Type & ArrayMask) == ArrayBase; }
+
+ static constexpr inline bool IsBinary(CbFieldType Type) { return GetType(Type) == CbFieldType::Binary; }
+ static constexpr inline bool IsString(CbFieldType Type) { return GetType(Type) == CbFieldType::String; }
+
+ static constexpr inline bool IsInteger(CbFieldType Type) { return (Type & IntegerMask) == IntegerBase; }
+ /** Whether the field is a float, or integer due to implicit conversion. */
+ static constexpr inline bool IsFloat(CbFieldType Type) { return (Type & FloatMask) == FloatBase; }
+ static constexpr inline bool IsBool(CbFieldType Type) { return (Type & BoolMask) == BoolBase; }
+
+ static constexpr inline bool IsCompactBinaryAttachment(CbFieldType Type)
+ {
+ return GetType(Type) == CbFieldType::CompactBinaryAttachment;
+ }
+ static constexpr inline bool IsBinaryAttachment(CbFieldType Type) { return GetType(Type) == CbFieldType::BinaryAttachment; }
+ static constexpr inline bool IsAttachment(CbFieldType Type) { return (Type & AttachmentMask) == AttachmentBase; }
+
+ static constexpr inline bool IsHash(CbFieldType Type)
+ {
+ switch (GetType(Type))
+ {
+ case CbFieldType::Hash:
+ case CbFieldType::BinaryAttachment:
+ case CbFieldType::CompactBinaryAttachment:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static constexpr inline bool IsUuid(CbFieldType Type) { return GetType(Type) == CbFieldType::Uuid; }
+ static constexpr inline bool IsObjectId(CbFieldType Type) { return GetType(Type) == CbFieldType::ObjectId; }
+
+ static constexpr inline bool IsDateTime(CbFieldType Type) { return GetType(Type) == CbFieldType::DateTime; }
+ static constexpr inline bool IsTimeSpan(CbFieldType Type) { return GetType(Type) == CbFieldType::TimeSpan; }
+
+ /** Whether the type is or may contain fields of any attachment type. */
+ static constexpr inline bool MayContainAttachments(CbFieldType Type)
+ {
+ // The use of !! will suppress V792 from static analysis. Using //-V792 did not work.
+ return !!IsObject(Type) | !!IsArray(Type) | !!IsAttachment(Type);
+ }
+};
+
+/** Errors that can occur when accessing a field. */
+enum class CbFieldError : uint8_t
+{
+ /** The field is not in an error state. */
+ None,
+ /** The value type does not match the requested type. */
+ TypeError,
+ /** The value is out of range for the requested type. */
+ RangeError,
+};
+
+class ICbVisitor
+{
+public:
+ virtual void SetName(std::string_view Name) = 0;
+ virtual void BeginObject() = 0;
+ virtual void EndObject() = 0;
+ virtual void BeginArray() = 0;
+ virtual void EndArray() = 0;
+ virtual void VisitNull() = 0;
+ virtual void VisitBinary(SharedBuffer Value) = 0;
+ virtual void VisitString(std::string_view Value) = 0;
+ virtual void VisitInteger(int64_t Value) = 0;
+ virtual void VisitInteger(uint64_t Value) = 0;
+ virtual void VisitFloat(float Value) = 0;
+ virtual void VisitDouble(double Value) = 0;
+ virtual void VisitBool(bool value) = 0;
+ virtual void VisitCbAttachment(const IoHash& Value) = 0;
+ virtual void VisitBinaryAttachment(const IoHash& Value) = 0;
+ virtual void VisitHash(const IoHash& Value) = 0;
+ virtual void VisitUuid(const Guid& Value) = 0;
+ virtual void VisitObjectId(const Oid& Value) = 0;
+ virtual void VisitDateTime(DateTime Value) = 0;
+ virtual void VisitTimeSpan(TimeSpan Value) = 0;
+};
+
+/**
+ * An atom of data in the compact binary format.
+ *
+ * Accessing the value of a field is always a safe operation, even if accessed as the wrong type.
+ * An invalid access will return a default value for the requested type, and set an error code on
+ * the field that can be checked with GetLastError and HasLastError. A valid access will clear an
+ * error from a previous invalid access.
+ *
+ * A field is encoded in one or more bytes, depending on its type and the type of object or array
+ * that contains it. A field of an object or array which is non-uniform encodes its field type in
+ * the first byte, and includes the HasFieldName flag for a field in an object. The field name is
+ * encoded in a variable-length unsigned integer of its size in bytes, for named fields, followed
+ * by that many bytes of the UTF-8 encoding of the name with no null terminator. The remainder of
+ * the field is the payload and is described in the field type enum. Every field must be uniquely
+ * addressable when encoded, which means a zero-byte field is not permitted, and only arises in a
+ * uniform array of fields with no payload, where the answer is to encode as a non-uniform array.
+ *
+ * This type only provides a view into memory and does not perform any memory management itself.
+ * Use CbFieldRef to hold a reference to the underlying memory when necessary.
+ */
+
+class CbFieldView
+{
+public:
+ CbFieldView() = default;
+
+ ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType);
+
+ /** Returns the name of the field if it has a name, otherwise an empty view. */
+ constexpr inline std::string_view GetName() const { return std::string_view(static_cast<const char*>(Payload) - NameLen, NameLen); }
+
+ ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView());
+ ZENCORE_API CbObjectView AsObjectView();
+ ZENCORE_API CbArrayView AsArrayView();
+ ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view());
+
+ ZENCORE_API void IterateAttachments(std::function<void(CbFieldView)> Visitor) const;
+
+ /** Access the field as an int8. Returns the provided default on error. */
+ inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); }
+ /** Access the field as an int16. Returns the provided default on error. */
+ inline int16_t AsInt16(int16_t Default = 0) { return AsInteger<int16_t>(Default); }
+ /** Access the field as an int32. Returns the provided default on error. */
+ inline int32_t AsInt32(int32_t Default = 0) { return AsInteger<int32_t>(Default); }
+ /** Access the field as an int64. Returns the provided default on error. */
+ inline int64_t AsInt64(int64_t Default = 0) { return AsInteger<int64_t>(Default); }
+ /** Access the field as a uint8. Returns the provided default on error. */
+ inline uint8_t AsUInt8(uint8_t Default = 0) { return AsInteger<uint8_t>(Default); }
+ /** Access the field as a uint16. Returns the provided default on error. */
+ inline uint16_t AsUInt16(uint16_t Default = 0) { return AsInteger<uint16_t>(Default); }
+ /** Access the field as a uint32. Returns the provided default on error. */
+ inline uint32_t AsUInt32(uint32_t Default = 0) { return AsInteger<uint32_t>(Default); }
+ /** Access the field as a uint64. Returns the provided default on error. */
+ inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); }
+
+ /** Access the field as a float. Returns the provided default on error. */
+ ZENCORE_API float AsFloat(float Default = 0.0f);
+ /** Access the field as a double. Returns the provided default on error. */
+ ZENCORE_API double AsDouble(double Default = 0.0);
+
+ /** Access the field as a bool. Returns the provided default on error. */
+ ZENCORE_API bool AsBool(bool bDefault = false);
+
+ /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsCompactBinaryAttachment(const IoHash& Default = IoHash());
+ /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash());
+ /** Access the field as a hash referencing an attachment. Returns the provided default on error. */
+ ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash());
+
+ /** Access the field as a hash. Returns the provided default on error. */
+ ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash());
+
+ /** Access the field as a UUID. Returns a nil UUID on error. */
+ ZENCORE_API Guid AsUuid();
+ /** Access the field as a UUID. Returns the provided default on error. */
+ ZENCORE_API Guid AsUuid(const Guid& Default);
+
+ /** Access the field as an OID. Returns a nil OID on error. */
+ ZENCORE_API Oid AsObjectId();
+ /** Access the field as a OID. Returns the provided default on error. */
+ ZENCORE_API Oid AsObjectId(const Oid& Default);
+
+ /** Access the field as a date/time tick count. Returns the provided default on error. */
+ ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0);
+
+ /** Access the field as a date/time. Returns a date/time at the epoch on error. */
+ ZENCORE_API DateTime AsDateTime();
+ /** Access the field as a date/time. Returns the provided default on error. */
+ ZENCORE_API DateTime AsDateTime(DateTime Default);
+
+ /** Access the field as a timespan tick count. Returns the provided default on error. */
+ ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0);
+
+ /** Access the field as a timespan. Returns an empty timespan on error. */
+ ZENCORE_API TimeSpan AsTimeSpan();
+ /** Access the field as a timespan. Returns the provided default on error. */
+ ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default);
+
+ /** True if the field has a name. */
+ constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); }
+
+ constexpr inline bool IsNull() const { return CbFieldTypeOps::IsNull(Type); }
+
+ constexpr inline bool IsObject() const { return CbFieldTypeOps::IsObject(Type); }
+ constexpr inline bool IsArray() const { return CbFieldTypeOps::IsArray(Type); }
+
+ constexpr inline bool IsBinary() const { return CbFieldTypeOps::IsBinary(Type); }
+ constexpr inline bool IsString() const { return CbFieldTypeOps::IsString(Type); }
+
+ /** Whether the field is an integer of unspecified range and sign. */
+ constexpr inline bool IsInteger() const { return CbFieldTypeOps::IsInteger(Type); }
+ /** Whether the field is a float, or integer that supports implicit conversion. */
+ constexpr inline bool IsFloat() const { return CbFieldTypeOps::IsFloat(Type); }
+ constexpr inline bool IsBool() const { return CbFieldTypeOps::IsBool(Type); }
+
+ constexpr inline bool IsCompactBinaryAttachment() const { return CbFieldTypeOps::IsCompactBinaryAttachment(Type); }
+ constexpr inline bool IsBinaryAttachment() const { return CbFieldTypeOps::IsBinaryAttachment(Type); }
+ constexpr inline bool IsAttachment() const { return CbFieldTypeOps::IsAttachment(Type); }
+
+ constexpr inline bool IsHash() const { return CbFieldTypeOps::IsHash(Type); }
+ constexpr inline bool IsUuid() const { return CbFieldTypeOps::IsUuid(Type); }
+ constexpr inline bool IsObjectId() const { return CbFieldTypeOps::IsObjectId(Type); }
+
+ constexpr inline bool IsDateTime() const { return CbFieldTypeOps::IsDateTime(Type); }
+ constexpr inline bool IsTimeSpan() const { return CbFieldTypeOps::IsTimeSpan(Type); }
+
+ /** Whether the field has a value. */
+ constexpr inline explicit operator bool() const { return HasValue(); }
+
+ /**
+ * Whether the field has a value.
+ *
+ * All fields in a valid object or array have a value. A field with no value is returned when
+ * finding a field by name fails or when accessing an iterator past the end.
+ */
+ constexpr inline bool HasValue() const { return !CbFieldTypeOps::IsNone(Type); };
+
+ /** Whether the last field access encountered an error. */
+ constexpr inline bool HasError() const { return Error != CbFieldError::None; }
+
+ /** The type of error that occurred on the last field access, or None. */
+ constexpr inline CbFieldError GetError() const { return Error; }
+
+ /** Returns the size of the field in bytes, including the type and name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the field, including the type and name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& HashStream) const;
+
+ /** Feed the field (including type and name) to the stream function */
+ inline void WriteToStream(auto Hash) const
+ {
+ const CbFieldType SerializedType = CbFieldTypeOps::GetSerializedType(Type);
+ Hash(&SerializedType, sizeof(SerializedType));
+ auto View = GetViewNoType();
+ Hash(View.GetData(), View.GetSize());
+ }
+
+ /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the field into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ /**
+ * Whether this field is identical to the other field.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be performed with ValidateCompactBinary, except for field order and field name case.
+ */
+ ZENCORE_API bool Equals(const CbFieldView& Other) const;
+
+ /** Returns a view of the field, including the type and name when present. */
+ ZENCORE_API MemoryView GetView() const;
+
+ /**
+ * Try to get a view of the field as it would be serialized, such as by CopyTo.
+ *
+ * A serialized view is not available if the field has an externally-provided type.
+ * Access the serialized form of such fields using CopyTo or FCbFieldRef::Clone.
+ */
+ inline bool TryGetSerializedView(MemoryView& OutView) const
+ {
+ if (CbFieldTypeOps::HasFieldType(Type))
+ {
+ OutView = GetView();
+ return true;
+ }
+ return false;
+ }
+
+protected:
+ /** Returns a view of the name and value payload, which excludes the type. */
+ ZENCORE_API MemoryView GetViewNoType() const;
+
+ /** Returns a view of the value payload, which excludes the type and name. */
+ inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); }
+
+ /** Returns the type of the field including flags. */
+ constexpr inline CbFieldType GetType() const { return Type; }
+
+ /** Returns the start of the value payload. */
+ constexpr inline const void* GetPayload() const { return Payload; }
+
+ /** Returns the end of the value payload. */
+ inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); }
+
+ /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */
+ ZENCORE_API uint64_t GetPayloadSize() const;
+
+ /** Assign a field from a pointer to its data and an optional externally-provided type. */
+ inline void Assign(const void* InData, const CbFieldType InType)
+ {
+ static_assert(std::is_trivially_destructible<CbFieldView>::value,
+ "This optimization requires CbField to be trivially destructible!");
+ new (this) CbFieldView(InData, InType);
+ }
+
+private:
+ /** Parameters for converting to an integer. */
+ struct IntegerParams
+ {
+ /** Whether the output type has a sign bit. */
+ uint32_t IsSigned : 1;
+ /** Bits of magnitude. (7 for int8) */
+ uint32_t MagnitudeBits : 31;
+ };
+
+ /** Make integer params for the given integer type. */
+ template<typename IntType>
+ static constexpr inline IntegerParams MakeIntegerParams()
+ {
+ IntegerParams Params;
+ Params.IsSigned = IntType(-1) < IntType(0);
+ Params.MagnitudeBits = 8 * sizeof(IntType) - Params.IsSigned;
+ return Params;
+ }
+
+ /**
+ * Access the field as the given integer type.
+ *
+ * Returns the provided default if the value cannot be represented in the output type.
+ */
+ template<typename IntType>
+ inline IntType AsInteger(IntType Default)
+ {
+ return IntType(AsInteger(uint64_t(Default), MakeIntegerParams<IntType>()));
+ }
+
+ ZENCORE_API uint64_t AsInteger(uint64_t Default, IntegerParams Params);
+
+ /** The field type, with the transient HasFieldType flag if the field contains its type. */
+ CbFieldType Type = CbFieldType::None;
+ /** The error (if any) that occurred on the last field access. */
+ CbFieldError Error = CbFieldError::None;
+ /** The number of bytes for the name stored before the payload. */
+ uint32_t NameLen = 0;
+ /** The value payload, which also points to the end of the name. */
+ const void* Payload = nullptr;
+};
+
+template<typename FieldType>
+class TCbFieldIterator : public FieldType
+{
+public:
+ /** Construct an empty field range. */
+ constexpr TCbFieldIterator() = default;
+
+ inline TCbFieldIterator& operator++()
+ {
+ const void* const PayloadEnd = FieldType::GetPayloadEnd();
+ const int64_t AtEndMask = int64_t(PayloadEnd == FieldsEnd) - 1;
+ const CbFieldType NextType = CbFieldType(int64_t(FieldType::GetType()) & AtEndMask);
+ const void* const NextField = reinterpret_cast<const void*>(int64_t(PayloadEnd) & AtEndMask);
+ const void* const NextFieldsEnd = reinterpret_cast<const void*>(int64_t(FieldsEnd) & AtEndMask);
+
+ FieldType::Assign(NextField, NextType);
+ FieldsEnd = NextFieldsEnd;
+ return *this;
+ }
+
+ inline TCbFieldIterator operator++(int)
+ {
+ TCbFieldIterator It(*this);
+ ++*this;
+ return It;
+ }
+
+ constexpr inline FieldType& operator*() { return *this; }
+ constexpr inline FieldType* operator->() { return this; }
+
+ /** Reset this to an empty field range. */
+ inline void Reset() { *this = TCbFieldIterator(); }
+
+ /** Returns the size of the fields in the range in bytes. */
+ ZENCORE_API uint64_t GetRangeSize() const;
+
+ /** Calculate the hash of every field in the range. */
+ ZENCORE_API IoHash GetRangeHash() const;
+ ZENCORE_API void GetRangeHash(IoHashStream& Hash) const;
+
+ using FieldType::Equals;
+
+ template<typename OtherFieldType>
+ constexpr inline bool Equals(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return FieldType::GetPayload() == Other.OtherFieldType::GetPayload() && FieldsEnd == Other.FieldsEnd;
+ }
+
+ template<typename OtherFieldType>
+ constexpr inline bool operator==(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return Equals(Other);
+ }
+
+ template<typename OtherFieldType>
+ constexpr inline bool operator!=(const TCbFieldIterator<OtherFieldType>& Other) const
+ {
+ return !Equals(Other);
+ }
+
+ /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */
+ ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const;
+
+ /** Invoke the visitor for every attachment in the field range. */
+ ZENCORE_API void IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const;
+
+ /** Create a view of every field in the range. */
+ inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); }
+
+ /**
+ * Try to get a view of every field in the range as they would be serialized.
+ *
+ * A serialized view is not available if the underlying fields have an externally-provided type.
+ * Access the serialized form of such ranges using CbFieldRefIterator::CloneRange.
+ */
+ inline bool TryGetSerializedRangeView(MemoryView& OutView) const
+ {
+ if (CbFieldTypeOps::HasFieldType(FieldType::GetType()))
+ {
+ OutView = GetRangeView();
+ return true;
+ }
+ return false;
+ }
+
+protected:
+ /** Construct a field range that contains exactly one field. */
+ constexpr inline explicit TCbFieldIterator(FieldType InField) : FieldType(std::move(InField)), FieldsEnd(FieldType::GetPayloadEnd()) {}
+
+ /**
+ * Construct a field range from the first field and a pointer to the end of the last field.
+ *
+ * @param InField The first field, or the default field if there are no fields.
+ * @param InFieldsEnd A pointer to the end of the payload of the last field, or null.
+ */
+ constexpr inline TCbFieldIterator(FieldType&& InField, const void* InFieldsEnd) : FieldType(std::move(InField)), FieldsEnd(InFieldsEnd)
+ {
+ }
+
+ /** Returns the end of the last field, or null for an iterator at the end. */
+ template<typename OtherFieldType>
+ static inline const void* GetFieldsEnd(const TCbFieldIterator<OtherFieldType>& It)
+ {
+ return It.FieldsEnd;
+ }
+
+private:
+ friend inline TCbFieldIterator begin(const TCbFieldIterator& Iterator) { return Iterator; }
+ friend inline TCbFieldIterator end(const TCbFieldIterator&) { return TCbFieldIterator(); }
+
+private:
+ template<typename OtherType>
+ friend class TCbFieldIterator;
+
+ friend class CbFieldViewIterator;
+
+ /** Pointer to the first byte past the end of the last field. Set to null at the end. */
+ const void* FieldsEnd = nullptr;
+};
+
+/**
+ * Iterator for CbField.
+ *
+ * @see CbFieldIterator
+ */
+class CbFieldViewIterator : public TCbFieldIterator<CbFieldView>
+{
+public:
+ constexpr CbFieldViewIterator() = default;
+
+ /** Construct a field range that contains exactly one field. */
+ static inline CbFieldViewIterator MakeSingle(const CbFieldView& Field) { return CbFieldViewIterator(Field); }
+
+ /**
+ * Construct a field range from a buffer containing zero or more valid fields.
+ *
+ * @param View A buffer containing zero or more valid fields.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ */
+ static inline CbFieldViewIterator MakeRange(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ return !View.IsEmpty() ? TCbFieldIterator(CbFieldView(View.GetData(), Type), View.GetDataEnd()) : CbFieldViewIterator();
+ }
+
+ /** Construct an iterator from another iterator. */
+ template<typename OtherFieldType>
+ inline CbFieldViewIterator(const TCbFieldIterator<OtherFieldType>& It)
+ : TCbFieldIterator(ImplicitConv<CbFieldView>(It), GetFieldsEnd(It))
+ {
+ }
+
+private:
+ using TCbFieldIterator::TCbFieldIterator;
+};
+
+/**
+ * Array of CbField that have no names.
+ *
+ * Accessing a field of the array requires iteration. Access by index is not provided because the
+ * cost of accessing an item by index scales linearly with the index.
+ *
+ * This type only provides a view into memory and does not perform any memory management itself.
+ * Use CbArrayRef to hold a reference to the underlying memory when necessary.
+ */
+class CbArrayView : protected CbFieldView
+{
+ friend class CbFieldView;
+
+public:
+ /** @see CbField::CbField */
+ using CbFieldView::CbFieldView;
+
+ /** Construct an array with no fields. */
+ ZENCORE_API CbArrayView();
+
+ /** Returns the number of items in the array. */
+ ZENCORE_API uint64_t Num() const;
+
+ /** Create an iterator for the fields of this array. */
+ ZENCORE_API CbFieldViewIterator CreateViewIterator() const;
+
+ /** Visit the fields of this array. */
+ ZENCORE_API void VisitFields(ICbVisitor& Visitor);
+
+ /** Access the array as an array field. */
+ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); }
+
+ /** Construct an array from an array field. No type check is performed! */
+ static inline CbArrayView FromFieldView(const CbFieldView& Field) { return CbArrayView(Field); }
+
+ /** Returns the size of the array in bytes if serialized by itself with no name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the array if serialized by itself with no name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& Stream) const;
+
+ /**
+ * Whether this array is identical to the other array.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be done with the All mode to check these assumptions about the format of the inputs.
+ */
+ ZENCORE_API bool Equals(const CbArrayView& Other) const;
+
+ /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the array into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ ///** Invoke the visitor for every attachment in the array. */
+ inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+ {
+ CreateViewIterator().IterateRangeAttachments(Visitor);
+ }
+
+ /** Returns a view of the array, including the type and name when present. */
+ using CbFieldView::GetView;
+
+private:
+ friend inline CbFieldViewIterator begin(const CbArrayView& Array) { return Array.CreateViewIterator(); }
+ friend inline CbFieldViewIterator end(const CbArrayView&) { return CbFieldViewIterator(); }
+
+ /** Construct an array from an array field. No type check is performed! Use via FromField. */
+ inline explicit CbArrayView(const CbFieldView& Field) : CbFieldView(Field) {}
+};
+
+class CbObjectView : protected CbFieldView
+{
+ friend class CbFieldView;
+
+public:
+ /** @see CbField::CbField */
+ using CbFieldView::CbFieldView;
+
+ /** Construct an object with no fields. */
+ ZENCORE_API CbObjectView();
+
+ /** Create an iterator for the fields of this object. */
+ ZENCORE_API CbFieldViewIterator CreateViewIterator() const;
+
+ /** Visit the fields of this object. */
+ ZENCORE_API void VisitFields(ICbVisitor& Visitor);
+
+ /**
+ * Find a field by case-sensitive name comparison.
+ *
+ * The cost of this operation scales linearly with the number of fields in the object. Prefer
+ * to iterate over the fields only once when consuming an object.
+ *
+ * @param Name The name of the field.
+ * @return The matching field if found, otherwise a field with no value.
+ */
+ ZENCORE_API CbFieldView FindView(std::string_view Name) const;
+
+ /** Find a field by case-insensitive name comparison. */
+ ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const;
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); }
+
+ /** Access the object as an object field. */
+ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); }
+
+ /** Construct an object from an object field. No type check is performed! */
+ static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); }
+
+ /** Returns the size of the object in bytes if serialized by itself with no name. */
+ ZENCORE_API uint64_t GetSize() const;
+
+ /** Calculate the hash of the object if serialized by itself with no name. */
+ ZENCORE_API IoHash GetHash() const;
+
+ ZENCORE_API void GetHash(IoHashStream& HashStream) const;
+
+ /**
+ * Whether this object is identical to the other object.
+ *
+ * Performs a deep comparison of any contained arrays or objects and their fields. Comparison
+ * assumes that both fields are valid and are written in the canonical format. Fields must be
+ * written in the same order in arrays and objects, and name comparison is case sensitive. If
+ * these assumptions do not hold, this may return false for equivalent inputs. Validation can
+ * be done with the All mode to check these assumptions about the format of the inputs.
+ */
+ ZENCORE_API bool Equals(const CbObjectView& Other) const;
+
+ /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */
+ ZENCORE_API void CopyTo(MutableMemoryView Buffer) const;
+
+ /** Copy the field into an archive, including its type and name. */
+ ZENCORE_API void CopyTo(BinaryWriter& Ar) const;
+
+ ///** Invoke the visitor for every attachment in the object. */
+ inline void IterateAttachments(std::function<void(CbFieldView)> Visitor) const
+ {
+ CreateViewIterator().IterateRangeAttachments(Visitor);
+ }
+
+ /** Returns a view of the object, including the type and name when present. */
+ using CbFieldView::GetView;
+
+ /** Whether the field has a value. */
+ using CbFieldView::operator bool;
+
+private:
+ friend inline CbFieldViewIterator begin(const CbObjectView& Object) { return Object.CreateViewIterator(); }
+ friend inline CbFieldViewIterator end(const CbObjectView&) { return CbFieldViewIterator(); }
+
+ /** Construct an object from an object field. No type check is performed! Use via FromField. */
+ inline explicit CbObjectView(const CbFieldView& Field) : CbFieldView(Field) {}
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+/** A reference to a function that is used to allocate buffers for compact binary data. */
+using BufferAllocator = std::function<UniqueBuffer(uint64_t Size)>;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** A wrapper that holds a reference to the buffer that contains its compact binary value. */
+template<typename BaseType>
+class CbBuffer : public BaseType
+{
+public:
+ /** Construct a default value. */
+ CbBuffer() = default;
+
+ /**
+ * Construct a value from a pointer to its data and an optional externally-provided type.
+ *
+ * @param ValueBuffer A buffer that exactly contains the value.
+ * @param Type HasFieldType means that ValueBuffer contains the type. Otherwise, use the given type.
+ */
+ inline explicit CbBuffer(SharedBuffer ValueBuffer, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ if (ValueBuffer)
+ {
+ BaseType::operator=(BaseType(ValueBuffer.GetData(), Type));
+ ZEN_ASSERT(ValueBuffer.GetView().Contains(BaseType::GetView()));
+ Buffer = std::move(ValueBuffer);
+ }
+ }
+
+ /** Construct a value that holds a reference to the buffer that contains it. */
+ inline CbBuffer(const BaseType& Value, SharedBuffer OuterBuffer) : BaseType(Value)
+ {
+ if (OuterBuffer)
+ {
+ ZEN_ASSERT(OuterBuffer.GetView().Contains(BaseType::GetView()));
+ Buffer = std::move(OuterBuffer);
+ }
+ }
+
+ /** Construct a value that holds a reference to the buffer of the outer that contains it. */
+ template<typename OtherBaseType>
+ inline CbBuffer(const BaseType& Value, CbBuffer<OtherBaseType> OuterRef) : CbBuffer(Value, std::move(OuterRef.Buffer))
+ {
+ }
+
+ /** Reset this to a default value and null buffer. */
+ inline void Reset() { *this = CbBuffer(); }
+
+ /** Whether this reference has ownership of the memory in its buffer. */
+ inline bool IsOwned() const { return Buffer && Buffer.IsOwned(); }
+
+ /** Clone the value, if necessary, to a buffer that this reference has ownership of. */
+ inline void MakeOwned()
+ {
+ if (!IsOwned())
+ {
+ UniqueBuffer MutableBuffer = UniqueBuffer::Alloc(BaseType::GetSize());
+ BaseType::CopyTo(MutableBuffer);
+ BaseType::operator=(BaseType(MutableBuffer.GetData()));
+ Buffer = std::move(MutableBuffer);
+ }
+ }
+
+ /** Returns a buffer that exactly contains this value. */
+ inline SharedBuffer GetBuffer() const
+ {
+ const MemoryView View = BaseType::GetView();
+ const SharedBuffer& OuterBuffer = GetOuterBuffer();
+ return View == OuterBuffer.GetView() ? OuterBuffer : SharedBuffer::MakeView(View, OuterBuffer);
+ }
+
+ /** Returns the outer buffer (if any) that contains this value. */
+ inline const SharedBuffer& GetOuterBuffer() const& { return Buffer; }
+ inline SharedBuffer GetOuterBuffer() && { return std::move(Buffer); }
+
+private:
+ template<typename OtherType>
+ friend class CbBuffer;
+
+ SharedBuffer Buffer;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Factory functions for types derived from CbBuffer.
+ *
+ * This uses the curiously recurring template pattern to construct the correct type of reference.
+ * The derived type inherits from CbBufferRef and this type to expose the factory functions.
+ */
+template<typename RefType, typename BaseType>
+class CbBufferFactory
+{
+public:
+ /** Construct a value from an owned clone of its memory. */
+ static inline RefType Clone(const void* const Data) { return Clone(BaseType(Data)); }
+
+ /** Construct a value from an owned clone of its memory. */
+ static inline RefType Clone(const BaseType& Value)
+ {
+ RefType Ref = MakeView(Value);
+ Ref.MakeOwned();
+ return Ref;
+ }
+
+ /** Construct a value from a read-only view of its memory and its optional outer buffer. */
+ static inline RefType MakeView(const void* const Data, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return MakeView(BaseType(Data), std::move(OuterBuffer));
+ }
+
+ /** Construct a value from a read-only view of its memory and its optional outer buffer. */
+ static inline RefType MakeView(const BaseType& Value, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return RefType(Value, std::move(OuterBuffer));
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CbArray;
+class CbObject;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A field that can hold a reference to the memory that contains it.
+ *
+ * @see CbBufferRef
+ */
+class CbField : public CbBuffer<CbFieldView>, public CbBufferFactory<CbField, CbFieldView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Access the field as an object. Defaults to an empty object on error. */
+ inline CbObject AsObject() &;
+
+ /** Access the field as an object. Defaults to an empty object on error. */
+ inline CbObject AsObject() &&;
+
+ /** Access the field as an array. Defaults to an empty array on error. */
+ inline CbArray AsArray() &;
+
+ /** Access the field as an array. Defaults to an empty array on error. */
+ inline CbArray AsArray() &&;
+};
+
+/**
+ * Iterator for CbFieldRef.
+ *
+ * @see CbFieldIterator
+ */
+class CbFieldIterator : public TCbFieldIterator<CbField>
+{
+public:
+ /** Construct a field range from an owned clone of a range. */
+ ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It);
+
+ /** Construct a field range from an owned clone of a range. */
+ static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); }
+
+ /** Construct a field range that contains exactly one field. */
+ static inline CbFieldIterator MakeSingle(CbField Field) { return CbFieldIterator(std::move(Field)); }
+
+ /**
+ * Construct a field range from a buffer containing zero or more valid fields.
+ *
+ * @param Buffer A buffer containing zero or more valid fields.
+ * @param Type HasFieldType means that Buffer contains the type. Otherwise, use the given type.
+ */
+ static inline CbFieldIterator MakeRange(SharedBuffer Buffer, CbFieldType Type = CbFieldType::HasFieldType)
+ {
+ if (Buffer.GetSize())
+ {
+ const void* const DataEnd = Buffer.GetView().GetDataEnd();
+ return CbFieldIterator(CbField(std::move(Buffer), Type), DataEnd);
+ }
+ return CbFieldIterator();
+ }
+
+ /** Construct a field range from an iterator and its optional outer buffer. */
+ static inline CbFieldIterator MakeRangeView(const CbFieldViewIterator& It, SharedBuffer OuterBuffer = SharedBuffer())
+ {
+ return CbFieldIterator(CbField(It, std::move(OuterBuffer)), GetFieldsEnd(It));
+ }
+
+ /** Construct an empty field range. */
+ constexpr CbFieldIterator() = default;
+
+ /** Clone the range, if necessary, to a buffer that this reference has ownership of. */
+ inline void MakeRangeOwned()
+ {
+ if (!IsOwned())
+ {
+ *this = CloneRange(*this);
+ }
+ }
+
+ /** Returns a buffer that exactly contains the field range. */
+ ZENCORE_API SharedBuffer GetRangeBuffer() const;
+
+private:
+ using TCbFieldIterator::TCbFieldIterator;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An array that can hold a reference to the memory that contains it.
+ *
+ * @see CbBuffer
+ */
+class CbArray : public CbBuffer<CbArrayView>, public CbBufferFactory<CbArray, CbArrayView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Create an iterator for the fields of this array. */
+ inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); }
+
+ /** Access the array as an array field. */
+ inline CbField AsField() const& { return CbField(CbArrayView::AsFieldView(), *this); }
+
+ /** Access the array as an array field. */
+ inline CbField AsField() && { return CbField(CbArrayView::AsFieldView(), std::move(*this)); }
+
+private:
+ friend inline CbFieldIterator begin(const CbArray& Array) { return Array.CreateIterator(); }
+ friend inline CbFieldIterator end(const CbArray&) { return CbFieldIterator(); }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An object that can hold a reference to the memory that contains it.
+ *
+ * @see CbBuffer
+ */
+class CbObject : public CbBuffer<CbObjectView>, public CbBufferFactory<CbObject, CbObjectView>
+{
+public:
+ using CbBuffer::CbBuffer;
+
+ /** Create an iterator for the fields of this object. */
+ inline CbFieldIterator CreateIterator() const { return CbFieldIterator::MakeRangeView(CreateViewIterator(), GetOuterBuffer()); }
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbField Find(std::string_view Name) const
+ {
+ if (CbFieldView Field = FindView(Name))
+ {
+ return CbField(Field, *this);
+ }
+ return CbField();
+ }
+
+ /** Find a field by case-insensitive name comparison. */
+ inline CbField FindIgnoreCase(std::string_view Name) const
+ {
+ if (CbFieldView Field = FindIgnoreCase(Name))
+ {
+ return CbField(Field, *this);
+ }
+ return CbField();
+ }
+
+ /** Find a field by case-sensitive name comparison. */
+ inline CbFieldView operator[](std::string_view Name) const { return Find(Name); }
+
+ /** Access the object as an object field. */
+ inline CbField AsField() const& { return CbField(CbObjectView::AsFieldView(), *this); }
+
+ /** Access the object as an object field. */
+ inline CbField AsField() && { return CbField(CbObjectView::AsFieldView(), std::move(*this)); }
+
+private:
+ friend inline CbFieldIterator begin(const CbObject& Object) { return Object.CreateIterator(); }
+ friend inline CbFieldIterator end(const CbObject&) { return CbFieldIterator(); }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+inline CbObject
+CbField::AsObject() &
+{
+ return IsObject() ? CbObject(AsObjectView(), *this) : CbObject();
+}
+
+inline CbObject
+CbField::AsObject() &&
+{
+ return IsObject() ? CbObject(AsObjectView(), std::move(*this)) : CbObject();
+}
+
+inline CbArray
+CbField::AsArray() &
+{
+ return IsArray() ? CbArray(AsArrayView(), *this) : CbArray();
+}
+
+inline CbArray
+CbField::AsArray() &&
+{
+ return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray();
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator);
+
+inline CbObject
+LoadCompactBinaryObject(IoBuffer Payload)
+{
+ return CbObject{SharedBuffer::MakeView(Payload.Data(), Payload.Size())};
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Determine the size in bytes of the compact binary field at the start of the view.
+ *
+ * This may be called on an incomplete or invalid field, in which case the returned size is zero.
+ * A size can always be extracted from a valid field with no name if a view of at least the first
+ * 10 bytes is provided, regardless of field size. For fields with names, the size of view needed
+ * to calculate a size is at most 10 + MaxNameLen + MeasureVarUInt(MaxNameLen).
+ *
+ * This function can be used when streaming a field, for example, to determine the size of buffer
+ * to fill before attempting to construct a field from it.
+ *
+ * @param View A memory view that may contain the start of a field.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ */
+ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType);
+
+/**
+ * Try to determine the type and size of the compact binary field at the start of the view.
+ *
+ * This may be called on an incomplete or invalid field, in which case it will return false, with
+ * OutSize being 0 for invalid fields, otherwise the minimum view size necessary to make progress
+ * in measuring the field on the next call to this function.
+ *
+ * @note A return of true from this function does not indicate that the entire field is valid.
+ *
+ * @param InView A memory view that may contain the start of a field.
+ * @param OutType The type (with flags) of the field. None is written until a value is available.
+ * @param OutSize The total field size for a return of true, 0 for invalid fields, or the size to
+ * make progress in measuring the field on the next call to this function.
+ * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type.
+ * @return true if the size of the field was determined, otherwise false.
+ */
+ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView,
+ CbFieldType& OutType,
+ uint64_t& OutSize,
+ CbFieldType InType = CbFieldType::HasFieldType);
+
+inline CbFieldViewIterator
+begin(CbFieldView& View)
+{
+ if (View.IsArray())
+ {
+ return View.AsArrayView().CreateViewIterator();
+ }
+ else if (View.IsObject())
+ {
+ return View.AsObjectView().CreateViewIterator();
+ }
+
+ return CbFieldViewIterator();
+}
+
+inline CbFieldViewIterator
+end(CbFieldView&)
+{
+ return CbFieldViewIterator();
+}
+
+void uson_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h
new file mode 100644
index 000000000..83d4309f7
--- /dev/null
+++ b/zencore/include/zencore/compactbinarybuilder.h
@@ -0,0 +1,633 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class CbAttachment;
+class BinaryWriter;
+
+/**
+ * A writer for compact binary object, arrays, and fields.
+ *
+ * The writer produces a sequence of fields that can be saved to a provided memory buffer or into
+ * a new owned buffer. The typical use case is to write a single object, which can be accessed by
+ * calling Save().AsObjectRef() or Save(Buffer).AsObject().
+ *
+ * The writer will assert on most incorrect usage and will always produce valid compact binary if
+ * provided with valid input. The writer does not check for invalid UTF-8 string encoding, object
+ * fields with duplicate names, or invalid compact binary being copied from another source.
+ *
+ * It is most convenient to use the streaming API for the writer, as demonstrated in the example.
+ *
+ * When writing a small amount of compact binary data, TCbWriter can be more efficient as it uses
+ * a fixed-size stack buffer for storage before spilling onto the heap.
+ *
+ * @see TCbWriter
+ *
+ * Example:
+ *
+ * CbObjectRef WriteObject()
+ * {
+ * CbWriter<256> Writer;
+ * Writer.BeginObject();
+ *
+ * Writer << "Resize" << true;
+ * Writer << "MaxWidth" << 1024;
+ * Writer << "MaxHeight" << 1024;
+ *
+ * Writer.BeginArray();
+ * Writer << "FormatA" << "FormatB" << "FormatC";
+ * Writer.EndArray();
+ *
+ * Writer.EndObject();
+ * return Writer.Save().AsObjectRef();
+ * }
+ */
+class CbWriter
+{
+public:
+ ZENCORE_API CbWriter();
+ ZENCORE_API ~CbWriter();
+
+ CbWriter(const CbWriter&) = delete;
+ CbWriter& operator=(const CbWriter&) = delete;
+
+ /** Empty the writer without releasing any allocated memory. */
+ ZENCORE_API void Reset();
+
+ /**
+ * Serialize the field(s) to an owned buffer and return it as an iterator.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ * The writer remains valid for further use when this function returns.
+ */
+ ZENCORE_API CbFieldIterator Save();
+
+ /**
+ * Serialize the field(s) to memory.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ * The writer remains valid for further use when this function returns.
+ *
+ * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes.
+ * @return An iterator for the field(s) written to the buffer.
+ */
+ ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer);
+
+ ZENCORE_API void Save(BinaryWriter& Writer);
+
+ /**
+ * The size of buffer (in bytes) required to serialize the fields that have been written.
+ *
+ * It is not valid to call this function in the middle of writing an object, array, or field.
+ */
+ ZENCORE_API uint64_t GetSaveSize() const;
+
+ /**
+ * Sets the name of the next field to be written.
+ *
+ * It is not valid to call this function when writing a field inside an array.
+ * Names must be valid UTF-8 and must be unique within an object.
+ */
+ ZENCORE_API CbWriter& SetName(std::string_view Name);
+
+ /** Copy the value (not the name) of an existing field. */
+ inline void AddField(std::string_view Name, const CbFieldView& Value)
+ {
+ SetName(Name);
+ AddField(Value);
+ }
+
+ ZENCORE_API void AddField(const CbFieldView& Value);
+
+ /** Copy the value (not the name) of an existing field. Holds a reference if owned. */
+ inline void AddField(std::string_view Name, const CbField& Value)
+ {
+ SetName(Name);
+ AddField(Value);
+ }
+ ZENCORE_API void AddField(const CbField& Value);
+
+ /** Begin a new object. Must have a matching call to EndObject. */
+ inline void BeginObject(std::string_view Name)
+ {
+ SetName(Name);
+ BeginObject();
+ }
+ ZENCORE_API void BeginObject();
+ /** End an object after its fields have been written. */
+ ZENCORE_API void EndObject();
+
+ /** Copy the value (not the name) of an existing object. */
+ inline void AddObject(std::string_view Name, const CbObjectView& Value)
+ {
+ SetName(Name);
+ AddObject(Value);
+ }
+ ZENCORE_API void AddObject(const CbObjectView& Value);
+ /** Copy the value (not the name) of an existing object. Holds a reference if owned. */
+ inline void AddObject(std::string_view Name, const CbObject& Value)
+ {
+ SetName(Name);
+ AddObject(Value);
+ }
+ ZENCORE_API void AddObject(const CbObject& Value);
+
+ /** Begin a new array. Must have a matching call to EndArray. */
+ inline void BeginArray(std::string_view Name)
+ {
+ SetName(Name);
+ BeginArray();
+ }
+ ZENCORE_API void BeginArray();
+ /** End an array after its fields have been written. */
+ ZENCORE_API void EndArray();
+
+ /** Copy the value (not the name) of an existing array. */
+ inline void AddArray(std::string_view Name, const CbArrayView& Value)
+ {
+ SetName(Name);
+ AddArray(Value);
+ }
+ ZENCORE_API void AddArray(const CbArrayView& Value);
+ /** Copy the value (not the name) of an existing array. Holds a reference if owned. */
+ inline void AddArray(std::string_view Name, const CbArray& Value)
+ {
+ SetName(Name);
+ AddArray(Value);
+ }
+ ZENCORE_API void AddArray(const CbArray& Value);
+
+ /** Write a null field. */
+ inline void AddNull(std::string_view Name)
+ {
+ SetName(Name);
+ AddNull();
+ }
+ ZENCORE_API void AddNull();
+
+ /** Write a binary field by copying Size bytes from Value. */
+ inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size)
+ {
+ SetName(Name);
+ AddBinary(Value, Size);
+ }
+ ZENCORE_API void AddBinary(const void* Value, uint64_t Size);
+ /** Write a binary field by copying the view. */
+ inline void AddBinary(std::string_view Name, MemoryView Value)
+ {
+ SetName(Name);
+ AddBinary(Value);
+ }
+ inline void AddBinary(MemoryView Value) { AddBinary(Value.GetData(), Value.GetSize()); }
+
+ /** Write a binary field by copying the buffer. Holds a reference if owned. */
+ inline void AddBinary(std::string_view Name, IoBuffer Value)
+ {
+ SetName(Name);
+ AddBinary(std::move(Value));
+ }
+ ZENCORE_API void AddBinary(IoBuffer Value);
+ ZENCORE_API void AddBinary(SharedBuffer Value);
+
+ /** Write a string field by copying the UTF-8 value. */
+ inline void AddString(std::string_view Name, std::string_view Value)
+ {
+ SetName(Name);
+ AddString(Value);
+ }
+ ZENCORE_API void AddString(std::string_view Value);
+ /** Write a string field by converting the UTF-16 value to UTF-8. */
+ inline void AddString(std::string_view Name, std::wstring_view Value)
+ {
+ SetName(Name);
+ AddString(Value);
+ }
+ ZENCORE_API void AddString(std::wstring_view Value);
+
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, int32_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(int32_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, int64_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(int64_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, uint32_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(uint32_t Value);
+ /** Write an integer field. */
+ inline void AddInteger(std::string_view Name, uint64_t Value)
+ {
+ SetName(Name);
+ AddInteger(Value);
+ }
+ ZENCORE_API void AddInteger(uint64_t Value);
+
+ /** Write a float field from a 32-bit float value. */
+ inline void AddFloat(std::string_view Name, float Value)
+ {
+ SetName(Name);
+ AddFloat(Value);
+ }
+ ZENCORE_API void AddFloat(float Value);
+
+ /** Write a float field from a 64-bit float value. */
+ inline void AddFloat(std::string_view Name, double Value)
+ {
+ SetName(Name);
+ AddFloat(Value);
+ }
+ ZENCORE_API void AddFloat(double Value);
+
+ /** Write a bool field. */
+ inline void AddBool(std::string_view Name, bool bValue)
+ {
+ SetName(Name);
+ AddBool(bValue);
+ }
+ ZENCORE_API void AddBool(bool bValue);
+
+ /** Write a field referencing a compact binary attachment by its hash. */
+ inline void AddCompactBinaryAttachment(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddCompactBinaryAttachment(Value);
+ }
+ ZENCORE_API void AddCompactBinaryAttachment(const IoHash& Value);
+
+ /** Write a field referencing a binary attachment by its hash. */
+ inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddBinaryAttachment(Value);
+ }
+ ZENCORE_API void AddBinaryAttachment(const IoHash& Value);
+
+ /** Write a field referencing the attachment by its hash. */
+ inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment)
+ {
+ SetName(Name);
+ AddAttachment(Attachment);
+ }
+ ZENCORE_API void AddAttachment(const CbAttachment& Attachment);
+
+ /** Write a hash field. */
+ inline void AddHash(std::string_view Name, const IoHash& Value)
+ {
+ SetName(Name);
+ AddHash(Value);
+ }
+ ZENCORE_API void AddHash(const IoHash& Value);
+
+ /** Write a UUID field. */
+ inline void AddUuid(std::string_view Name, const Guid& Value)
+ {
+ SetName(Name);
+ AddUuid(Value);
+ }
+ ZENCORE_API void AddUuid(const Guid& Value);
+
+ /** Write an ObjectId field. */
+ inline void AddObjectId(std::string_view Name, const Oid& Value)
+ {
+ SetName(Name);
+ AddObjectId(Value);
+ }
+ ZENCORE_API void AddObjectId(const Oid& Value);
+
+ /** Write a date/time field with the specified count of 100ns ticks since the epoch. */
+ inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks)
+ {
+ SetName(Name);
+ AddDateTimeTicks(Ticks);
+ }
+ ZENCORE_API void AddDateTimeTicks(int64_t Ticks);
+
+ /** Write a date/time field. */
+ inline void AddDateTime(std::string_view Name, DateTime Value)
+ {
+ SetName(Name);
+ AddDateTime(Value);
+ }
+ ZENCORE_API void AddDateTime(DateTime Value);
+
+ /** Write a time span field with the specified count of 100ns ticks. */
+ inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks)
+ {
+ SetName(Name);
+ AddTimeSpanTicks(Ticks);
+ }
+ ZENCORE_API void AddTimeSpanTicks(int64_t Ticks);
+
+ /** Write a time span field. */
+ inline void AddTimeSpan(std::string_view Name, TimeSpan Value)
+ {
+ SetName(Name);
+ AddTimeSpan(Value);
+ }
+ ZENCORE_API void AddTimeSpan(TimeSpan Value);
+
+ /** Private flags that are public to work with ENUM_CLASS_FLAGS. */
+ enum class StateFlags : uint8_t;
+
+protected:
+ /** Reserve the specified size up front until the format is optimized. */
+ ZENCORE_API explicit CbWriter(int64_t InitialSize);
+
+private:
+ friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue);
+
+ /** Begin writing a field. May be called twice for named fields. */
+ void BeginField();
+
+ /** Finish writing a field by writing its type. */
+ void EndField(CbFieldType Type);
+
+ /** Set the field name if valid in this state, otherwise write add a string field. */
+ ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue);
+
+ /** Returns a view of the name of the active field, if any, otherwise the empty view. */
+ std::string_view GetActiveName() const;
+
+ /** Remove field types after the first to make the sequence uniform. */
+ void MakeFieldsUniform(int64_t FieldBeginOffset, int64_t FieldEndOffset);
+
+ /** State of the object, array, or top-level field being written. */
+ struct WriterState
+ {
+ StateFlags Flags{};
+ /** The type of the fields in the sequence if uniform, otherwise None. */
+ CbFieldType UniformType{};
+ /** The offset of the start of the current field. */
+ int64_t Offset{};
+ /** The number of fields written in this state. */
+ uint64_t Count{};
+ };
+
+private:
+ // This is a prototype-quality format for the writer. Using an array of bytes is inefficient,
+ // and will lead to many unnecessary copies and moves of the data to resize the array, insert
+ // object and array sizes, and remove field types for uniform objects and uniform arrays. The
+ // optimized format will be a list of power-of-two blocks and an optional first block that is
+ // provided externally, such as on the stack. That format will store the offsets that require
+ // object or array sizes to be inserted and field types to be removed, and will perform those
+ // operations only when saving to a buffer.
+ std::vector<uint8_t> Data;
+ std::vector<WriterState> States;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A writer for compact binary object, arrays, and fields that uses a fixed-size stack buffer.
+ *
+ * @see CbWriter
+ */
+template<uint32_t InlineBufferSize>
+class FixedCbWriter : public CbWriter
+{
+public:
+ inline FixedCbWriter() : CbWriter(InlineBufferSize) {}
+
+ FixedCbWriter(const FixedCbWriter&) = delete;
+ FixedCbWriter& operator=(const FixedCbWriter&) = delete;
+
+private:
+ // Reserve the inline buffer now even though we are unable to use it. This will avoid causing
+ // new stack overflows when this functionality is properly implemented in the future.
+ uint8_t Buffer[InlineBufferSize];
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CbObjectWriter : public CbWriter
+{
+public:
+ CbObjectWriter() { BeginObject(); }
+
+ ZENCORE_API CbObject Save()
+ {
+ Finalize();
+ return CbWriter::Save().AsObject();
+ }
+
+ ZENCORE_API void Save(BinaryWriter& Writer)
+ {
+ Finalize();
+ return CbWriter::Save(Writer);
+ }
+
+ uint64_t GetSaveSize() = delete;
+
+ void Finalize()
+ {
+ if (m_Finalized == false)
+ {
+ EndObject();
+ m_Finalized = true;
+ }
+ }
+
+ CbObjectWriter(const CbWriter&) = delete;
+ CbObjectWriter& operator=(const CbWriter&) = delete;
+
+private:
+ bool m_Finalized = false;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Write the field name if valid in this state, otherwise write the string value. */
+inline CbWriter&
+operator<<(CbWriter& Writer, std::string_view NameOrValue)
+{
+ Writer.SetNameOrAddString(NameOrValue);
+ return Writer;
+}
+
+/** Write the field name if valid in this state, otherwise write the string value. */
+inline CbWriter&
+operator<<(CbWriter& Writer, const char* NameOrValue)
+{
+ return Writer << std::string_view(NameOrValue);
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbFieldView& Value)
+{
+ Writer.AddField(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbField& Value)
+{
+ Writer.AddField(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbObjectView& Value)
+{
+ Writer.AddObject(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbObject& Value)
+{
+ Writer.AddObject(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbArrayView& Value)
+{
+ Writer.AddArray(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbArray& Value)
+{
+ Writer.AddArray(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, nullptr_t)
+{
+ Writer.AddNull();
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, std::wstring_view Value)
+{
+ Writer.AddString(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const wchar_t* Value)
+{
+ Writer.AddString(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, int32_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, int64_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, uint32_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, uint64_t Value)
+{
+ Writer.AddInteger(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, float Value)
+{
+ Writer.AddFloat(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, double Value)
+{
+ Writer.AddFloat(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, bool Value)
+{
+ Writer.AddBool(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const CbAttachment& Attachment)
+{
+ Writer.AddAttachment(Attachment);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const IoHash& Value)
+{
+ Writer.AddHash(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const Guid& Value)
+{
+ Writer.AddUuid(Value);
+ return Writer;
+}
+
+inline CbWriter&
+operator<<(CbWriter& Writer, const Oid& Value)
+{
+ Writer.AddObjectId(Value);
+ return Writer;
+}
+
+ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value);
+ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void usonbuilder_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h
new file mode 100644
index 000000000..c98ab047f
--- /dev/null
+++ b/zencore/include/zencore/compactbinarypackage.h
@@ -0,0 +1,305 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/iohash.h>
+
+#include <functional>
+#include <span>
+
+namespace zen {
+
+class CbWriter;
+class BinaryReader;
+class BinaryWriter;
+class IoBuffer;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An attachment is either binary or compact binary and is identified by its hash.
+ *
+ * A compact binary attachment is also a valid binary attachment and may be accessed as binary.
+ *
+ * Attachments are serialized as one or two compact binary fields with no name. A Binary field is
+ * written first with its content. The content hash is omitted when the content size is zero, and
+ * is otherwise written as a BinaryReference or CompactBinaryReference depending on the type.
+ */
+class CbAttachment
+{
+public:
+ /** Construct a null attachment. */
+ CbAttachment() = default;
+
+ /** Construct a compact binary attachment. Value is cloned if not owned. */
+ inline explicit CbAttachment(CbFieldIterator Value) : CbAttachment(std::move(Value), nullptr) {}
+
+ /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */
+ inline explicit CbAttachment(CbFieldIterator Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {}
+
+ /** Construct a binary attachment. Value is cloned if not owned. */
+ inline explicit CbAttachment(SharedBuffer Value) : CbAttachment(std::move(Value), nullptr) {}
+
+ /** Construct a binary attachment. Value is cloned if not owned. Hash must match Value. */
+ inline explicit CbAttachment(SharedBuffer Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {}
+
+ /** Reset this to a null attachment. */
+ inline void Reset() { *this = CbAttachment(); }
+
+ /** Whether the attachment has a value. */
+ inline explicit operator bool() const { return !IsNull(); }
+
+ /** Whether the attachment has a value. */
+ inline bool IsNull() const { return !Buffer; }
+
+ /** Access the attachment as binary. Defaults to a null buffer on error. */
+ ZENCORE_API SharedBuffer AsBinaryView() const;
+
+ /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */
+ ZENCORE_API CbFieldIterator AsCompactBinary() const;
+
+ /** Returns whether the attachment is binary or compact binary. */
+ inline bool IsBinary() const { return !Buffer.IsNull(); }
+
+ /** Returns whether the attachment is compact binary. */
+ inline bool IsCompactBinary() const { return CompactBinary.HasValue(); }
+
+ /** Returns the hash of the attachment value. */
+ inline const IoHash& GetHash() const { return Hash; }
+
+ /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */
+ inline bool operator==(const CbAttachment& Attachment) const { return Hash == Attachment.Hash; }
+ inline bool operator!=(const CbAttachment& Attachment) const { return Hash != Attachment.Hash; }
+ inline bool operator<(const CbAttachment& Attachment) const { return Hash < Attachment.Hash; }
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ *
+ * The attachment references the input iterator if it is owned, and otherwise clones the value.
+ *
+ * The iterator is advanced as attachment fields are consumed from it.
+ */
+ ZENCORE_API void Load(CbFieldIterator& Fields);
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ */
+ ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /**
+ * Load the attachment from compact binary as written by Save.
+ */
+ ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /** Save the attachment into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(CbWriter& Writer) const;
+
+ /** Save the attachment into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(BinaryWriter& Writer) const;
+
+private:
+ ZENCORE_API CbAttachment(CbFieldIterator Value, const IoHash* Hash);
+ ZENCORE_API CbAttachment(SharedBuffer Value, const IoHash* Hash);
+
+ /** An owned buffer containing the binary or compact binary data. */
+ SharedBuffer Buffer;
+ /** A field iterator that is valid only for compact binary attachments. */
+ CbFieldViewIterator CompactBinary;
+ /** A hash of the attachment value. */
+ IoHash Hash;
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * A package is a compact binary object with attachments for its external references.
+ *
+ * A package is basically a Merkle tree with compact binary as its root and other non-leaf nodes,
+ * and either binary or compact binary as its leaf nodes. A node references its child nodes using
+ * BinaryHash or FieldHash fields in its compact binary representation.
+ *
+ * It is invalid for a package to include attachments that are not referenced by its object or by
+ * one of its referenced compact binary attachments. When attachments are added explicitly, it is
+ * the responsibility of the package creator to follow this requirement. Attachments that are not
+ * referenced may not survive a round-trip through certain storage systems.
+ *
+ * It is valid for a package to exclude referenced attachments, but then it is the responsibility
+ * of the package consumer to have a mechanism for resolving those references when necessary.
+ *
+ * A package is serialized as a sequence of compact binary fields with no name. The object may be
+ * both preceded and followed by attachments. The object itself is written as an Object field and
+ * followed by its hash in a CompactBinaryReference field when the object is non-empty. A package
+ * ends with a Null field. The canonical order of components is the object and its hash, followed
+ * by the attachments ordered by hash, followed by a Null field. It is valid for the a package to
+ * have its components serialized in any order, provided there is at most one object and the null
+ * field is written last.
+ */
+class CbPackage
+{
+public:
+ /**
+ * A function that resolves a hash to a buffer containing the data matching that hash.
+ *
+ * The resolver may return a null buffer to skip resolving an attachment for the hash.
+ */
+ using AttachmentResolver = std::function<SharedBuffer(const IoHash& Hash)>;
+
+ /** Construct a null package. */
+ CbPackage() = default;
+
+ /**
+ * Construct a package from a root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ */
+ inline explicit CbPackage(CbObject InObject) { SetObject(std::move(InObject)); }
+
+ /**
+ * Construct a package from a root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline explicit CbPackage(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), InResolver); }
+
+ /**
+ * Construct a package from a root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ */
+ inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), InObjectHash); }
+
+ /**
+ * Construct a package from a root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline explicit CbPackage(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver)
+ {
+ SetObject(std::move(InObject), InObjectHash, InResolver);
+ }
+
+ /** Reset this to a null package. */
+ inline void Reset() { *this = CbPackage(); }
+
+ /** Whether the package has a non-empty object or attachments. */
+ inline explicit operator bool() const { return !IsNull(); }
+
+ /** Whether the package has an empty object and no attachments. */
+ inline bool IsNull() const { return !Object.CreateIterator() && Attachments.size() == 0; }
+
+ /** Returns the compact binary object for the package. */
+ inline const CbObject& GetObject() const { return Object; }
+
+ /** Returns the has of the compact binary object for the package. */
+ inline const IoHash& GetObjectHash() const { return ObjectHash; }
+
+ /**
+ * Set the root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ */
+ inline void SetObject(CbObject InObject) { SetObject(std::move(InObject), nullptr, nullptr); }
+
+ /**
+ * Set the root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline void SetObject(CbObject InObject, AttachmentResolver InResolver) { SetObject(std::move(InObject), nullptr, &InResolver); }
+
+ /**
+ * Set the root object without gathering attachments.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ */
+ inline void SetObject(CbObject InObject, const IoHash& InObjectHash) { SetObject(std::move(InObject), &InObjectHash, nullptr); }
+
+ /**
+ * Set the root object and gather attachments using the resolver.
+ *
+ * @param InObject The root object, which will be cloned unless it is owned.
+ * @param InObjectHash The hash of the object, which must match to avoid validation errors.
+ * @param InResolver A function that is invoked for every reference and binary reference field.
+ */
+ inline void SetObject(CbObject InObject, const IoHash& InObjectHash, AttachmentResolver InResolver)
+ {
+ SetObject(std::move(InObject), &InObjectHash, &InResolver);
+ }
+
+ /** Returns the attachments in this package. */
+ inline std::span<const CbAttachment> GetAttachments() const { return Attachments; }
+
+ /**
+ * Find an attachment by its hash.
+ *
+ * @return The attachment, or null if the attachment is not found.
+ * @note The returned pointer is only valid until the attachments on this package are modified.
+ */
+ ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const;
+
+ /** Find an attachment if it exists in the package. */
+ inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); }
+
+ /** Add the attachment to this package. */
+ inline void AddAttachment(const CbAttachment& Attachment) { AddAttachment(Attachment, nullptr); }
+
+ /** Add the attachment to this package, along with any references that can be resolved. */
+ inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); }
+
+ /**
+ * Remove an attachment by hash.
+ *
+ * @return Number of attachments removed, which will be either 0 or 1.
+ */
+ ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash);
+ inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); }
+
+ /** Compares packages by their object and attachment hashes. */
+ ZENCORE_API bool Equals(const CbPackage& Package) const;
+ inline bool operator==(const CbPackage& Package) const { return Equals(Package); }
+ inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); }
+
+ /**
+ * Load the object and attachments from compact binary as written by Save.
+ *
+ * The object and attachments reference the input iterator, if it is owned, and otherwise clones
+ * the object and attachments individually to make owned copies.
+ *
+ * The iterator is advanced as object and attachment fields are consumed from it.
+ */
+ ZENCORE_API void Load(CbFieldIterator& Fields);
+
+ ZENCORE_API void Load(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ ZENCORE_API void Load(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc);
+
+ /** Save the object and attachments into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(CbWriter& Writer) const;
+
+ /** Save the object and attachments into the writer as a stream of compact binary fields. */
+ ZENCORE_API void Save(BinaryWriter& Writer) const;
+
+private:
+ ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver);
+ ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver);
+
+ void GatherAttachments(const CbFieldViewIterator& Fields, AttachmentResolver Resolver);
+
+ /** Attachments ordered by their hash. */
+ std::vector<CbAttachment> Attachments;
+ CbObject Object;
+ IoHash ObjectHash;
+};
+
+void usonpackage_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h
new file mode 100644
index 000000000..3a3f432be
--- /dev/null
+++ b/zencore/include/zencore/compactbinaryvalidation.h
@@ -0,0 +1,192 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+/** Flags for validating compact binary data. */
+enum class CbValidateMode : uint32_t
+{
+ /** Skip validation if no other validation modes are enabled. */
+ None = 0,
+
+ /**
+ * Validate that the value can be read and stays inside the bounds of the memory view.
+ *
+ * This is the minimum level of validation required to be able to safely read a field, array,
+ * or object without the risk of crashing or reading out of bounds.
+ */
+ Default = 1 << 0,
+
+ /**
+ * Validate that object fields have unique non-empty names and array fields have no names.
+ *
+ * Name validation failures typically do not inhibit reading the input, but duplicated fields
+ * cannot be looked up by name other than the first, and converting to other data formats can
+ * fail in the presence of naming issues.
+ */
+ Names = 1 << 1,
+
+ /**
+ * Validate that fields are serialized in the canonical format.
+ *
+ * Format validation failures typically do not inhibit reading the input. Values that fail in
+ * this mode require more memory than in the canonical format, and comparisons of such values
+ * for equality are not reliable. Examples of failures include uniform arrays or objects that
+ * were not encoded uniformly, variable-length integers that could be encoded in fewer bytes,
+ * or 64-bit floats that could be encoded in 32 bits without loss of precision.
+ */
+ Format = 1 << 2,
+
+ /**
+ * Validate that there is no padding after the value before the end of the memory view.
+ *
+ * Padding validation failures have no impact on the ability to read the input, but are using
+ * more memory than necessary.
+ */
+ Padding = 1 << 3,
+
+ /**
+ * Validate that a package or attachment has the expected fields and matches its saved hashes.
+ */
+ Package = 1 << 4,
+
+ /** Perform all validation described above. */
+ All = Default | Names | Format | Padding | Package,
+};
+
+ENUM_CLASS_FLAGS(CbValidateMode);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Flags for compact binary validation errors. Multiple flags may be combined. */
+enum class CbValidateError : uint32_t
+{
+ /** The input had no validation errors. */
+ None = 0,
+
+ // Mode: Default
+
+ /** The input cannot be read without reading out of bounds. */
+ OutOfBounds = 1 << 0,
+ /** The input has a field with an unrecognized or invalid type. */
+ InvalidType = 1 << 1,
+
+ // Mode: Names
+
+ /** An object had more than one field with the same name. */
+ DuplicateName = 1 << 2,
+ /** An object had a field with no name. */
+ MissingName = 1 << 3,
+ /** An array field had a name. */
+ ArrayName = 1 << 4,
+
+ // Mode: Format
+
+ /** A name or string payload is not valid UTF-8. */
+ InvalidString = 1 << 5,
+ /** A size or integer payload can be encoded in fewer bytes. */
+ InvalidInteger = 1 << 6,
+ /** A float64 payload can be encoded as a float32 without loss of precision. */
+ InvalidFloat = 1 << 7,
+ /** An object has the same type for every field but is not uniform. */
+ NonUniformObject = 1 << 8,
+ /** An array has the same type for every field and non-empty payloads but is not uniform. */
+ NonUniformArray = 1 << 9,
+
+ // Mode: Padding
+
+ /** A value did not use the entire memory view given for validation. */
+ Padding = 1 << 10,
+
+ // Mode: Package
+
+ /** The package or attachment had missing fields or fields out of order. */
+ InvalidPackageFormat = 1 << 11,
+ /** The object or an attachment did not match the hash stored for it. */
+ InvalidPackageHash = 1 << 12,
+ /** The package contained more than one copy of the same attachment. */
+ DuplicateAttachments = 1 << 13,
+ /** The package contained more than one object. */
+ MultiplePackageObjects = 1 << 14,
+ /** The package contained an object with no fields. */
+ NullPackageObject = 1 << 15,
+ /** The package contained a null attachment. */
+ NullPackageAttachment = 1 << 16,
+};
+
+ENUM_CLASS_FLAGS(CbValidateError);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Validate the compact binary data for one field in the view as specified by the mode flags.
+ *
+ * Only one top-level field is processed from the view, and validation recurses into any array or
+ * object within that field. To validate multiple consecutive top-level fields, call the function
+ * once for each top-level field. If the given view might contain multiple top-level fields, then
+ * either exclude the Padding flag from the Mode or use MeasureCompactBinary to break up the view
+ * into its constituent fields before validating.
+ *
+ * @param View A memory view containing at least one top-level field.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @param Type HasFieldType means that View contains the type. Otherwise, use the given type.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType);
+
+/**
+ * Validate the compact binary data for every field in the view as specified by the mode flags.
+ *
+ * This function expects the entire view to contain fields. Any trailing region of the view which
+ * does not contain a valid field will produce an OutOfBounds or InvalidType error instead of the
+ * Padding error that would be produced by the single field validation function.
+ *
+ * @see ValidateCompactBinary
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode);
+
+/**
+ * Validate the compact binary attachment pointed to by the view as specified by the mode flags.
+ *
+ * The attachment is validated with ValidateCompactBinary by using the validation mode specified.
+ * Include ECbValidateMode::Package to validate the attachment format and hash.
+ *
+ * @see ValidateCompactBinary
+ *
+ * @param View A memory view containing a package.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryAttachment(MemoryView View, CbValidateMode Mode);
+
+/**
+ * Validate the compact binary package pointed to by the view as specified by the mode flags.
+ *
+ * The package, and attachments, are validated with ValidateCompactBinary by using the validation
+ * mode specified. Include ECbValidateMode::Package to validate the package format and hashes.
+ *
+ * @see ValidateCompactBinary
+ *
+ * @param View A memory view containing a package.
+ * @param Mode A combination of the flags for the types of validation to perform.
+ * @return None on success, otherwise the flags for the types of errors that were detected.
+ */
+ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void usonvalidation_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
new file mode 100644
index 000000000..759cf8444
--- /dev/null
+++ b/zencore/include/zencore/compress.h
@@ -0,0 +1,53 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore/zencore.h"
+
+namespace zen::CompressedBuffer {
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static constexpr uint64_t DefaultBlockSize = 256 * 1024;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Method used to compress the data in a compressed buffer. */
+enum class Method : uint8_t
+{
+ /** Header is followed by one uncompressed block. */
+ None = 0,
+ /** Header is followed by an array of compressed block sizes then the compressed blocks. */
+ LZ4 = 4,
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+/** Header used on every compressed buffer. Always stored in big-endian format */
+struct BufferHeader
+{
+ static constexpr uint32_t ExpectedMagic = 0xb7756362;
+
+ /** A magic number to identify a compressed buffer. Always 0xb7756362 */
+ uint32_t Magic = ExpectedMagic;
+ /** A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 */
+ uint32_t Crc32 = 0;
+ /** The method used to compress the buffer. Affects layout of data following the header */
+ Method Method = Method::None;
+ /** The reserved bytes must be initialized to zero */
+ uint8_t Reserved[2]{};
+ /** The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent */
+ uint8_t BlockSizeExponent = 0;
+ /** The number of blocks that follow the header */
+ uint32_t BlockCount = 0;
+ /** The total size of the uncompressed data */
+ uint64_t TotalRawSize = 0;
+ /** The total size of the compressed data including the header */
+ uint64_t TotalCompressedSize = 0;
+};
+
+static_assert(sizeof(BufferHeader) == 32, "BufferHeader is the wrong size");
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace zen::CompressedBuffer
diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h
new file mode 100644
index 000000000..27c831bb1
--- /dev/null
+++ b/zencore/include/zencore/endian.h
@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+namespace zen {
+
+inline uint16_t
+ByteSwap(uint16_t x)
+{
+ return _byteswap_ushort(x);
+}
+
+inline uint32_t
+ByteSwap(uint32_t x)
+{
+ return _byteswap_ulong(x);
+}
+
+inline uint64_t
+ByteSwap(uint64_t x)
+{
+ return _byteswap_uint64(x);
+}
+
+inline uint16_t
+FromNetworkOrder(uint16_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint32_t
+FromNetworkOrder(uint32_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint64_t
+FromNetworkOrder(uint64_t x)
+{
+ return ByteSwap(x);
+}
+
+inline uint16_t
+FromNetworkOrder(int16_t x)
+{
+ return ByteSwap(uint16_t(x));
+}
+
+inline uint32_t
+FromNetworkOrder(int32_t x)
+{
+ return ByteSwap(uint32_t(x));
+}
+
+inline uint64_t
+FromNetworkOrder(int64_t x)
+{
+ return ByteSwap(uint64_t(x));
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/enumflags.h b/zencore/include/zencore/enumflags.h
new file mode 100644
index 000000000..ebe747bf0
--- /dev/null
+++ b/zencore/include/zencore/enumflags.h
@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+// Enum class helpers
+
+// Defines all bitwise operators for enum classes so it can be (mostly) used as a regular flags enum
+#define ENUM_CLASS_FLAGS(Enum) \
+ inline Enum& operator|=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \
+ inline Enum& operator&=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \
+ inline Enum& operator^=(Enum& Lhs, Enum Rhs) { return Lhs = (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator|(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs | (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator&(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs & (__underlying_type(Enum))Rhs); } \
+ inline constexpr Enum operator^(Enum Lhs, Enum Rhs) { return (Enum)((__underlying_type(Enum))Lhs ^ (__underlying_type(Enum))Rhs); } \
+ inline constexpr bool operator!(Enum E) { return !(__underlying_type(Enum))E; } \
+ inline constexpr Enum operator~(Enum E) { return (Enum) ~(__underlying_type(Enum))E; }
+
+// Friends all bitwise operators for enum classes so the definition can be kept private / protected.
+#define FRIEND_ENUM_CLASS_FLAGS(Enum) \
+ friend Enum& operator|=(Enum& Lhs, Enum Rhs); \
+ friend Enum& operator&=(Enum& Lhs, Enum Rhs); \
+ friend Enum& operator^=(Enum& Lhs, Enum Rhs); \
+ friend constexpr Enum operator|(Enum Lhs, Enum Rhs); \
+ friend constexpr Enum operator&(Enum Lhs, Enum Rhs); \
+ friend constexpr Enum operator^(Enum Lhs, Enum Rhs); \
+ friend constexpr bool operator!(Enum E); \
+ friend constexpr Enum operator~(Enum E);
+
+template<typename Enum>
+constexpr bool
+EnumHasAllFlags(Enum Flags, Enum Contains)
+{
+ return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) == ((__underlying_type(Enum))Contains);
+}
+
+template<typename Enum>
+constexpr bool
+EnumHasAnyFlags(Enum Flags, Enum Contains)
+{
+ return (((__underlying_type(Enum))Flags) & (__underlying_type(Enum))Contains) != 0;
+}
+
+template<typename Enum>
+void
+EnumAddFlags(Enum& Flags, Enum FlagsToAdd)
+{
+ Flags |= FlagsToAdd;
+}
+
+template<typename Enum>
+void
+EnumRemoveFlags(Enum& Flags, Enum FlagsToRemove)
+{
+ Flags &= ~FlagsToRemove;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h
new file mode 100644
index 000000000..07c4833ff
--- /dev/null
+++ b/zencore/include/zencore/except.h
@@ -0,0 +1,60 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/string.h>
+#include <zencore/windows.h>
+#include <string>
+
+namespace zen {
+
+class WindowsException : public std::exception
+{
+public:
+ WindowsException(const char* Message)
+ {
+ m_hResult = HRESULT_FROM_WIN32(GetLastError());
+ m_Message = Message;
+ }
+
+ WindowsException(HRESULT hRes, const char* Message)
+ {
+ m_hResult = hRes;
+ m_Message = Message;
+ }
+
+ WindowsException(HRESULT hRes, const char* Message, const char* Detail)
+ {
+ m_hResult = hRes;
+
+ ExtendableStringBuilder<128> msg;
+ msg.Append(Message);
+ msg.Append(" (detail: '");
+ msg.Append(Detail);
+ msg.Append("')");
+
+ m_Message = msg.c_str();
+ }
+
+ virtual const char* what() const override { return m_Message.c_str(); }
+
+private:
+ std::string m_Message;
+ HRESULT m_hResult;
+};
+
+ZENCORE_API void ThrowSystemException(HRESULT hRes, const char* Message);
+inline void
+ThrowSystemException(const char* Message)
+{
+ throw WindowsException(Message);
+}
+
+inline void
+ThrowIfFailed(HRESULT hRes, const char* Message)
+{
+ if (FAILED(hRes))
+ ThrowSystemException(hRes, Message);
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
new file mode 100644
index 000000000..b20a1e7c6
--- /dev/null
+++ b/zencore/include/zencore/filesystem.h
@@ -0,0 +1,74 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "stream.h"
+#include "zencore.h"
+
+#include <filesystem>
+#include <functional>
+
+namespace zen {
+
+class IoBuffer;
+
+/** Delete directory (after deleting any contents)
+ */
+ZENCORE_API bool DeleteDirectories(const wchar_t* dir);
+ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir);
+
+/** Ensure directory exists.
+
+ Will also create any required parent directories
+ */
+ZENCORE_API bool CreateDirectories(const wchar_t* dir);
+ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir);
+
+/** Ensure directory exists and delete contents (if any) before returning
+ */
+ZENCORE_API bool CleanDirectory(const wchar_t* dir);
+ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir);
+
+struct FileContents
+{
+ std::vector<IoBuffer> Data;
+ std::error_code ErrorCode;
+};
+
+ZENCORE_API FileContents ReadFile(std::filesystem::path Path);
+ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc);
+ZENCORE_API bool WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount);
+
+struct CopyFileOptions
+{
+ bool EnableClone = true;
+ bool MustClone = false;
+};
+
+ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
+ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
+
+ZENCORE_API std::string ToUtf8(const std::filesystem::path& Path);
+
+/**
+ * Efficient file system traversal
+ *
+ * Uses the best available mechanism for the platform in question and could take
+ * advantage of any file system tracking mechanisms in the future
+ *
+ */
+class FileSystemTraversal
+{
+public:
+ struct TreeVisitor
+ {
+ virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0;
+
+ // This should return true if we should recurse into the directory
+ virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0;
+ };
+
+ void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/fmtutils.h b/zencore/include/zencore/fmtutils.h
new file mode 100644
index 000000000..fb5a08d56
--- /dev/null
+++ b/zencore/include/zencore/fmtutils.h
@@ -0,0 +1,49 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+
+#include <fmt/format.h>
+#include <filesystem>
+#include <string_view>
+
+// Custom formatting for some zencore types
+
+template<>
+struct fmt::formatter<zen::IoHash> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::IoHash& Hash, FormatContext& ctx)
+ {
+ zen::IoHash::String_t String;
+ Hash.ToHexString(String);
+ return formatter<string_view>::format({String, zen::IoHash::StringLength}, ctx);
+ }
+};
+
+template<>
+struct fmt::formatter<zen::Oid> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const zen::Oid& Id, FormatContext& ctx)
+ {
+ zen::StringBuilder<32> String;
+ Id.ToString(String);
+ return formatter<string_view>::format({String.c_str(), zen::Oid::StringLength}, ctx);
+ }
+};
+
+template<>
+struct fmt::formatter<std::filesystem::path> : formatter<string_view>
+{
+ template<typename FormatContext>
+ auto format(const std::filesystem::path& Path, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<128> String;
+ WideToUtf8(Path.c_str(), String);
+ return formatter<string_view>::format(String.ToView(), ctx);
+ }
+};
diff --git a/zencore/include/zencore/httpclient.h b/zencore/include/zencore/httpclient.h
new file mode 100644
index 000000000..4ede6839c
--- /dev/null
+++ b/zencore/include/zencore/httpclient.h
@@ -0,0 +1,18 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/string.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class HttpClient
+{
+};
+
+} // namespace zen
+
+void httpclient_forcelink(); // internal
diff --git a/zencore/include/zencore/httpserver.h b/zencore/include/zencore/httpserver.h
new file mode 100644
index 000000000..563245264
--- /dev/null
+++ b/zencore/include/zencore/httpserver.h
@@ -0,0 +1,373 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/enumflags.h>
+#include <zencore/refcount.h>
+#include <zencore/string.h>
+#include <functional>
+#include <gsl/gsl-lite.hpp>
+#include <list>
+#include <regex>
+#include <span>
+#include <unordered_map>
+#include "zencore.h"
+
+namespace zen {
+
+class IoBuffer;
+class CbObject;
+class StringBuilderBase;
+
+enum class HttpVerb
+{
+ kGet = 1 << 0,
+ kPut = 1 << 1,
+ kPost = 1 << 2,
+ kDelete = 1 << 3,
+ kHead = 1 << 4,
+ kCopy = 1 << 5,
+ kOptions = 1 << 6
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb);
+
+enum class HttpResponse
+{
+ // 1xx - Informational
+
+ Continue = 100, //!< Indicates that the initial part of a request has been received and has not yet been rejected by the server.
+ SwitchingProtocols = 101, //!< Indicates that the server understands and is willing to comply with the client's request, via the
+ //!< Upgrade header field, for a change in the application protocol being used on this connection.
+ Processing = 102, //!< Is an interim response used to inform the client that the server has accepted the complete request, but has not
+ //!< yet completed it.
+ EarlyHints = 103, //!< Indicates to the client that the server is likely to send a final response with the header fields included in
+ //!< the informational response.
+
+ // 2xx - Successful
+
+ OK = 200, //!< Indicates that the request has succeeded.
+ Created = 201, //!< Indicates that the request has been fulfilled and has resulted in one or more new resources being created.
+ Accepted = 202, //!< Indicates that the request has been accepted for processing, but the processing has not been completed.
+ NonAuthoritativeInformation = 203, //!< Indicates that the request was successful but the enclosed payload has been modified from that
+ //!< of the origin server's 200 (OK) response by a transforming proxy.
+ NoContent = 204, //!< Indicates that the server has successfully fulfilled the request and that there is no additional content to send
+ //!< in the response payload body.
+ ResetContent = 205, //!< Indicates that the server has fulfilled the request and desires that the user agent reset the \"document
+ //!< view\", which caused the request to be sent, to its original state as received from the origin server.
+ PartialContent = 206, //!< Indicates that the server is successfully fulfilling a range request for the target resource by transferring
+ //!< one or more parts of the selected representation that correspond to the satisfiable ranges found in the
+ //!< requests's Range header field.
+ MultiStatus = 207, //!< Provides status for multiple independent operations.
+ AlreadyReported = 208, //!< Used inside a DAV:propstat response element to avoid enumerating the internal members of multiple bindings
+ //!< to the same collection repeatedly. [RFC 5842]
+ IMUsed = 226, //!< The server has fulfilled a GET request for the resource, and the response is a representation of the result of one
+ //!< or more instance-manipulations applied to the current instance.
+
+ // 3xx - Redirection
+
+ MultipleChoices = 300, //!< Indicates that the target resource has more than one representation, each with its own more specific
+ //!< identifier, and information about the alternatives is being provided so that the user (or user agent) can
+ //!< select a preferred representation by redirecting its request to one or more of those identifiers.
+ MovedPermanently = 301, //!< Indicates that the target resource has been assigned a new permanent URI and any future references to this
+ //!< resource ought to use one of the enclosed URIs.
+ Found = 302, //!< Indicates that the target resource resides temporarily under a different URI.
+ SeeOther = 303, //!< Indicates that the server is redirecting the user agent to a different resource, as indicated by a URI in the
+ //!< Location header field, that is intended to provide an indirect response to the original request.
+ NotModified = 304, //!< Indicates that a conditional GET request has been received and would have resulted in a 200 (OK) response if it
+ //!< were not for the fact that the condition has evaluated to false.
+ UseProxy = 305, //!< \deprecated \parblock Due to security concerns regarding in-band configuration of a proxy. \endparblock
+ //!< The requested resource MUST be accessed through the proxy given by the Location field.
+ TemporaryRedirect = 307, //!< Indicates that the target resource resides temporarily under a different URI and the user agent MUST NOT
+ //!< change the request method if it performs an automatic redirection to that URI.
+ PermanentRedirect = 308, //!< The target resource has been assigned a new permanent URI and any future references to this resource
+ //!< ought to use one of the enclosed URIs. [...] This status code is similar to 301 Moved Permanently
+ //!< (Section 7.3.2 of rfc7231), except that it does not allow rewriting the request method from POST to GET.
+
+ // 4xx - Client Error
+ BadRequest = 400, //!< Indicates that the server cannot or will not process the request because the received syntax is invalid,
+ //!< nonsensical, or exceeds some limitation on what the server is willing to process.
+ Unauthorized = 401, //!< Indicates that the request has not been applied because it lacks valid authentication credentials for the
+ //!< target resource.
+ PaymentRequired = 402, //!< *Reserved*
+ Forbidden = 403, //!< Indicates that the server understood the request but refuses to authorize it.
+ NotFound = 404, //!< Indicates that the origin server did not find a current representation for the target resource or is not willing
+ //!< to disclose that one exists.
+ MethodNotAllowed = 405, //!< Indicates that the method specified in the request-line is known by the origin server but not supported by
+ //!< the target resource.
+ NotAcceptable = 406, //!< Indicates that the target resource does not have a current representation that would be acceptable to the
+ //!< user agent, according to the proactive negotiation header fields received in the request, and the server is
+ //!< unwilling to supply a default representation.
+ ProxyAuthenticationRequired =
+ 407, //!< Is similar to 401 (Unauthorized), but indicates that the client needs to authenticate itself in order to use a proxy.
+ RequestTimeout =
+ 408, //!< Indicates that the server did not receive a complete request message within the time that it was prepared to wait.
+ Conflict = 409, //!< Indicates that the request could not be completed due to a conflict with the current state of the resource.
+ Gone = 410, //!< Indicates that access to the target resource is no longer available at the origin server and that this condition is
+ //!< likely to be permanent.
+ LengthRequired = 411, //!< Indicates that the server refuses to accept the request without a defined Content-Length.
+ PreconditionFailed =
+ 412, //!< Indicates that one or more preconditions given in the request header fields evaluated to false when tested on the server.
+ PayloadTooLarge = 413, //!< Indicates that the server is refusing to process a request because the request payload is larger than the
+ //!< server is willing or able to process.
+ URITooLong = 414, //!< Indicates that the server is refusing to service the request because the request-target is longer than the
+ //!< server is willing to interpret.
+ UnsupportedMediaType = 415, //!< Indicates that the origin server is refusing to service the request because the payload is in a format
+ //!< not supported by the target resource for this method.
+ RangeNotSatisfiable = 416, //!< Indicates that none of the ranges in the request's Range header field overlap the current extent of the
+ //!< selected resource or that the set of ranges requested has been rejected due to invalid ranges or an
+ //!< excessive request of small or overlapping ranges.
+ ExpectationFailed = 417, //!< Indicates that the expectation given in the request's Expect header field could not be met by at least
+ //!< one of the inbound servers.
+ ImATeapot = 418, //!< Any attempt to brew coffee with a teapot should result in the error code 418 I'm a teapot.
+ UnprocessableEntity = 422, //!< Means the server understands the content type of the request entity (hence a 415(Unsupported Media
+ //!< Type) status code is inappropriate), and the syntax of the request entity is correct (thus a 400 (Bad
+ //!< Request) status code is inappropriate) but was unable to process the contained instructions.
+ Locked = 423, //!< Means the source or destination resource of a method is locked.
+ FailedDependency = 424, //!< Means that the method could not be performed on the resource because the requested action depended on
+ //!< another action and that action failed.
+ UpgradeRequired = 426, //!< Indicates that the server refuses to perform the request using the current protocol but might be willing to
+ //!< do so after the client upgrades to a different protocol.
+ PreconditionRequired = 428, //!< Indicates that the origin server requires the request to be conditional.
+ TooManyRequests = 429, //!< Indicates that the user has sent too many requests in a given amount of time (\"rate limiting\").
+ RequestHeaderFieldsTooLarge =
+ 431, //!< Indicates that the server is unwilling to process the request because its header fields are too large.
+ UnavailableForLegalReasons =
+ 451, //!< This status code indicates that the server is denying access to the resource in response to a legal demand.
+
+ // 5xx - Server Error
+
+ InternalServerError =
+ 500, //!< Indicates that the server encountered an unexpected condition that prevented it from fulfilling the request.
+ NotImplemented = 501, //!< Indicates that the server does not support the functionality required to fulfill the request.
+ BadGateway = 502, //!< Indicates that the server, while acting as a gateway or proxy, received an invalid response from an inbound
+ //!< server it accessed while attempting to fulfill the request.
+ ServiceUnavailable = 503, //!< Indicates that the server is currently unable to handle the request due to a temporary overload or
+ //!< scheduled maintenance, which will likely be alleviated after some delay.
+ GatewayTimeout = 504, //!< Indicates that the server, while acting as a gateway or proxy, did not receive a timely response from an
+ //!< upstream server it needed to access in order to complete the request.
+ HTTPVersionNotSupported = 505, //!< Indicates that the server does not support, or refuses to support, the protocol version that was
+ //!< used in the request message.
+ VariantAlsoNegotiates =
+ 506, //!< Indicates that the server has an internal configuration error: the chosen variant resource is configured to engage in
+ //!< transparent content negotiation itself, and is therefore not a proper end point in the negotiation process.
+ InsufficientStorage = 507, //!< Means the method could not be performed on the resource because the server is unable to store the
+ //!< representation needed to successfully complete the request.
+ LoopDetected = 508, //!< Indicates that the server terminated an operation because it encountered an infinite loop while processing a
+ //!< request with "Depth: infinity". [RFC 5842]
+ NotExtended = 510, //!< The policy for accessing the resource has not been met in the request. [RFC 2774]
+ NetworkAuthenticationRequired = 511, //!< Indicates that the client needs to authenticate to gain network access.
+};
+
+enum class HttpContentType
+{
+ kBinary,
+ kText,
+ kJSON,
+ kCbObject,
+ kCbPackage
+};
+
+/** HTTP Server Request
+ */
+class HttpServerRequest
+{
+public:
+ HttpServerRequest();
+ ~HttpServerRequest();
+
+ // Synchronous operations
+
+ inline [[nodiscard]] std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix
+ inline [[nodiscard]] std::string_view QueryString() const { return m_QueryString; }
+ inline bool IsHandled() const { return m_IsHandled; }
+
+ struct QueryParams
+ {
+ std::vector<std::pair<std::string_view, std::string_view>> KvPairs;
+
+ std::string_view GetValue(std::string_view ParamName)
+ {
+ for (const auto& Kv : KvPairs)
+ {
+ const std::string_view& Key = Kv.first;
+
+ if (Key.size() == ParamName.size())
+ {
+ if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size()))
+ {
+ return Kv.second;
+ }
+ }
+ }
+
+ return std::string_view();
+ }
+ };
+
+ QueryParams GetQueryParams();
+
+ inline HttpVerb RequestVerb() const { return m_Verb; }
+
+ const char* HeaderAccept() const;
+ const char* HeaderAcceptEncoding() const;
+ const char* HeaderContentType() const;
+ const char* HeaderContentEncoding() const;
+ inline uint64_t HeaderContentLength() const { return m_ContentLength; }
+
+ void SetSuppressResponseBody() { m_SuppressBody = true; }
+
+ // Asynchronous operations
+
+ /** Read POST/PUT payload
+
+ This will return a null buffer if the contents are not fully available yet, and the handler should
+ at that point return - another completion request will be issues once the contents have been received
+ fully.
+ */
+ virtual IoBuffer ReadPayload() = 0;
+
+ /** Respond with payload
+
+ Note that this is destructive in the sense that the IoBuffer instances referred to by Blobs will be
+ moved into our response handler array where they are kept alive, in order to reduce ref-counting storms
+ */
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) = 0;
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob);
+ virtual void WriteResponse(HttpResponse HttpResponseCode) = 0;
+
+ virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0;
+
+ void WriteResponse(HttpResponse HttpResponseCode, CbObject Data);
+ void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString);
+
+protected:
+ bool m_IsHandled = false;
+ bool m_SuppressBody = false;
+ HttpVerb m_Verb = HttpVerb::kGet;
+ uint64_t m_ContentLength = ~0ull;
+ ExtendableStringBuilder<256> m_Uri;
+ ExtendableStringBuilder<256> m_QueryString;
+};
+
+class HttpServerException : public std::exception
+{
+public:
+ HttpServerException(const char* Message, uint32_t Error) : std::exception(Message), m_ErrorCode(Error) {}
+
+private:
+ uint32_t m_ErrorCode;
+};
+
+class HttpService
+{
+public:
+ HttpService() = default;
+ virtual ~HttpService() = default;
+
+ virtual const char* BaseUri() const = 0;
+ virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) = 0;
+
+ // Internals
+
+ inline void SetUriPrefixLength(size_t PrefixLength) { m_UriPrefixLength = (int)PrefixLength; }
+ inline int UriPrefixLength() const { return m_UriPrefixLength; }
+
+private:
+ int m_UriPrefixLength = 0;
+};
+
+/** HTTP server
+ */
+class HttpServer
+{
+public:
+ HttpServer();
+ ~HttpServer();
+
+ void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler);
+ void AddEndpoint(HttpService& Service);
+
+ void Initialize(int BasePort);
+ void Run(bool TestMode);
+ void RequestExit();
+
+private:
+ struct Impl;
+
+ RefPtr<Impl> m_Impl;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpRouterRequest
+{
+public:
+ HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {}
+
+ ZENCORE_API std::string GetCapture(int Index) const;
+ inline HttpServerRequest& ServerRequest() { return m_HttpRequest; }
+
+private:
+ using MatchResults_t = std::match_results<std::string_view::const_iterator>;
+
+ HttpServerRequest& m_HttpRequest;
+ MatchResults_t m_Match;
+
+ friend class HttpRequestRouter;
+};
+
+inline std::string
+HttpRouterRequest::GetCapture(int Index) const
+{
+ ZEN_ASSERT(Index < m_Match.size());
+
+ return m_Match[Index];
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpRequestRouter
+{
+public:
+ typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t;
+
+ void AddPattern(const char* Id, const char* Regex);
+ void RegisterRoute(const char* Regex, HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs);
+ bool HandleRequest(zen::HttpServerRequest& Request);
+
+private:
+ struct HandlerEntry
+ {
+ HandlerEntry(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern)
+ : RegEx(Regex, std::regex::icase | std::regex::ECMAScript)
+ , Verbs(SupportedVerbs)
+ , Handler(std::move(Handler))
+ , Pattern(Pattern)
+ {
+ }
+
+ ~HandlerEntry() = default;
+
+ std::regex RegEx;
+ HttpVerb Verbs;
+ HandlerFunc_t Handler;
+ const char* Pattern;
+ };
+
+ std::list<HandlerEntry> m_Handlers;
+ std::unordered_map<std::string, std::string> m_PatternMap;
+};
+
+//////////////////////////////////////////////////////////////////////////
+//
+// HTTP Client
+//
+
+class HttpClient
+{
+};
+
+} // namespace zen
+
+void http_forcelink(); // internal
diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h
new file mode 100644
index 000000000..9d39b8226
--- /dev/null
+++ b/zencore/include/zencore/intmath.h
@@ -0,0 +1,140 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <stdint.h>
+#include <concepts>
+
+//////////////////////////////////////////////////////////////////////////
+
+#pragma intrinsic(_BitScanReverse)
+#pragma intrinsic(_BitScanReverse64)
+
+namespace zen {
+
+inline constexpr bool
+IsPow2(uint64_t n)
+{
+ return 0 == (n & (n - 1));
+}
+
+/// Round an integer up to the closest integer multiplier of 'base' ('base' must be a power of two)
+template<std::integral T>
+T
+RoundUp(T Value, auto Base)
+{
+ ZEN_ASSERT_SLOW(IsPow2(Base));
+ return ((Value + T(Base - 1)) & (~T(Base - 1)));
+}
+
+bool
+IsMultipleOf(std::integral auto Value, auto MultiplierPow2)
+{
+ ZEN_ASSERT_SLOW(IsPow2(MultiplierPow2));
+ return (Value & (MultiplierPow2 - 1)) == 0;
+}
+
+inline uint64_t
+NextPow2(uint64_t n)
+{
+ // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+
+ --n;
+
+ n |= n >> 1;
+ n |= n >> 2;
+ n |= n >> 4;
+ n |= n >> 8;
+ n |= n >> 16;
+ n |= n >> 32;
+
+ return n + 1;
+}
+
+static inline uint32_t
+FloorLog2(uint32_t Value)
+{
+ // Use BSR to return the log2 of the integer
+ unsigned long Log2;
+ if (_BitScanReverse(&Log2, Value) != 0)
+ {
+ return Log2;
+ }
+
+ return 0;
+}
+
+static inline uint32_t
+CountLeadingZeros(uint32_t Value)
+{
+ unsigned long Log2;
+ _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1);
+ return 32 - Log2;
+}
+
+static inline uint64_t
+FloorLog2_64(uint64_t Value)
+{
+ unsigned long Log2;
+ long Mask = -long(_BitScanReverse64(&Log2, Value) != 0);
+ return Log2 & Mask;
+}
+
+static inline uint64_t
+CountLeadingZeros64(uint64_t Value)
+{
+ unsigned long Log2;
+ long Mask = -long(_BitScanReverse64(&Log2, Value) != 0);
+ return ((63 - Log2) & Mask) | (64 & ~Mask);
+}
+
+static inline uint64_t
+CeilLogTwo64(uint64_t Arg)
+{
+ int64_t Bitmask = ((int64_t)(CountLeadingZeros64(Arg) << 57)) >> 63;
+ return (64 - CountLeadingZeros64(Arg - 1)) & (~Bitmask);
+}
+
+static inline uint64_t
+CountTrailingZeros64(uint64_t Value)
+{
+ if (Value == 0)
+ {
+ return 64;
+ }
+ unsigned long BitIndex; // 0-based, where the LSB is 0 and MSB is 31
+ _BitScanForward64(&BitIndex, Value); // Scans from LSB to MSB
+ return BitIndex;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+static inline bool
+IsPointerAligned(const void* Ptr, uint64_t Alignment)
+{
+ ZEN_ASSERT_SLOW(IsPow2(Alignment));
+
+ return 0 == (reinterpret_cast<uintptr_t>(Ptr) & (Alignment - 1));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifdef min
+# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead"
+#endif
+
+auto
+Min(auto x, auto y)
+{
+ return x < y ? x : y;
+}
+
+auto
+Max(auto x, auto y)
+{
+ return x > y ? x : y;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
new file mode 100644
index 000000000..c93d27959
--- /dev/null
+++ b/zencore/include/zencore/iobuffer.h
@@ -0,0 +1,272 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <memory.h>
+#include "refcount.h"
+#include "zencore.h"
+
+namespace zen {
+
+struct IoBufferExtendedCore;
+
+struct IoBufferFileReference
+{
+ void* FileHandle;
+ uint64_t FileChunkOffset;
+ uint64_t FileChunkSize;
+};
+
+struct IoBufferCore
+{
+public:
+ IoBufferCore() = default;
+ inline IoBufferCore(const void* DataPtr, size_t SizeBytes) : m_DataPtr(DataPtr), m_DataBytes(SizeBytes) {}
+ inline IoBufferCore(const IoBufferCore* Outer, const void* DataPtr, size_t SizeBytes)
+ : m_DataPtr(DataPtr)
+ , m_DataBytes(SizeBytes)
+ , m_OuterCore(Outer)
+ {
+ }
+
+ ZENCORE_API explicit IoBufferCore(size_t SizeBytes);
+ ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment);
+ ZENCORE_API ~IoBufferCore();
+
+ // Reference counting
+
+ inline uint32_t AddRef() const { return AtomicIncrement(const_cast<IoBufferCore*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ const uint32_t NewRefCount = AtomicDecrement(const_cast<IoBufferCore*>(this)->m_RefCount);
+ if (NewRefCount == 0)
+ {
+ DeleteThis();
+ }
+ return NewRefCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ IoBufferCore(const IoBufferCore&) = delete;
+ IoBufferCore(IoBufferCore&&) = delete;
+ IoBufferCore& operator=(const IoBufferCore&) = delete;
+ IoBufferCore& operator=(IoBufferCore&&) = delete;
+
+ //
+
+ ZENCORE_API void Materialize() const;
+ ZENCORE_API void DeleteThis() const;
+ ZENCORE_API void MakeOwned(bool Immutable = true);
+
+ inline void EnsureDataValid() const
+ {
+ if ((m_Flags & kIsExtended) && !(m_Flags & kIsMaterialized))
+ Materialize();
+ }
+
+ inline bool IsOwned() const { return !!(m_Flags & kIsOwned); }
+ inline bool IsImmutable() const { return !(m_Flags & kIsMutable); }
+ inline bool IsNull() const { return m_DataBytes == 0; }
+
+ inline IoBufferExtendedCore* ExtendedCore();
+ inline const IoBufferExtendedCore* ExtendedCore() const;
+
+ inline const void* DataPointer() const
+ {
+ EnsureDataValid();
+ return m_DataPtr;
+ }
+
+ inline size_t DataBytes() const { return m_DataBytes; }
+
+ inline void Set(const void* Ptr, size_t Sz)
+ {
+ m_DataPtr = Ptr;
+ m_DataBytes = Sz;
+ }
+
+ inline void SetIsOwned(bool NewState)
+ {
+ if (NewState)
+ {
+ m_Flags |= kIsOwned;
+ }
+ else
+ {
+ m_Flags &= ~kIsOwned;
+ }
+ }
+
+ inline void SetIsImmutable(bool NewState)
+ {
+ if (!NewState)
+ {
+ m_Flags |= kIsMutable;
+ }
+ else
+ {
+ m_Flags &= ~kIsMutable;
+ }
+ }
+
+ inline uint32_t GetRefCount() const { return m_RefCount; }
+
+protected:
+ uint32_t m_RefCount = 0;
+ mutable uint32_t m_Flags = 0;
+ mutable const void* m_DataPtr = nullptr;
+ size_t m_DataBytes = 0;
+ RefPtr<const IoBufferCore> m_OuterCore;
+
+ enum Flags
+ {
+ kIsOwned = 1 << 0,
+ kIsMutable = 1 << 1,
+ kIsExtended = 1 << 2, // Is actually a SharedBufferExtendedCore
+ kIsMaterialized = 1 << 3, // Data pointers are valid
+ kLowLevelAlloc = 1 << 4, // Using direct memory allocation
+ };
+
+ void* AllocateBuffer(size_t InSize, size_t Alignment);
+ void FreeBuffer();
+};
+
+/**
+ * An "Extended" core references a segment of a file
+ */
+
+struct IoBufferExtendedCore : public IoBufferCore
+{
+ IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership);
+ IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size);
+ ~IoBufferExtendedCore();
+
+ enum ExtendedFlags
+ {
+ kOwnsFile = 1 << 8,
+ kOwnsMmap = 1 << 9
+ };
+
+ void Materialize() const;
+ bool GetFileReference(IoBufferFileReference& OutRef) const;
+
+private:
+ void* m_FileHandle = nullptr;
+ uint64_t m_FileOffset = 0;
+ mutable void* m_MmapHandle = nullptr;
+ mutable void* m_MappedPointer = nullptr;
+};
+
+inline IoBufferExtendedCore*
+IoBufferCore::ExtendedCore()
+{
+ if (m_Flags & kIsExtended)
+ {
+ return static_cast<IoBufferExtendedCore*>(this);
+ }
+
+ return nullptr;
+}
+
+inline const IoBufferExtendedCore*
+IoBufferCore::ExtendedCore() const
+{
+ if (m_Flags & kIsExtended)
+ {
+ return static_cast<const IoBufferExtendedCore*>(this);
+ }
+
+ return nullptr;
+}
+
+/**
+ * I/O buffer
+ *
+ * This represents a reference to a payload in memory or on disk
+ *
+ */
+class IoBuffer
+{
+public:
+ enum EAssumeOwnershipTag
+ {
+ AssumeOwnership
+ };
+ enum ECloneTag
+ {
+ Clone
+ };
+ enum EWrapTag
+ {
+ Wrap
+ };
+ enum EFileTag
+ {
+ File
+ };
+ enum EBorrowedFileTag
+ {
+ BorrowedFile
+ };
+
+ inline IoBuffer() = default;
+ inline IoBuffer(IoBuffer&& Rhs) noexcept = default;
+ inline IoBuffer(const IoBuffer& Rhs) = default;
+ inline IoBuffer& operator=(const IoBuffer& Rhs) = default;
+ inline IoBuffer& operator=(IoBuffer&& Rhs) noexcept = default;
+
+ /** Create an uninitialized buffer of the given size
+ */
+ ZENCORE_API explicit IoBuffer(size_t InSize);
+
+ /** Create an uninitialized buffer of the given size with the specified alignment
+ */
+ ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment);
+
+ /** Create a buffer which references a sequence of bytes inside another buffer
+ */
+ ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes);
+
+ /** Create a buffer which references a range of bytes which we assume will live
+ * for the entire life time.
+ */
+ inline IoBuffer(EWrapTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(DataPtr, SizeBytes)) {}
+
+ inline IoBuffer(ECloneTag, const void* DataPtr, size_t SizeBytes) : m_Core(new IoBufferCore(SizeBytes))
+ {
+ memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes);
+ }
+
+ inline IoBuffer(EAssumeOwnershipTag, const void* DataPtr, size_t Sz) : m_Core(new IoBufferCore(DataPtr, Sz))
+ {
+ m_Core->SetIsOwned(true);
+ }
+
+ ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
+ ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize);
+
+ inline operator bool() const { return !m_Core->IsNull(); }
+ ZENCORE_API void MakeOwned() { return m_Core->MakeOwned(); }
+ inline bool IsOwned() const { return m_Core->IsOwned(); }
+ const void* Data() const { return m_Core->DataPointer(); }
+ const size_t Size() const { return m_Core->DataBytes(); }
+ ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const;
+
+private:
+ RefPtr<IoBufferCore> m_Core = new IoBufferCore;
+};
+
+class IoBufferBuilder
+{
+public:
+ ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
+
+private:
+};
+
+void iobuffer_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/iohash.h b/zencore/include/zencore/iohash.h
new file mode 100644
index 000000000..4eac7e328
--- /dev/null
+++ b/zencore/include/zencore/iohash.h
@@ -0,0 +1,95 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/blake3.h>
+#include <zencore/memory.h>
+
+#include <string_view>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * Hash used for content addressable storage
+ *
+ * This is basically a BLAKE3-160 hash (note: this is probably not an officially
+ * recognized identifier). It is generated by computing a 32-byte BLAKE3 hash and
+ * picking the first 20 bytes of the resulting hash.
+ *
+ */
+struct IoHash
+{
+ uint8_t Hash[20];
+
+ static IoHash MakeFrom(const void* data /* 20 bytes */)
+ {
+ IoHash Io;
+ memcpy(Io.Hash, data, sizeof Io);
+ return Io;
+ }
+
+ static IoHash HashMemory(const void* data, size_t byteCount);
+ static IoHash HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); }
+ static IoHash FromHexString(const char* string);
+ static IoHash FromHexString(const std::string_view string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+ std::string ToHexString() const;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static IoHash Zero; // Initialized to all zeros
+
+ inline auto operator<=>(const IoHash& rhs) const = default;
+
+ struct Hasher
+ {
+ size_t operator()(const IoHash& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct IoHashStream
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { m_Blake3Stream.Reset(); }
+
+ /// Append another chunk
+ IoHashStream& Append(const void* data, size_t byteCount)
+ {
+ m_Blake3Stream.Append(data, byteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ IoHashStream& Append(MemoryView Data)
+ {
+ m_Blake3Stream.Append(Data.GetData(), Data.GetSize());
+ return *this;
+ }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ IoHash GetHash()
+ {
+ BLAKE3 b3 = m_Blake3Stream.GetHash();
+
+ IoHash Io;
+ memcpy(Io.Hash, b3.Hash, sizeof Io.Hash);
+
+ return Io;
+ }
+
+private:
+ BLAKE3Stream m_Blake3Stream;
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/md5.h b/zencore/include/zencore/md5.h
new file mode 100644
index 000000000..4ed4f6c56
--- /dev/null
+++ b/zencore/include/zencore/md5.h
@@ -0,0 +1,50 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <stdint.h>
+#include <compare>
+#include "zencore.h"
+
+namespace zen {
+
+class StringBuilderBase;
+
+struct MD5
+{
+ uint8_t Hash[16];
+
+ inline auto operator<=>(const MD5& rhs) const = default;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static MD5 HashMemory(const void* data, size_t byteCount);
+ static MD5 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static MD5 Zero; // Initialized to all zeroes
+};
+
+/**
+ * Utility class for computing SHA1 hashes
+ */
+class MD5Stream
+{
+public:
+ MD5Stream();
+
+ /// Begin streaming MD5 compute (not needed on freshly constructed MD5Stream instance)
+ void Reset();
+ /// Append another chunk
+ MD5Stream& Append(const void* data, size_t byteCount);
+ /// Obtain final MD5 hash. If you wish to reuse the MD5Stream instance call reset()
+ MD5 GetHash();
+
+private:
+};
+
+void md5_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h
new file mode 100644
index 000000000..8a16126ef
--- /dev/null
+++ b/zencore/include/zencore/memory.h
@@ -0,0 +1,213 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/thread.h>
+
+#include <algorithm>
+#include <vector>
+
+namespace zen {
+
+class MemoryArena
+{
+public:
+ ZENCORE_API MemoryArena();
+ ZENCORE_API ~MemoryArena();
+
+ ZENCORE_API void* Alloc(size_t size, size_t alignment);
+ ZENCORE_API void Free(void* ptr);
+
+private:
+};
+
+class Memory
+{
+public:
+ ZENCORE_API static void* Alloc(size_t size, size_t alignment = sizeof(void*));
+ ZENCORE_API static void Free(void* ptr);
+};
+
+/** Allocator which claims fixed-size blocks from the underlying allocator.
+
+ There is no way to free individual memory blocks.
+
+ \note This is not thread-safe, you will need to provide synchronization yourself
+*/
+
+class ChunkingLinearAllocator
+{
+public:
+ ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment = sizeof(std::max_align_t));
+ ~ChunkingLinearAllocator();
+
+ ZENCORE_API void Reset();
+
+ ZENCORE_API void* Alloc(size_t Size, size_t Alignment = sizeof(void*));
+ inline void Free(void* Ptr) { ZEN_UNUSED(Ptr); /* no-op */ }
+
+private:
+ uint8_t* m_ChunkCursor = nullptr;
+ uint64_t m_ChunkBytesRemain = 0;
+ const uint64_t m_ChunkSize = 0;
+ const uint64_t m_ChunkAlignment = 0;
+ std::vector<void*> m_ChunkList;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct MutableMemoryView
+{
+ MutableMemoryView() = default;
+
+ MutableMemoryView(void* DataPtr, size_t DataSize)
+ : m_Data(reinterpret_cast<uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<uint8_t*>(DataPtr) + DataSize)
+ {
+ }
+
+ inline bool IsEmpty() const { return m_Data == m_DataEnd; }
+ void* GetData() const { return m_Data; }
+ void* GetDataEnd() const { return m_DataEnd; }
+ size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); }
+
+ inline bool EqualBytes(const MutableMemoryView& InView) const
+ {
+ const size_t Size = GetSize();
+
+ return Size == InView.GetSize() && (memcmp(m_Data, InView.m_Data, Size) == 0);
+ }
+
+ /** Modifies the view by chopping the given number of bytes from the left. */
+ inline void RightChopInline(uint64_t InSize)
+ {
+ const uint64_t Offset = std::min(GetSize(), InSize);
+ m_Data = GetDataAtOffsetNoCheck(Offset);
+ }
+
+ /** Returns the left-most part of the view by taking the given number of bytes from the left. */
+ constexpr inline MutableMemoryView Left(uint64_t InSize) const
+ {
+ MutableMemoryView View(*this);
+ View.LeftInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the left. */
+ constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); }
+
+private:
+ uint8_t* m_Data = nullptr;
+ uint8_t* m_DataEnd = nullptr;
+
+ /** Returns the data pointer advanced by an offset in bytes. */
+ inline uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; }
+};
+
+struct MemoryView
+{
+ MemoryView() = default;
+
+ MemoryView(const MutableMemoryView& MutableView)
+ : m_Data(reinterpret_cast<const uint8_t*>(MutableView.GetData()))
+ , m_DataEnd(m_Data + MutableView.GetSize())
+ {
+ }
+
+ MemoryView(const void* DataPtr, size_t DataSize)
+ : m_Data(reinterpret_cast<const uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<const uint8_t*>(DataPtr) + DataSize)
+ {
+ }
+
+ MemoryView(const void* DataPtr, const void* DataEndPtr)
+ : m_Data(reinterpret_cast<const uint8_t*>(DataPtr))
+ , m_DataEnd(reinterpret_cast<const uint8_t*>(DataEndPtr))
+ {
+ }
+
+ inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); }
+ inline bool IsEmpty() const { return m_Data == m_DataEnd; }
+ const void* GetData() const { return m_Data; }
+ const void* GetDataEnd() const { return m_DataEnd; }
+ size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); }
+ inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; }
+
+ inline bool EqualBytes(const MemoryView& InView) const
+ {
+ const size_t Size = GetSize();
+
+ return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0);
+ }
+
+ inline MemoryView& operator+=(size_t InSize)
+ {
+ RightChopInline(InSize);
+ return *this;
+ }
+
+ /** Modifies the view by chopping the given number of bytes from the left. */
+ inline void RightChopInline(uint64_t InSize)
+ {
+ const uint64_t Offset = std::min(GetSize(), InSize);
+ m_Data = GetDataAtOffsetNoCheck(Offset);
+ }
+
+ inline MemoryView RightChop(uint64_t InSize)
+ {
+ MemoryView View(*this);
+ View.RightChopInline(InSize);
+ return View;
+ }
+
+ /** Returns the left-most part of the view by taking the given number of bytes from the left. */
+ constexpr inline MemoryView Left(uint64_t InSize) const
+ {
+ MemoryView View(*this);
+ View.LeftInline(InSize);
+ return View;
+ }
+
+ /** Modifies the view to be the given number of bytes from the left. */
+ constexpr inline void LeftInline(uint64_t InSize) { m_DataEnd = std::min(m_DataEnd, m_Data + InSize); }
+
+ constexpr void Reset()
+ {
+ m_Data = nullptr;
+ m_DataEnd = nullptr;
+ }
+
+private:
+ const uint8_t* m_Data = nullptr;
+ const uint8_t* m_DataEnd = nullptr;
+
+ /** Returns the data pointer advanced by an offset in bytes. */
+ inline const uint8_t* GetDataAtOffsetNoCheck(uint64_t InOffset) const { return m_Data + InOffset; }
+};
+
+/**
+ * Make a non-owning view of the memory of the initializer list.
+ *
+ * This overload is only available when the element type does not need to be deduced.
+ */
+template<typename T>
+[[nodiscard]] inline MemoryView
+MakeMemoryView(std::initializer_list<typename std::type_identity<T>::type> List)
+{
+ return MemoryView(List.begin(), List.size() * sizeof(T));
+}
+
+/** Make a non-owning view of the memory of the contiguous container. */
+template<std::ranges::contiguous_range R>
+[[nodiscard]] constexpr inline MemoryView
+MakeMemoryView(const R& Container)
+{
+ const auto& Front = *std::begin(Container);
+ return MemoryView(std::addressof(Front), std::size(Container) * sizeof(Front));
+}
+
+void memory_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/meta.h b/zencore/include/zencore/meta.h
new file mode 100644
index 000000000..82eb5cc30
--- /dev/null
+++ b/zencore/include/zencore/meta.h
@@ -0,0 +1,30 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/* This file contains utility functions for meta programming
+ *
+ * Since you're in here you're probably quite observant, and you'll
+ * note that it's quite barren here. This is because template
+ * metaprogramming is awful and I try not to engage in it. However,
+ * sometimes these things are forced upon us.
+ *
+ */
+
+namespace zen {
+
+/**
+ * Uses implicit conversion to create an instance of a specific type.
+ * Useful to make things clearer or circumvent unintended type deduction in templates.
+ * Safer than C casts and static_casts, e.g. does not allow down-casts
+ *
+ * @param Obj The object (usually pointer or reference) to convert.
+ *
+ * @return The object converted to the specified type.
+ */
+template<typename T>
+inline T
+ImplicitConv(typename std::type_identity<T>::type Obj)
+{
+ return Obj;
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h
new file mode 100644
index 000000000..de7d315f4
--- /dev/null
+++ b/zencore/include/zencore/refcount.h
@@ -0,0 +1,144 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
+
+#include "atomic.h"
+#include "zencore.h"
+
+namespace zen {
+
+/**
+ * Helper base class for reference counted objects using intrusive reference counts
+ *
+ * This class is pretty straightforward but does one thing which may be unexpected:
+ *
+ * - Instances on the stack are initialized with a reference count of one to ensure
+ * nobody tries to accidentally delete it. (TODO: is this really useful?)
+ */
+class RefCounted
+{
+public:
+ RefCounted() : m_RefCount(IsPointerToStack(this)){};
+ virtual ~RefCounted() = default;
+
+ inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); }
+ inline uint32_t Release() const
+ {
+ uint32_t refCount = AtomicDecrement(const_cast<RefCounted*>(this)->m_RefCount);
+ if (refCount == 0)
+ {
+ delete this;
+ }
+ return refCount;
+ }
+
+ // Copying reference counted objects doesn't make a lot of sense generally, so let's prevent it
+
+ RefCounted(const RefCounted&) = delete;
+ RefCounted(RefCounted&&) = delete;
+ RefCounted& operator=(const RefCounted&) = delete;
+ RefCounted& operator=(RefCounted&&) = delete;
+
+protected:
+ inline uint32_t RefCount() const { return m_RefCount; }
+
+private:
+ uint32_t m_RefCount = 0;
+};
+
+/**
+ * Smart pointer for classes derived from RefCounted
+ */
+
+template<class T>
+class RefPtr
+{
+public:
+ inline RefPtr() = default;
+ inline RefPtr(const RefPtr& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline RefPtr(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
+ inline ~RefPtr() { m_Ref && m_Ref->Release(); }
+
+ inline explicit operator bool() const { return m_Ref != nullptr; }
+ inline operator T*() const { return m_Ref; }
+ inline T* operator->() const { return m_Ref; }
+
+ inline std::strong_ordering operator<=>(const RefPtr& Rhs) const = default;
+
+ inline RefPtr& operator=(T* Rhs)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs;
+ return *this;
+ }
+ inline RefPtr& operator=(const RefPtr& Rhs)
+ {
+ m_Ref && m_Ref->Release();
+ auto Ref = m_Ref = Rhs.m_Ref;
+ Ref && Ref->AddRef();
+ return *this;
+ }
+ inline RefPtr& operator=(RefPtr&& Rhs) noexcept
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ return *this;
+ }
+ inline RefPtr(RefPtr&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+
+private:
+ T* m_Ref = nullptr;
+};
+
+/**
+ * Smart pointer for classes derived from RefCounted
+ *
+ * This variant does not decay to a raw pointer
+ *
+ */
+
+template<class T>
+class Ref
+{
+public:
+ inline Ref() = default;
+ inline Ref(const Ref& Rhs) : m_Ref(Rhs.m_Ref) { m_Ref && m_Ref->AddRef(); }
+ inline Ref(T* Ptr) : m_Ref(Ptr) { m_Ref && m_Ref->AddRef(); }
+ inline ~Ref() { m_Ref && m_Ref->Release(); }
+
+ inline explicit operator bool() const { return m_Ref != nullptr; }
+ inline T* operator->() const { return m_Ref; }
+
+ inline std::strong_ordering operator<=>(const Ref& Rhs) const = default;
+
+ inline Ref& operator=(T* Rhs)
+ {
+ Rhs && Rhs->AddRef();
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs;
+ return *this;
+ }
+ inline Ref& operator=(const Ref& Rhs)
+ {
+ m_Ref && m_Ref->Release();
+ auto Ref = m_Ref = Rhs.m_Ref;
+ Ref && Ref->AddRef();
+ return *this;
+ }
+ inline Ref& operator=(Ref&& Rhs) noexcept
+ {
+ m_Ref && m_Ref->Release();
+ m_Ref = Rhs.m_Ref;
+ Rhs.m_Ref = nullptr;
+ return *this;
+ }
+ inline Ref(Ref&& Rhs) noexcept : m_Ref(Rhs.m_Ref) { Rhs.m_Ref = nullptr; }
+
+private:
+ T* m_Ref = nullptr;
+};
+
+void refcount_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/scopeguard.h b/zencore/include/zencore/scopeguard.h
new file mode 100644
index 000000000..ba8cd3094
--- /dev/null
+++ b/zencore/include/zencore/scopeguard.h
@@ -0,0 +1,33 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <type_traits>
+#include "zencore.h"
+
+namespace zen {
+
+template<typename T>
+class ScopeGuardImpl
+{
+public:
+ inline ScopeGuardImpl(T&& func) : m_guardFunc(func) {}
+ ~ScopeGuardImpl()
+ {
+ if (!m_dismissed)
+ m_guardFunc();
+ }
+
+ void Dismiss() { m_dismissed = true; }
+
+private:
+ bool m_dismissed = false;
+ T m_guardFunc;
+};
+
+template<typename T>
+ScopeGuardImpl<T>
+MakeGuard(T&& fn)
+{
+ return ScopeGuardImpl<T>(std::move(fn));
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/sha1.h b/zencore/include/zencore/sha1.h
new file mode 100644
index 000000000..fc26f442b
--- /dev/null
+++ b/zencore/include/zencore/sha1.h
@@ -0,0 +1,76 @@
+// //////////////////////////////////////////////////////////
+// sha1.h
+// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved.
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+#pragma once
+
+#include <stdint.h>
+#include <compare>
+#include "zencore.h"
+
+namespace zen {
+
+class StringBuilderBase;
+
+struct SHA1
+{
+ uint8_t Hash[20];
+
+ inline auto operator<=>(const SHA1& rhs) const = default;
+
+ static const int StringLength = 40;
+ typedef char String_t[StringLength + 1];
+
+ static SHA1 HashMemory(const void* data, size_t byteCount);
+ static SHA1 FromHexString(const char* string);
+ const char* ToHexString(char* outString /* 40 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static SHA1 Zero; // Initialized to all zeroes
+};
+
+/**
+ * Utility class for computing SHA1 hashes
+ */
+class SHA1Stream
+{
+public:
+ SHA1Stream();
+
+ /** compute SHA1 of a memory block
+
+ \note SHA1 class contains a slightly more convenient helper function for this use case
+ \see SHA1::fromMemory()
+ */
+ SHA1 Compute(const void* data, size_t byteCount);
+
+ /// Begin streaming SHA1 compute (not needed on freshly constructed SHA1Stream instance)
+ void Reset();
+ /// Append another chunk
+ SHA1Stream& Append(const void* data, size_t byteCount);
+ /// Obtain final SHA1 hash. If you wish to reuse the SHA1Stream instance call reset()
+ SHA1 GetHash();
+
+private:
+ void ProcessBlock(const void* data);
+ void ProcessBuffer();
+
+ enum
+ {
+ /// split into 64 byte blocks (=> 512 bits)
+ BlockSize = 512 / 8,
+ HashBytes = 20,
+ HashValues = HashBytes / 4
+ };
+
+ uint64_t m_NumBytes; // size of processed data in bytes
+ size_t m_BufferSize; // valid bytes in m_buffer
+ uint8_t m_Buffer[BlockSize]; // bytes not processed yet
+ uint32_t m_Hash[HashValues];
+};
+
+void sha1_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h
new file mode 100644
index 000000000..c6206f780
--- /dev/null
+++ b/zencore/include/zencore/sharedbuffer.h
@@ -0,0 +1,169 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+#include <zencore/refcount.h>
+
+#include <memory.h>
+
+namespace zen {
+
+class BufferOwner : public RefCounted
+{
+protected:
+ inline BufferOwner(void* DataPtr, uint64_t DataSize, bool IsOwned, BufferOwner* OuterBuffer = nullptr)
+ : m_IsOwned(IsOwned)
+ , m_Data(DataPtr)
+ , m_Size(DataSize)
+ , m_Outer(OuterBuffer)
+ {
+ }
+
+ virtual ~BufferOwner();
+
+ // Ownership is a transitive property, and m_IsOwned currently only flags that this instance is responsible
+ // for managing the allocated memory, so we need to make recursive calls. Could be optimized slightly by
+ // adding a dedicated flag
+ inline bool IsOwned() const
+ {
+ if (m_IsOwned)
+ {
+ return true;
+ }
+ else
+ {
+ return m_Outer && m_Outer->IsOwned();
+ }
+ }
+
+ BufferOwner(const BufferOwner&) = delete;
+ BufferOwner& operator=(const BufferOwner&) = delete;
+
+private:
+ bool m_IsOwned;
+ void* m_Data;
+ uint64_t m_Size;
+ RefPtr<BufferOwner> m_Outer;
+
+ friend class UniqueBuffer;
+ friend class SharedBuffer;
+};
+
+/**
+ * Reference to a memory buffer with a single owner (see std::unique_ptr)
+ */
+class UniqueBuffer
+{
+public:
+ UniqueBuffer(const UniqueBuffer&) = delete;
+ UniqueBuffer& operator=(const UniqueBuffer&) = delete;
+
+ UniqueBuffer() = default;
+ ZENCORE_API explicit UniqueBuffer(BufferOwner* Owner);
+
+ void* GetData() { return m_buffer->m_Data; }
+ const void* GetData() const { return m_buffer->m_Data; }
+ size_t GetSize() const { return m_buffer->m_Size; }
+ operator MutableMemoryView() { return GetView(); }
+ operator MemoryView() const { return MemoryView(m_buffer->m_Data, m_buffer->m_Size); }
+
+ MutableMemoryView GetView() { return MutableMemoryView(m_buffer->m_Data, m_buffer->m_Size); }
+
+ /** Make an uninitialized owned buffer of the specified size. */
+ ZENCORE_API static UniqueBuffer Alloc(uint64_t Size);
+
+ /** Make a non-owned view of the input. */
+ ZENCORE_API static UniqueBuffer MakeView(void* DataPtr, uint64_t Size);
+
+private:
+ RefPtr<BufferOwner> m_buffer;
+
+ friend class SharedBuffer;
+};
+
+/**
+ * Reference to a memory buffer with shared ownership
+ */
+class SharedBuffer
+{
+public:
+ SharedBuffer() = default;
+ ZENCORE_API explicit SharedBuffer(UniqueBuffer&&);
+ inline explicit SharedBuffer(BufferOwner* Owner) : m_buffer(Owner) {}
+
+ void* GetData()
+ {
+ if (m_buffer)
+ {
+ return m_buffer->m_Data;
+ }
+ return nullptr;
+ }
+
+ const void* GetData() const
+ {
+ if (m_buffer)
+ {
+ return m_buffer->m_Data;
+ }
+ return nullptr;
+ }
+
+ size_t GetSize() const
+ {
+ if (m_buffer)
+ {
+ return m_buffer->m_Size;
+ }
+ return 0;
+ }
+
+ ZENCORE_API void MakeOwned();
+ bool IsOwned() const { return m_buffer && m_buffer->IsOwned(); }
+ inline explicit operator bool() const { return m_buffer; }
+ inline bool IsNull() const { return !m_buffer; }
+ inline void Reset() { m_buffer = nullptr; }
+
+ MemoryView GetView() const
+ {
+ if (m_buffer)
+ {
+ return MemoryView(m_buffer->m_Data, m_buffer->m_Size);
+ }
+ else
+ {
+ return MemoryView();
+ }
+ }
+
+ operator MemoryView() const { return GetView(); }
+
+ SharedBuffer& operator=(UniqueBuffer&& Rhs)
+ {
+ m_buffer = std::move(Rhs.m_buffer);
+ return *this;
+ }
+
+ std::strong_ordering operator<=>(const SharedBuffer& Rhs) const = default;
+
+ /** Make a non-owned view of the input */
+ inline static SharedBuffer MakeView(MemoryView View) { return MakeView(View.GetData(), View.GetSize()); }
+ /** Make a non-owned view of the input */
+ ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size);
+ /** Make a non-owned view of the input */
+ ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer Buffer);
+ /** Make am owned clone of the buffer */
+ ZENCORE_API SharedBuffer Clone();
+ /** Make an owned clone of the memory in the input view */
+ ZENCORE_API static SharedBuffer Clone(MemoryView View);
+
+private:
+ RefPtr<BufferOwner> m_buffer;
+};
+
+void sharedbuffer_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/snapshot_manifest.h b/zencore/include/zencore/snapshot_manifest.h
new file mode 100644
index 000000000..95e64773a
--- /dev/null
+++ b/zencore/include/zencore/snapshot_manifest.h
@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+#include <zencore/zencore.h>
+
+#include <filesystem>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace zen {
+
+struct LeafNode
+{
+ uint64_t FileSize = 0;
+ uint64_t FileModifiedTime = 0;
+ zen::IoHash ChunkHash = zen::IoHash::Zero;
+ std::wstring Name;
+};
+
+struct TreeNode
+{
+ std::vector<TreeNode> Children;
+ std::vector<LeafNode> Leaves;
+ std::wstring Name;
+ zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero;
+
+ ZENCORE_API void VisitModifyFiles(std::function<void(LeafNode& node)> func);
+ ZENCORE_API void VisitFiles(std::function<void(const LeafNode& node)> func);
+ ZENCORE_API void Finalize();
+};
+
+struct SnapshotManifest
+{
+ std::string Id;
+ TreeNode Root;
+ zen::BLAKE3 ChunkHash = zen::BLAKE3::Zero;
+
+ ZENCORE_API void finalize();
+};
+
+class InStream;
+class OutStream;
+
+ZENCORE_API void ReadManifest(SnapshotManifest& Manifest, InStream& FromStream);
+ZENCORE_API void WriteManifest(const SnapshotManifest& Manifest, OutStream& ToStream);
+ZENCORE_API void PrintManifest(const SnapshotManifest& Manifest, OutStream& ToStream);
+
+// Translate a user-provided manifest specification into a file path.
+// Supports hashtag syntax to implicitly refer to user documents zenfs folder
+ZENCORE_API std::filesystem::path ManifestSpecToPath(const char* ManifestSpec);
+
+void snapshotmanifest_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h
new file mode 100644
index 000000000..7290fd914
--- /dev/null
+++ b/zencore/include/zencore/stats.h
@@ -0,0 +1,66 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <atomic>
+#include <type_traits>
+#include "zencore.h"
+
+namespace zen {
+
+template<typename T>
+class Gauge
+{
+public:
+ Gauge() : m_value{0} {}
+
+private:
+ T m_value;
+};
+
+class Counter
+{
+public:
+ inline void SetValue(uint64_t Value) { m_count = Value; }
+ inline uint64_t Value() const { return m_count; }
+
+ inline void Increment(int64_t AddValue) { m_count += AddValue; }
+ inline void Decrement(int64_t SubValue) { m_count -= SubValue; }
+ inline void Clear() { m_count = 0; }
+
+private:
+ std::atomic_uint64_t m_count{0};
+};
+
+/// <summary>
+/// Exponential Weighted Moving Average
+/// </summary>
+class EWMA
+{
+public:
+ /// <summary>
+ /// Update EWMA with new measure
+ /// </summary>
+ /// <param name="Alpha">Smoothing factor (between 0 and 1)</param>
+ /// <param name="Interval">Elapsed time since last</param>
+ /// <param name="Count">Value</param>
+ /// <param name="IsInitialUpdate">Whether this is the first update or not</param>
+ void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate);
+ double Rate() const;
+
+private:
+ double m_rate = 0;
+};
+
+/// <summary>
+/// Tracks rate of events over time (i.e requests/sec)
+/// </summary>
+class Meter
+{
+public:
+private:
+};
+
+extern void stats_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h
new file mode 100644
index 000000000..4e8c58382
--- /dev/null
+++ b/zencore/include/zencore/stream.h
@@ -0,0 +1,318 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+#include <zencore/refcount.h>
+#include <zencore/thread.h>
+
+#include <string_view>
+#include <vector>
+
+namespace zen {
+
+/**
+ * Basic byte stream interface
+ *
+ * This is intended as a minimal base class offering only the absolute minimum of functionality.
+ *
+ * IMPORTANT: To better support concurrency, this abstraction offers no "file pointer". Thus
+ * every read or write operation needs to specify the offset from which they wish to read.
+ *
+ * Most client code will likely want to use reader/writer classes like BinaryWriter/BinaryReader
+ *
+ */
+class OutStream : public RefCounted
+{
+public:
+ virtual void Write(const void* Data, size_t ByteCount, uint64_t Offset) = 0;
+ virtual void Flush() = 0;
+};
+
+class InStream : public RefCounted
+{
+public:
+ virtual void Read(void* DataPtr, size_t ByteCount, uint64_t Offset) = 0;
+ virtual uint64_t Size() const = 0;
+};
+
+/**
+ * Stream which writes into a growing memory buffer
+ */
+class MemoryOutStream : public OutStream
+{
+public:
+ MemoryOutStream() = default;
+ ~MemoryOutStream() = default;
+
+ virtual void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset) override;
+ virtual void Flush() override;
+ inline const uint8_t* Data() const { return m_Buffer.data(); }
+ inline uint64_t Size() const { return m_Buffer.size(); }
+
+private:
+ RwLock m_Lock;
+ std::vector<uint8_t> m_Buffer;
+};
+
+inline MemoryView
+MakeMemoryView(const MemoryOutStream& Stream)
+{
+ return MemoryView(Stream.Data(), Stream.Size());
+}
+
+/**
+ * Stream which reads from a memory buffer
+ */
+class MemoryInStream : public InStream
+{
+public:
+ MemoryInStream(const void* Buffer, size_t Size);
+ MemoryInStream(MemoryView View) : MemoryInStream(View.GetData(), View.GetSize()) {}
+ ~MemoryInStream() = default;
+
+ virtual void Read(void* DataPtr, size_t ByteCount, uint64_t ReadOffset) override;
+ virtual uint64_t Size() const override { return m_Buffer.size(); }
+ inline const uint8_t* Data() const { return m_Buffer.data(); }
+
+private:
+ RwLock m_Lock;
+ std::vector<uint8_t> m_Buffer;
+};
+
+/**
+ * Binary stream writer
+ */
+
+class BinaryWriter
+{
+public:
+ inline BinaryWriter(OutStream& Stream) : m_Stream(&Stream) {}
+ ~BinaryWriter() = default;
+
+ inline void Write(const void* DataPtr, size_t ByteCount)
+ {
+ m_Stream->Write(DataPtr, ByteCount, m_Offset);
+ m_Offset += ByteCount;
+ }
+
+ uint64_t CurrentOffset() const { return m_Offset; }
+
+private:
+ RefPtr<OutStream> m_Stream;
+ uint64_t m_Offset = 0;
+};
+
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, bool Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int8_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int16_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int32_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, int64_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint8_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint16_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint32_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+inline BinaryWriter&
+operator<<(BinaryWriter& Writer, uint64_t Value)
+{
+ Writer.Write(&Value, sizeof Value);
+ return Writer;
+}
+
+/**
+ * Binary stream reader
+ */
+
+class BinaryReader
+{
+public:
+ inline BinaryReader(InStream& Stream) : m_Stream(&Stream) {}
+ ~BinaryReader() = default;
+
+ inline void Read(void* DataPtr, size_t ByteCount)
+ {
+ m_Stream->Read(DataPtr, ByteCount, m_Offset);
+ m_Offset += ByteCount;
+ }
+
+ void Seek(uint64_t Offset)
+ {
+ ZEN_ASSERT(Offset <= m_Stream->Size());
+ m_Offset = Offset;
+ }
+
+ void Skip(uint64_t SkipOffset)
+ {
+ ZEN_ASSERT((m_Offset + SkipOffset) <= m_Stream->Size());
+ m_Offset += SkipOffset;
+ }
+
+ inline uint64_t CurrentOffset() const { return m_Offset; }
+ inline uint64_t AvailableBytes() const { return m_Stream->Size() - m_Offset; }
+
+private:
+ RefPtr<InStream> m_Stream;
+ uint64_t m_Offset = 0;
+};
+
+inline BinaryReader&
+operator>>(BinaryReader& Reader, bool& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int8_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int16_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int32_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, int64_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint8_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint16_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint32_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+inline BinaryReader&
+operator>>(BinaryReader& Reader, uint64_t& Value)
+{
+ Reader.Read(&Value, sizeof Value);
+ return Reader;
+}
+
+/**
+ * Text stream writer
+ */
+
+class TextWriter
+{
+public:
+ ZENCORE_API TextWriter(OutStream& Stream);
+ ZENCORE_API ~TextWriter();
+
+ ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount);
+ ZENCORE_API void Writef(const char* FormatString, ...);
+
+ inline uint64_t CurrentOffset() const { return m_CurrentOffset; }
+
+private:
+ RefPtr<OutStream> m_Stream;
+ uint64_t m_CurrentOffset = 0;
+};
+
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const char* Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const std::string_view& Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, bool Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int8_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int16_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int32_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int64_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint8_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint16_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint32_t Value);
+ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint64_t Value);
+
+class IndentTextWriter : public TextWriter
+{
+public:
+ ZENCORE_API IndentTextWriter(OutStream& stream);
+ ZENCORE_API ~IndentTextWriter();
+
+ ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount) override;
+
+ inline void Indent(int Amount) { m_IndentAmount += Amount; }
+
+ struct Scope
+ {
+ Scope(IndentTextWriter& Outer, int IndentAmount = 2) : m_Outer(Outer), m_IndentAmount(IndentAmount)
+ {
+ m_Outer.Indent(IndentAmount);
+ }
+
+ ~Scope() { m_Outer.Indent(-m_IndentAmount); }
+
+ private:
+ IndentTextWriter& m_Outer;
+ int m_IndentAmount;
+ };
+
+private:
+ int m_IndentAmount = 0;
+ int m_LineCursor = 0;
+ char m_LineBuffer[2048];
+};
+
+void stream_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/streamutil.h b/zencore/include/zencore/streamutil.h
new file mode 100644
index 000000000..190cd18eb
--- /dev/null
+++ b/zencore/include/zencore/streamutil.h
@@ -0,0 +1,118 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <fmt/format.h>
+#include <zencore/string.h>
+#include <string>
+#include <string_view>
+
+#include "blake3.h"
+#include "iohash.h"
+#include "sha1.h"
+#include "stream.h"
+
+namespace zen {
+
+ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::string_view& value);
+ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::string& value);
+
+ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::wstring_view& value);
+ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::wstring& value);
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const std::wstring_view& value);
+
+inline BinaryWriter&
+operator<<(BinaryWriter& writer, const SHA1& value)
+{
+ writer.Write(value.Hash, sizeof value.Hash);
+ return writer;
+}
+inline BinaryReader&
+operator>>(BinaryReader& reader, SHA1& value)
+{
+ reader.Read(value.Hash, sizeof value.Hash);
+ return reader;
+}
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const zen::SHA1& value);
+
+inline BinaryWriter&
+operator<<(BinaryWriter& writer, const BLAKE3& value)
+{
+ writer.Write(value.Hash, sizeof value.Hash);
+ return writer;
+}
+inline BinaryReader&
+operator>>(BinaryReader& reader, BLAKE3& value)
+{
+ reader.Read(value.Hash, sizeof value.Hash);
+ return reader;
+}
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const BLAKE3& value);
+
+inline BinaryWriter&
+operator<<(BinaryWriter& writer, const IoHash& value)
+{
+ writer.Write(value.Hash, sizeof value.Hash);
+ return writer;
+}
+inline BinaryReader&
+operator>>(BinaryReader& reader, IoHash& value)
+{
+ reader.Read(value.Hash, sizeof value.Hash);
+ return reader;
+}
+ZENCORE_API TextWriter& operator<<(TextWriter& writer, const IoHash& value);
+
+} // namespace zen
+
+//////////////////////////////////////////////////////////////////////////
+
+template<>
+struct fmt::formatter<zen::IoHash>
+{
+ constexpr auto parse(format_parse_context& ctx)
+ {
+ // Parse the presentation format and store it in the formatter:
+ auto it = ctx.begin(), end = ctx.end();
+
+ // Check if reached the end of the range:
+ if (it != end && *it != '}')
+ throw format_error("invalid format");
+
+ // Return an iterator past the end of the parsed range:
+ return it;
+ }
+
+ template<typename FormatContext>
+ auto format(const zen::IoHash& h, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<48> String;
+ h.ToHexString(String);
+ return format_to(ctx.out(), std::string_view(String));
+ }
+};
+
+template<>
+struct fmt::formatter<zen::BLAKE3>
+{
+ constexpr auto parse(format_parse_context& ctx)
+ {
+ // Parse the presentation format and store it in the formatter:
+ auto it = ctx.begin(), end = ctx.end();
+
+ // Check if reached the end of the range:
+ if (it != end && *it != '}')
+ throw format_error("invalid format");
+
+ // Return an iterator past the end of the parsed range:
+ return it;
+ }
+
+ template<typename FormatContext>
+ auto format(const zen::BLAKE3& h, FormatContext& ctx)
+ {
+ zen::ExtendableStringBuilder<80> String;
+ h.ToHexString(String);
+ return format_to(ctx.out(), std::string_view(String));
+ }
+};
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
new file mode 100644
index 000000000..d7727ca08
--- /dev/null
+++ b/zencore/include/zencore/string.h
@@ -0,0 +1,595 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "intmath.h"
+#include "zencore.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <charconv>
+#include <codecvt>
+#include <concepts>
+#include <optional>
+#include <span>
+#include <string_view>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+inline bool
+StringEquals(const char8_t* s1, const char* s2)
+{
+ return strcmp(reinterpret_cast<const char*>(s1), s2) == 0;
+}
+
+inline bool
+StringEquals(const char* s1, const char* s2)
+{
+ return strcmp(s1, s2) == 0;
+}
+
+inline size_t
+StringLength(const char* str)
+{
+ return strlen(str);
+}
+
+inline bool
+StringEquals(const wchar_t* s1, const wchar_t* s2)
+{
+ return wcscmp(s1, s2) == 0;
+}
+
+inline size_t
+StringLength(const wchar_t* str)
+{
+ return wcslen(str);
+}
+
+//////////////////////////////////////////////////////////////////////////
+// File name helpers
+//
+
+ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr);
+
+//////////////////////////////////////////////////////////////////////////
+// Text formatting of numbers
+//
+
+ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num);
+ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num);
+
+struct TextNumBase
+{
+ inline const char* c_str() const { return m_Buffer; }
+ inline operator std::string_view() const { return std::string_view(m_Buffer); }
+
+protected:
+ char m_Buffer[24];
+};
+
+struct IntNum : public TextNumBase
+{
+ inline IntNum(std::unsigned_integral auto Number) { ToString(m_Buffer, uint64_t(Number)); }
+ inline IntNum(std::signed_integral auto Number) { ToString(m_Buffer, int64_t(Number)); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Quick-and-dirty string builder. Good enough for me, but contains traps
+// and not-quite-ideal behaviour especially when mixing character types etc
+//
+
+template<typename C>
+class StringBuilderImpl
+{
+public:
+ StringBuilderImpl() = default;
+ ZENCORE_API ~StringBuilderImpl();
+
+ StringBuilderImpl(const StringBuilderImpl&) = delete;
+ StringBuilderImpl(const StringBuilderImpl&&) = delete;
+ const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete;
+ const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete;
+
+ StringBuilderImpl& Append(C OneChar)
+ {
+ EnsureCapacity(1);
+
+ *m_CurPos++ = OneChar;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const std::string_view& String)
+ {
+ const size_t len = String.size();
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = String[i];
+
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const std::u8string_view& String)
+ {
+ const size_t len = String.size();
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = String[i];
+
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAscii(const char* NulTerminatedString)
+ {
+ size_t StringLen = StringLength(NulTerminatedString);
+
+ return AppendAscii({NulTerminatedString, StringLen});
+ }
+
+ inline StringBuilderImpl& Append(const char8_t* NulTerminatedString)
+ {
+ // This is super hacky and not fully functional - needs better
+ // solution
+ if constexpr (sizeof(C) == 1)
+ {
+ size_t len = StringLength((const char*)NulTerminatedString);
+
+ EnsureCapacity(len);
+
+ for (size_t i = 0; i < len; ++i)
+ m_CurPos[i] = C(NulTerminatedString[i]);
+
+ m_CurPos += len;
+ }
+ else
+ {
+ ZEN_NOT_IMPLEMENTED();
+ }
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendAsciiRange(const char* BeginString, const char* EndString)
+ {
+ EnsureCapacity(EndString - BeginString);
+
+ while (BeginString != EndString)
+ *m_CurPos++ = *BeginString++;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const C* NulTerminatedString)
+ {
+ size_t Len = StringLength(NulTerminatedString);
+
+ EnsureCapacity(Len);
+ memcpy(m_CurPos, NulTerminatedString, Len * sizeof(C));
+ m_CurPos += Len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const C* NulTerminatedString, size_t MaxChars)
+ {
+ size_t len = Min(MaxChars, StringLength(NulTerminatedString));
+
+ EnsureCapacity(len);
+ memcpy(m_CurPos, NulTerminatedString, len * sizeof(C));
+ m_CurPos += len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& AppendRange(const C* BeginString, const C* EndString)
+ {
+ size_t Len = EndString - BeginString;
+
+ EnsureCapacity(Len);
+ memcpy(m_CurPos, BeginString, Len * sizeof(C));
+ m_CurPos += Len;
+
+ return *this;
+ }
+
+ inline StringBuilderImpl& Append(const std::basic_string_view<C>& String)
+ {
+ return AppendRange(String.data(), String.data() + String.size());
+ }
+
+ inline const C* c_str() const
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline C* Data()
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline const C* Data() const
+ {
+ EnsureNulTerminated();
+ return m_Base;
+ }
+
+ inline size_t Size() const { return m_CurPos - m_Base; }
+ inline bool IsDynamic() const { return m_IsDynamic; }
+ inline void Reset() { m_CurPos = m_Base; }
+
+ inline StringBuilderImpl& operator<<(uint64_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int64_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint32_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int32_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint16_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int16_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(uint8_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+ inline StringBuilderImpl& operator<<(int8_t n)
+ {
+ IntNum Str(n);
+ return AppendAscii(Str);
+ }
+
+ inline StringBuilderImpl& operator<<(const char* str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(const std::string_view str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(const std::u8string_view str) { return AppendAscii(str); }
+ inline StringBuilderImpl& operator<<(bool v)
+ {
+ using namespace std::literals;
+ if (v)
+ {
+ return AppendAscii("true"sv);
+ }
+ return AppendAscii("false"sv);
+ }
+
+protected:
+ inline void Init(C* Base, size_t Capacity)
+ {
+ m_Base = m_CurPos = Base;
+ m_End = Base + Capacity;
+ }
+
+ inline void EnsureNulTerminated() const { *m_CurPos = '\0'; }
+
+ inline void EnsureCapacity(size_t ExtraRequired)
+ {
+ // precondition: we know the current buffer has enough capacity
+ // for the existing string including NUL terminator
+
+ if ((m_CurPos + ExtraRequired) < m_End)
+ return;
+
+ Extend(ExtraRequired);
+ }
+
+ ZENCORE_API void Extend(size_t ExtraCapacity);
+ ZENCORE_API void* AllocBuffer(size_t ByteCount);
+ ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount);
+
+ ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception
+
+ C* m_Base;
+ C* m_CurPos;
+ C* m_End;
+ bool m_IsDynamic = false;
+ bool m_IsExtendable = false;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+extern template StringBuilderImpl<char>;
+
+class StringBuilderBase : public StringBuilderImpl<char>
+{
+public:
+ inline StringBuilderBase(char* bufferPointer, size_t bufferCapacity) { Init(bufferPointer, bufferCapacity); }
+ inline ~StringBuilderBase() = default;
+
+ // Note that we don't need a terminator for the string_view so we avoid calling data() here
+ inline operator std::string_view() const { return std::string_view(m_Base, m_CurPos - m_Base); }
+ inline std::string_view ToView() const { return std::string_view(m_Base, m_CurPos - m_Base); }
+ inline std::string ToString() const { return std::string{Data(), Size()}; }
+
+ inline void AppendCodepoint(uint32_t cp)
+ {
+ if (cp < 0x80) // one octet
+ {
+ Append(static_cast<char8_t>(cp));
+ }
+ else if (cp < 0x800)
+ {
+ EnsureCapacity(2); // two octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 6) | 0xc0);
+ m_CurPos[1] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 2;
+ }
+ else if (cp < 0x10000)
+ {
+ EnsureCapacity(3); // three octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 12) | 0xe0);
+ m_CurPos[1] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80);
+ m_CurPos[2] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 3;
+ }
+ else
+ {
+ EnsureCapacity(4); // four octets
+ m_CurPos[0] = static_cast<char8_t>((cp >> 18) | 0xf0);
+ m_CurPos[1] = static_cast<char8_t>(((cp >> 12) & 0x3f) | 0x80);
+ m_CurPos[2] = static_cast<char8_t>(((cp >> 6) & 0x3f) | 0x80);
+ m_CurPos[3] = static_cast<char8_t>((cp & 0x3f) | 0x80);
+ m_CurPos += 4;
+ }
+ }
+};
+
+template<size_t N>
+class StringBuilder : public StringBuilderBase
+{
+public:
+ inline StringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) {}
+ inline ~StringBuilder() = default;
+
+private:
+ char m_StringBuffer[N];
+};
+
+template<size_t N>
+class ExtendableStringBuilder : public StringBuilderBase
+{
+public:
+ inline ExtendableStringBuilder() : StringBuilderBase(m_StringBuffer, sizeof m_StringBuffer) { m_IsExtendable = true; }
+ inline ~ExtendableStringBuilder() = default;
+
+private:
+ char m_StringBuffer[N];
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+extern template StringBuilderImpl<wchar_t>;
+
+class WideStringBuilderBase : public StringBuilderImpl<wchar_t>
+{
+public:
+ inline WideStringBuilderBase(wchar_t* BufferPointer, size_t BufferCapacity) { Init(BufferPointer, BufferCapacity); }
+ inline ~WideStringBuilderBase() = default;
+
+ inline operator std::wstring_view() const { return std::wstring_view{Data(), Size()}; }
+ inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; }
+ inline std::wstring toString() const { return std::wstring{Data(), Size()}; }
+
+ inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); }
+ inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); }
+ using StringBuilderImpl:: operator<<;
+};
+
+template<size_t N>
+class WideStringBuilder : public WideStringBuilderBase
+{
+public:
+ inline WideStringBuilder() : WideStringBuilderBase(m_Buffer, N) {}
+ ~WideStringBuilder() = default;
+
+private:
+ wchar_t m_Buffer[N];
+};
+
+template<size_t N>
+class ExtendableWideStringBuilder : public WideStringBuilderBase
+{
+public:
+ inline ExtendableWideStringBuilder() : WideStringBuilderBase(m_Buffer, N) { m_IsExtendable = true; }
+ ~ExtendableWideStringBuilder() = default;
+
+private:
+ wchar_t m_Buffer[N];
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out);
+void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out);
+void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out);
+std::wstring Utf8ToWide(const std::string_view& wstr);
+
+void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out);
+std::string WideToUtf8(const wchar_t* wstr);
+void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out);
+void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out);
+std::string WideToUtf8(const std::wstring_view Wstr);
+
+/// <summary>
+/// Parse hex string into a byte buffer
+/// </summary>
+/// <param name="string">Input string</param>
+/// <param name="characterCount">Number of characters in string</param>
+/// <param name="outPtr">Pointer to output buffer</param>
+/// <returns>true if the input consisted of all valid hexadecimal characters</returns>
+
+inline bool
+ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
+{
+ ZEN_ASSERT((CharacterCount & 1) == 0);
+
+ auto char2nibble = [](char c) {
+ uint8_t c8 = uint8_t(c - '0');
+
+ if (c8 < 10)
+ return c8;
+
+ c8 -= 'A' - '0' - 10;
+
+ if (c8 < 16)
+ return c8;
+
+ c8 -= 'a' - 'A';
+
+ if (c8 < 16)
+ return c8;
+
+ return uint8_t(0xff);
+ };
+
+ uint8_t allBits = 0;
+
+ while (CharacterCount)
+ {
+ uint8_t n0 = char2nibble(InputString[0]);
+ uint8_t n1 = char2nibble(InputString[1]);
+
+ allBits |= n0 | n1;
+
+ *OutPtr = (n0 << 4) | n1;
+
+ OutPtr += 1;
+ InputString += 2;
+ CharacterCount -= 2;
+ }
+
+ return (allBits & 0x80) == 0;
+}
+
+inline void
+ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString)
+{
+ const char hexchars[] = "0123456789abcdef";
+
+ while (ByteCount--)
+ {
+ uint8_t byte = *InputData++;
+
+ *OutString++ = hexchars[byte >> 4];
+ *OutString++ = hexchars[byte & 15];
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Format numbers for humans
+//
+
+ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer);
+ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer);
+ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer);
+ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer);
+ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer);
+
+struct NiceBase
+{
+ inline const char* c_str() const { return m_Buffer; }
+ inline operator std::string_view() const { return std::string_view(m_Buffer); }
+
+protected:
+ char m_Buffer[16];
+};
+
+struct NiceNum : public NiceBase
+{
+ inline NiceNum(uint64_t Num) { NiceNumToBuffer(Num, m_Buffer); }
+};
+
+struct NiceBytes : public NiceBase
+{
+ inline NiceBytes(uint64_t Num) { NiceBytesToBuffer(Num, m_Buffer); }
+};
+
+struct NiceByteRate : public NiceBase
+{
+ inline NiceByteRate(uint64_t Bytes, uint64_t TimeMilliseconds) { NiceByteRateToBuffer(Bytes, TimeMilliseconds, m_Buffer); }
+};
+
+struct NiceLatencyNs : public NiceBase
+{
+ inline NiceLatencyNs(uint64_t Milliseconds) { NiceLatencyNsToBuffer(Milliseconds, m_Buffer); }
+};
+
+struct NiceTimeSpanMs : public NiceBase
+{
+ inline NiceTimeSpanMs(uint64_t Milliseconds) { NiceTimeSpanMsToBuffer(Milliseconds, m_Buffer); }
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+inline std::string
+NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B")
+{
+ char Buffer[32];
+
+ if (DurationMilliseconds)
+ {
+ NiceNumToBuffer(Num * 1000 / DurationMilliseconds, Buffer);
+ }
+ else
+ {
+ strcpy_s(Buffer, "0");
+ }
+
+ strcat_s(Buffer, Unit);
+ strcat_s(Buffer, "/s");
+
+ return Buffer;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template<std::integral T>
+std::optional<T>
+ParseInt(const std::string_view& Input)
+{
+ T Out;
+ const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out);
+ if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range)
+ {
+ return std::nullopt;
+ }
+ return Out;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void string_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h
new file mode 100644
index 000000000..48afad33f
--- /dev/null
+++ b/zencore/include/zencore/thread.h
@@ -0,0 +1,118 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+namespace zen {
+
+/**
+ * Reader-writer lock
+ *
+ * - A single thread may hold an exclusive lock at any given moment
+ *
+ * - Multiple threads may hold shared locks, but only if no thread has
+ * acquired an exclusive lock
+ */
+class RwLock
+{
+public:
+ ZENCORE_API void AcquireShared();
+ ZENCORE_API void ReleaseShared();
+
+ ZENCORE_API void AcquireExclusive();
+ ZENCORE_API void ReleaseExclusive();
+
+ struct SharedLockScope
+ {
+ SharedLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireShared(); }
+ ~SharedLockScope() { m_Lock.ReleaseShared(); }
+
+ private:
+ RwLock& m_Lock;
+ };
+
+ struct ExclusiveLockScope
+ {
+ ExclusiveLockScope(RwLock& lock) : m_Lock(lock) { m_Lock.AcquireExclusive(); }
+ ~ExclusiveLockScope() { m_Lock.ReleaseExclusive(); }
+
+ private:
+ RwLock& m_Lock;
+ };
+
+private:
+ void* m_Srw = nullptr;
+};
+
+/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore')
+ */
+class Event
+{
+public:
+ ZENCORE_API Event();
+ ZENCORE_API ~Event();
+
+ Event(Event&& Rhs) : m_EventHandle(Rhs.m_EventHandle) { Rhs.m_EventHandle = nullptr; }
+
+ Event(const Event& Rhs) = delete;
+ Event& operator=(const Event& Rhs) = delete;
+
+ inline Event& operator=(Event&& Rhs)
+ {
+ m_EventHandle = Rhs.m_EventHandle;
+ Rhs.m_EventHandle = nullptr;
+ return *this;
+ }
+
+ ZENCORE_API void Set();
+ ZENCORE_API void Reset();
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+
+protected:
+ explicit Event(void* EventHandle) : m_EventHandle(EventHandle) {}
+
+ void* m_EventHandle = nullptr;
+};
+
+/** Basic abstraction of an IPC mechanism (aka 'binary semaphore')
+ */
+class NamedEvent : public Event
+{
+public:
+ ZENCORE_API explicit NamedEvent(std::string_view EventName);
+ ZENCORE_API explicit NamedEvent(std::u8string_view EventName);
+};
+
+/** Basic process abstraction
+ */
+class Process
+{
+public:
+ ZENCORE_API Process();
+
+ Process(const Process&) = delete;
+ Process& operator=(const Process&) = delete;
+
+ ZENCORE_API ~Process();
+
+ ZENCORE_API void Initialize(int Pid);
+ ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle
+ ZENCORE_API bool IsRunning() const;
+ ZENCORE_API bool IsValid() const;
+ ZENCORE_API bool Wait(int TimeoutMs = -1);
+ ZENCORE_API void Terminate(int ExitCode);
+ inline int Pid() const { return m_Pid; }
+
+private:
+ void* m_ProcessHandle = nullptr;
+ int m_Pid = 0;
+};
+
+ZENCORE_API bool IsProcessRunning(int pid);
+
+ZENCORE_API void Sleep(int ms);
+
+void thread_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/timer.h b/zencore/include/zencore/timer.h
new file mode 100644
index 000000000..c9122eb44
--- /dev/null
+++ b/zencore/include/zencore/timer.h
@@ -0,0 +1,41 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <intrin.h>
+#include <stdint.h>
+#include "zencore.h"
+
+namespace zen {
+
+// High frequency timers
+
+ZENCORE_API uint64_t GetHifreqTimerValue();
+ZENCORE_API uint64_t GetHifreqTimerFrequency();
+ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init
+
+class Stopwatch
+{
+public:
+ Stopwatch() : m_StartValue(GetHifreqTimerValue()) {}
+
+ inline uint64_t getElapsedTimeMs() { return (GetHifreqTimerValue() - m_StartValue) * 1000 / GetHifreqTimerFrequency(); }
+
+ inline void reset() { m_StartValue = GetHifreqTimerValue(); }
+
+private:
+ uint64_t m_StartValue;
+};
+
+// CPU timers
+
+inline uint64_t
+GetCpuTimerValue()
+{
+ unsigned int foo;
+ return __rdtscp(&foo);
+}
+
+void timer_forcelink(); // internal
+
+} // namespace zen
diff --git a/zencore/include/zencore/trace.h b/zencore/include/zencore/trace.h
new file mode 100644
index 000000000..191ce4a3a
--- /dev/null
+++ b/zencore/include/zencore/trace.h
@@ -0,0 +1,91 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <inttypes.h>
+#include "zencore.h"
+
+#pragma section("trace_events", read)
+#define U_TRACE_DECL __declspec(allocate("trace_events"))
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+struct TraceSite
+{
+ const char* sourceFile;
+ const uint32_t sourceLine;
+ const uint32_t flags;
+};
+
+struct TraceEvent
+{
+ const TraceSite* site;
+ ThreadId_t threadId;
+ const char* message;
+};
+
+enum TraceFlags
+{
+ kTrace_Debug = 1 << 0,
+ kTrace_Info = 1 << 1,
+ kTrace_Warn = 1 << 2,
+ kTrace_Error = 1 << 3,
+ kTrace_Fatal = 1 << 4,
+
+ kTrace_Trace = 1 << 7,
+};
+
+class Tracer
+{
+public:
+ void Log(const TraceEvent& e);
+
+ __forceinline uint32_t Accept(const TraceSite& e) const { return (m_acceptFlags & e.flags); }
+
+private:
+ uint32_t m_acceptFlags = ~0u;
+};
+
+ZENCORE_API extern Tracer g_globalTracer;
+
+/** Trace event handler
+ */
+class TraceHandler
+{
+public:
+ virtual void Trace(const TraceEvent& e) = 0;
+
+private:
+};
+
+ZENCORE_API static void TraceBroadcast(const TraceEvent& e);
+
+void trace_forcelink(); // internal
+
+} // namespace zen
+
+__forceinline zen::Tracer&
+CurrentTracer()
+{
+ return zen::g_globalTracer;
+}
+
+#define U_LOG_GENERIC(msg, flags) \
+ do \
+ { \
+ zen::Tracer& t = CurrentTracer(); \
+ static U_TRACE_DECL constexpr zen::TraceSite traceSite{__FILE__, __LINE__, flags}; \
+ const zen::TraceEvent traceEvent = {&traceSite, 0u, msg}; \
+ if (t.Accept(traceSite)) \
+ t.Log(traceEvent); \
+ } while (false)
+
+//////////////////////////////////////////////////////////////////////////
+
+#define U_LOG_DEBUG(msg) U_LOG_GENERIC(msg, zen::kTrace_Debug)
+#define U_LOG_INFO(msg) U_LOG_GENERIC(msg, zen::kTrace_Info)
+#define U_LOG_WARN(msg) U_LOG_GENERIC(msg, zen::kTrace_Warn)
+#define U_LOG_ERROR(msg) U_LOG_GENERIC(msg, zen::kTrace_Error)
+#define U_LOG_FATAL(msg) U_LOG_GENERIC(msg, zen::kTrace_Fatal)
diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h
new file mode 100644
index 000000000..a793b160a
--- /dev/null
+++ b/zencore/include/zencore/uid.h
@@ -0,0 +1,78 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+#include <compare>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/** Object identifier
+
+ Can be used as a GUID essentially, but is more compact (12 bytes) and as such
+ is more susceptible to collisions than a 16-byte GUID but also I don't expect
+ the population to be large so in practice the risk should be minimal due to
+ how the identifiers work.
+
+ Similar in spirit to MongoDB ObjectId
+
+ When serialized, object identifiers generated in a given session in sequence
+ will sort in chronological order since the timestamp is in the MSB in big
+ endian format. This makes it suitable as a database key since most indexing
+ structures work better when keys are inserted in lexicographically
+ increasing order.
+
+ The current layout is basically:
+
+ |----------------|----------------|----------------|
+ | timestamp | serial # | run id |
+ |----------------|----------------|----------------|
+ MSB LSB
+
+ - Timestamp is a unsigned 32-bit value (seconds since Jan 1 1970)
+ - Serial # is another unsigned 32-bit value which is assigned a (strong)
+ random number at initialization time which is incremented when a new Oid
+ is generated
+ - The run id is generated from a strong random number generator
+ at initialization time and stays fixed for the duration of the program
+
+ Timestamp and serial are stored in memory in such a way that they can be
+ ordered lexicographically. I.e they are in big-endian byte order.
+
+ */
+
+struct Oid
+{
+ static const int StringLength = 24;
+ typedef char String_t[StringLength + 1];
+
+ static void Initialize();
+ [[nodiscard]] static Oid NewOid();
+
+ const Oid& Generate();
+ [[nodiscard]] static Oid FromHexString(const std::string_view String);
+ StringBuilderBase& ToString(StringBuilderBase& OutString) const;
+
+ auto operator<=>(const Oid& rhs) const = default;
+
+ static const Oid Zero; // Min (can be used to signify a "null" value, or for open range queries)
+ static const Oid Max; // Max (can be used for open range queries)
+
+ struct Hasher
+ {
+ size_t operator()(const Oid& id) const
+ {
+ const size_t seed = id.OidBits[0];
+ return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32);
+ }
+ };
+
+ // You should not assume anything about these words
+ uint32_t OidBits[3];
+};
+
+extern void uid_forcelink();
+
+} // namespace zen
diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h
new file mode 100644
index 000000000..0c40dd66b
--- /dev/null
+++ b/zencore/include/zencore/varint.h
@@ -0,0 +1,255 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "intmath.h"
+
+namespace zen {
+
+// Variable-Length Integer Encoding
+//
+// ZigZag encoding is used to convert signed integers into unsigned integers in a way that allows
+// integers with a small magnitude to have a smaller encoded representation.
+//
+// An unsigned integer is encoded into 1-9 bytes based on its magnitude. The first byte indicates
+// how many additional bytes are used by the number of leading 1-bits that it has. The additional
+// bytes are stored in big endian order, and the most significant bits of the value are stored in
+// the remaining bits in the first byte. The encoding of the first byte allows the reader to skip
+// over the encoded integer without consuming its bytes individually.
+//
+// Encoded unsigned integers sort the same in a byte-wise comparison as when their decoded values
+// are compared. The same property does not hold for signed integers due to ZigZag encoding.
+//
+// 32-bit inputs encode to 1-5 bytes.
+// 64-bit inputs encode to 1-9 bytes.
+//
+// 0x0000'0000'0000'0000 - 0x0000'0000'0000'007f : 0b0_______ 1 byte
+// 0x0000'0000'0000'0080 - 0x0000'0000'0000'3fff : 0b10______ 2 bytes
+// 0x0000'0000'0000'4000 - 0x0000'0000'001f'ffff : 0b110_____ 3 bytes
+// 0x0000'0000'0020'0000 - 0x0000'0000'0fff'ffff : 0b1110____ 4 bytes
+// 0x0000'0000'1000'0000 - 0x0000'0007'ffff'ffff : 0b11110___ 5 bytes
+// 0x0000'0008'0000'0000 - 0x0000'03ff'ffff'ffff : 0b111110__ 6 bytes
+// 0x0000'0400'0000'0000 - 0x0001'ffff'ffff'ffff : 0b1111110_ 7 bytes
+// 0x0002'0000'0000'0000 - 0x00ff'ffff'ffff'ffff : 0b11111110 8 bytes
+// 0x0100'0000'0000'0000 - 0xffff'ffff'ffff'ffff : 0b11111111 9 bytes
+//
+// Encoding Examples
+// -42 => ZigZag => 0x53 => 0x53
+// 42 => ZigZag => 0x54 => 0x54
+// 0x1 => 0x01
+// 0x12 => 0x12
+// 0x123 => 0x81 0x23
+// 0x1234 => 0x92 0x34
+// 0x12345 => 0xc1 0x23 0x45
+// 0x123456 => 0xd2 0x34 0x56
+// 0x1234567 => 0xe1 0x23 0x45 0x67
+// 0x12345678 => 0xf0 0x12 0x34 0x56 0x78
+// 0x123456789 => 0xf1 0x23 0x45 0x67 0x89
+// 0x123456789a => 0xf8 0x12 0x34 0x56 0x78 0x9a
+// 0x123456789ab => 0xfb 0x23 0x45 0x67 0x89 0xab
+// 0x123456789abc => 0xfc 0x12 0x34 0x56 0x78 0x9a 0xbc
+// 0x123456789abcd => 0xfd 0x23 0x45 0x67 0x89 0xab 0xcd
+// 0x123456789abcde => 0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde
+// 0x123456789abcdef => 0xff 0x01 0x23 0x45 0x67 0x89 0xab 0xcd 0xef
+// 0x123456789abcdef0 => 0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0
+
+/**
+ * Measure the length in bytes (1-9) of an encoded variable-length integer.
+ *
+ * @param InData A variable-length encoding of an (signed or unsigned) integer.
+ * @return The number of bytes used to encode the integer, in the range 1-9.
+ */
+inline uint32_t
+MeasureVarUInt(const void* InData)
+{
+ return CountLeadingZeros(uint8_t(~*static_cast<const uint8_t*>(InData))) - 23;
+}
+
+/** Measure the length in bytes (1-9) of an encoded variable-length integer. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(const void* InData)
+{
+ return MeasureVarUInt(InData);
+}
+
+/** Measure the number of bytes (1-5) required to encode the 32-bit input. */
+inline uint32_t
+MeasureVarUInt(uint32_t InValue)
+{
+ return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1);
+}
+
+/** Measure the number of bytes (1-9) required to encode the 64-bit input. */
+inline uint32_t
+MeasureVarUInt(uint64_t InValue)
+{
+ return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9));
+}
+
+/** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(int32_t InValue)
+{
+ return MeasureVarUInt(uint32_t((InValue >> 31) ^ (InValue << 1)));
+}
+
+/** Measure the number of bytes (1-9) required to encode the 64-bit input. \see \ref MeasureVarUInt */
+inline uint32_t
+MeasureVarInt(int64_t InValue)
+{
+ return MeasureVarUInt(uint64_t((InValue >> 63) ^ (InValue << 1)));
+}
+
+/**
+ * Read a variable-length unsigned integer.
+ *
+ * @param InData A variable-length encoding of an unsigned integer.
+ * @param OutByteCount The number of bytes consumed from the input.
+ * @return An unsigned integer.
+ */
+inline uint64_t
+ReadVarUInt(const void* InData, uint32_t& OutByteCount)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InData);
+ OutByteCount = ByteCount;
+
+ const uint8_t* InBytes = static_cast<const uint8_t*>(InData);
+ uint64_t Value = *InBytes++ & uint8_t(0xff >> ByteCount);
+ switch (ByteCount - 1)
+ {
+ case 8:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 7:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 6:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 5:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 4:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 3:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 2:
+ Value <<= 8;
+ Value |= *InBytes++;
+ case 1:
+ Value <<= 8;
+ Value |= *InBytes++;
+ default:
+ return Value;
+ }
+}
+
+/**
+ * Read a variable-length signed integer.
+ *
+ * @param InData A variable-length encoding of a signed integer.
+ * @param OutByteCount The number of bytes consumed from the input.
+ * @return A signed integer.
+ */
+inline int64_t
+ReadVarInt(const void* InData, uint32_t& OutByteCount)
+{
+ const uint64_t Value = ReadVarUInt(InData, OutByteCount);
+ return -int64_t(Value & 1) ^ int64_t(Value >> 1);
+}
+
+/**
+ * Write a variable-length unsigned integer.
+ *
+ * @param InValue An unsigned integer to encode.
+ * @param OutData A buffer of at least 5 bytes to write the output to.
+ * @return The number of bytes used in the output.
+ */
+inline uint32_t
+WriteVarUInt(uint32_t InValue, void* OutData)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InValue);
+ uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1;
+ switch (ByteCount - 1)
+ {
+ case 4:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 3:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 2:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 1:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ default:
+ break;
+ }
+ *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue);
+ return ByteCount;
+}
+
+/**
+ * Write a variable-length unsigned integer.
+ *
+ * @param InValue An unsigned integer to encode.
+ * @param OutData A buffer of at least 9 bytes to write the output to.
+ * @return The number of bytes used in the output.
+ */
+inline uint32_t
+WriteVarUInt(uint64_t InValue, void* OutData)
+{
+ const uint32_t ByteCount = MeasureVarUInt(InValue);
+ uint8_t* OutBytes = static_cast<uint8_t*>(OutData) + ByteCount - 1;
+ switch (ByteCount - 1)
+ {
+ case 8:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 7:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 6:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 5:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 4:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 3:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 2:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ case 1:
+ *OutBytes-- = uint8_t(InValue);
+ InValue >>= 8;
+ default:
+ break;
+ }
+ *OutBytes = uint8_t(0xff << (9 - ByteCount)) | uint8_t(InValue);
+ return ByteCount;
+}
+
+/** Write a variable-length signed integer. \see \ref WriteVarUInt */
+inline uint32_t
+WriteVarInt(int32_t InValue, void* OutData)
+{
+ const uint32_t Value = uint32_t((InValue >> 31) ^ (InValue << 1));
+ return WriteVarUInt(Value, OutData);
+}
+
+/** Write a variable-length signed integer. \see \ref WriteVarUInt */
+inline uint32_t
+WriteVarInt(int64_t InValue, void* OutData)
+{
+ const uint64_t Value = uint64_t((InValue >> 63) ^ (InValue << 1));
+ return WriteVarUInt(Value, OutData);
+}
+
+} // namespace zen
diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h
new file mode 100644
index 000000000..8888bf757
--- /dev/null
+++ b/zencore/include/zencore/windows.h
@@ -0,0 +1,10 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
+#ifndef NOMINMAX
+# define NOMINMAX // We don't want your min/max macros
+#endif
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
diff --git a/zencore/include/zencore/xxhash.h b/zencore/include/zencore/xxhash.h
new file mode 100644
index 000000000..5407755df
--- /dev/null
+++ b/zencore/include/zencore/xxhash.h
@@ -0,0 +1,87 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zencore.h"
+
+#include <zencore/memory.h>
+
+#include <xxh3.h>
+#include <string_view>
+
+namespace zen {
+
+class StringBuilderBase;
+
+/**
+ * XXH3 hash
+ */
+struct XXH3_128
+{
+ uint8_t Hash[16];
+
+ static XXH3_128 MakeFrom(const void* data /* 16 bytes */)
+ {
+ XXH3_128 Xx;
+ memcpy(Xx.Hash, data, sizeof Xx);
+ return Xx;
+ }
+
+ static inline XXH3_128 HashMemory(const void* data, size_t byteCount)
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits(data, byteCount));
+ return Hash;
+ }
+ static XXH3_128 HashMemory(MemoryView Data) { return HashMemory(Data.GetData(), Data.GetSize()); }
+ static XXH3_128 FromHexString(const char* string);
+ static XXH3_128 FromHexString(const std::string_view string);
+ const char* ToHexString(char* outString /* 32 characters + NUL terminator */) const;
+ StringBuilderBase& ToHexString(StringBuilderBase& outBuilder) const;
+
+ static const int StringLength = 32;
+ typedef char String_t[StringLength + 1];
+
+ static XXH3_128 Zero; // Initialized to all zeros
+
+ inline auto operator<=>(const XXH3_128& rhs) const = default;
+
+ struct Hasher
+ {
+ size_t operator()(const XXH3_128& v) const
+ {
+ size_t h;
+ memcpy(&h, v.Hash, sizeof h);
+ return h;
+ }
+ };
+};
+
+struct XXH3_128Stream
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { memset(&m_State, 0, sizeof m_State); }
+
+ /// Append another chunk
+ XXH3_128Stream& Append(const void* Data, size_t ByteCount)
+ {
+ XXH3_128bits_update(&m_State, Data, ByteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ XXH3_128Stream& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ XXH3_128 GetHash()
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State));
+ return Hash;
+ }
+
+private:
+ XXH3_state_s m_State{};
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
new file mode 100644
index 000000000..4a448776b
--- /dev/null
+++ b/zencore/include/zencore/zencore.h
@@ -0,0 +1,134 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <cinttypes>
+#include <exception>
+#include <string>
+
+//////////////////////////////////////////////////////////////////////////
+// Platform
+//
+
+#define ZEN_PLATFORM_WINDOWS 1
+#define ZEN_PLATFORM_LINUX 0
+#define ZEN_PLATFORM_MACOS 0
+
+//////////////////////////////////////////////////////////////////////////
+// Compiler
+//
+
+#ifdef _MSC_VER
+# define ZEN_COMPILER_MSC 1
+#endif
+
+#ifndef ZEN_COMPILER_MSC
+# define ZEN_COMPILER_MSC 0
+#endif
+
+#ifndef ZEN_COMPILER_CLANG
+# define ZEN_COMPILER_CLANG 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+// Build flavor
+//
+
+#ifdef NDEBUG
+# define ZEN_BUILD_DEBUG 0
+# define ZEN_BUILD_RELEASE 1
+#else
+# define ZEN_BUILD_DEBUG 1
+# define ZEN_BUILD_RELEASE 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1
+
+//////////////////////////////////////////////////////////////////////////
+// Assert
+//
+
+namespace zen {
+
+class AssertException : public std::exception
+{
+public:
+ AssertException(const char* Msg) : m_Msg(Msg) {}
+
+ [[nodiscard]] virtual char const* what() const override { return m_Msg.c_str(); }
+
+private:
+ std::string m_Msg;
+};
+
+} // namespace zen
+
+#define ZEN_ASSERT(x, ...) \
+ do \
+ { \
+ if (x) \
+ break; \
+ throw ::zen::AssertException{#x}; \
+ } while (false)
+
+#ifndef NDEBUG
+# define ZEN_ASSERT_SLOW(x, ...) \
+ do \
+ { \
+ if (x) \
+ break; \
+ throw ::zen::AssertException{#x}; \
+ } while (false)
+#else
+# define ZEN_ASSERT_SLOW(x, ...)
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+#ifdef __clang__
+template<typename T>
+auto ZenArrayCountHelper(T& t) -> typename std::enable_if<__is_array(T), char (&)[sizeof(t) / sizeof(t[0]) + 1]>::Type;
+#else
+template<typename T, uint32_t N>
+char (&ZenArrayCountHelper(const T (&)[N]))[N + 1];
+#endif
+
+#define ZEN_ARRAY_COUNT(array) (sizeof(ZenArrayCountHelper(array)) - 1)
+
+//////////////////////////////////////////////////////////////////////////
+
+#define ZEN_NOINLINE __declspec(noinline)
+#define ZEN_UNUSED(...) ((void)__VA_ARGS__)
+#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false)
+#define ZENCORE_API // Placeholder to allow DLL configs in the future
+
+ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread
+ZENCORE_API bool IsApplicationExitRequested();
+ZENCORE_API void RequestApplicationExit(int ExitCode);
+
+ZENCORE_API void zencore_forcelinktests();
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_COMPILER_MSC
+# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off))
+# define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on))
+#else
+#endif
+
+// Set up optimization control macros, now that we have both the build settings and the platform macros
+#define ZEN_DISABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL
+
+#if ZEN_BUILD_DEBUG
+# define ZEN_ENABLE_OPTIMIZATION ZEN_DISABLE_OPTIMIZATION_ACTUAL
+#else
+# define ZEN_ENABLE_OPTIMIZATION ZEN_ENABLE_OPTIMIZATION_ACTUAL
+#endif
+
+#define ZEN_ENABLE_OPTIMIZATION_ALWAYS ZEN_ENABLE_OPTIMIZATION_ACTUAL
+
+//////////////////////////////////////////////////////////////////////////
+
+using ThreadId_t = uint32_t;
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
new file mode 100644
index 000000000..a42dd83f4
--- /dev/null
+++ b/zencore/iobuffer.cpp
@@ -0,0 +1,341 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/iobuffer.h>
+
+#include <doctest/doctest.h>
+#include <memory.h>
+#include <zencore/memory.h>
+#include <zencore/thread.h>
+#include <system_error>
+
+#include <atlfile.h>
+#include <spdlog/spdlog.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+void*
+IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment)
+{
+ if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000))
+ {
+ m_Flags |= kLowLevelAlloc;
+ return VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE);
+ }
+ else
+ {
+ return Memory::Alloc(InSize, Alignment);
+ }
+}
+
+void
+IoBufferCore::FreeBuffer()
+{
+ if (m_Flags & kLowLevelAlloc)
+ {
+ VirtualFree(const_cast<void*>(m_DataPtr), 0, MEM_DECOMMIT);
+ }
+ else
+ {
+ return Memory::Free(const_cast<void*>(m_DataPtr));
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+IoBufferCore::IoBufferCore(size_t InSize)
+{
+ static_assert(sizeof(IoBufferCore) == 32);
+
+ m_DataPtr = AllocateBuffer(InSize, sizeof(void*));
+ m_DataBytes = InSize;
+
+ SetIsOwned(true);
+}
+
+IoBufferCore::IoBufferCore(size_t InSize, size_t Alignment)
+{
+ m_DataPtr = AllocateBuffer(InSize, Alignment);
+ m_DataBytes = InSize;
+
+ SetIsOwned(true);
+}
+
+IoBufferCore::~IoBufferCore()
+{
+ if (IsOwned() && m_DataPtr)
+ {
+ FreeBuffer();
+ m_DataPtr = nullptr;
+ }
+}
+
+void
+IoBufferCore::DeleteThis() const
+{
+ // We do this just to avoid paying for the cost of a vtable
+ if (const IoBufferExtendedCore* _ = ExtendedCore())
+ {
+ delete _;
+ }
+ else
+ {
+ delete this;
+ }
+}
+
+void
+IoBufferCore::Materialize() const
+{
+ if (const IoBufferExtendedCore* _ = ExtendedCore())
+ {
+ _->Materialize();
+ }
+}
+
+void
+IoBufferCore::MakeOwned(bool Immutable)
+{
+ if (!IsOwned())
+ {
+ void* OwnedDataPtr = AllocateBuffer(m_DataBytes, sizeof(void*));
+ memcpy(OwnedDataPtr, m_DataPtr, m_DataBytes);
+
+ m_DataPtr = OwnedDataPtr;
+ SetIsOwned(true);
+ }
+
+ SetIsImmutable(Immutable);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+IoBufferExtendedCore::IoBufferExtendedCore(void* FileHandle, uint64_t Offset, uint64_t Size, bool TransferHandleOwnership)
+: IoBufferCore(nullptr, Size)
+, m_FileHandle(FileHandle)
+, m_FileOffset(Offset)
+{
+ m_Flags |= kIsOwned | kIsExtended;
+
+ if (TransferHandleOwnership)
+ {
+ m_Flags |= kOwnsFile;
+ }
+}
+
+IoBufferExtendedCore::IoBufferExtendedCore(const IoBufferExtendedCore* Outer, uint64_t Offset, uint64_t Size)
+: IoBufferCore(Outer, nullptr, Size)
+, m_FileHandle(Outer->m_FileHandle)
+, m_FileOffset(Outer->m_FileOffset + Offset)
+{
+ m_Flags |= kIsOwned | kIsExtended;
+}
+
+IoBufferExtendedCore::~IoBufferExtendedCore()
+{
+ if (m_MappedPointer)
+ {
+ UnmapViewOfFile(m_MappedPointer);
+ }
+
+ if (m_Flags & kOwnsMmap)
+ {
+ CloseHandle(m_MmapHandle);
+ }
+
+ if (m_Flags & kOwnsFile)
+ {
+ BOOL Success = CloseHandle(m_FileHandle);
+
+ if (!Success)
+ {
+ spdlog::warn("Error reported on file handle close!");
+ }
+ }
+
+ m_DataPtr = nullptr;
+}
+
+RwLock g_MappingLock;
+
+void
+IoBufferExtendedCore::Materialize() const
+{
+ // The synchronization scheme here is very primitive, if we end up with
+ // a lot of contention we can make it more fine-grained
+
+ if (m_MmapHandle)
+ return;
+
+ RwLock::ExclusiveLockScope _(g_MappingLock);
+
+ // Someone could have gotten here first
+ if (m_MmapHandle)
+ return;
+
+ m_MmapHandle = CreateFileMapping(m_FileHandle,
+ /* lpFileMappingAttributes */ nullptr,
+ /* flProtect */ PAGE_READONLY,
+ /* dwMaximumSizeLow */ 0,
+ /* dwMaximumSizeHigh */ 0,
+ /* lpName */ nullptr);
+
+ if (m_MmapHandle == nullptr)
+ {
+ throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file copy failed");
+ }
+
+ m_Flags |= kOwnsMmap;
+
+ const uint64_t MapOffset = m_FileOffset & ~0xffffull;
+ const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset;
+ const uint64_t MapSize = (MappedOffsetDisplacement + m_DataBytes + 0xffffu) & ~0xffffull;
+
+ void* MappedBase = MapViewOfFile(m_MmapHandle,
+ /* dwDesiredAccess */ FILE_MAP_READ,
+ /* FileOffsetHigh */ uint32_t(MapOffset >> 32),
+ /* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu),
+ /* dwNumberOfBytesToMap */ m_DataBytes);
+
+ if (MappedBase == nullptr)
+ {
+ throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "MapViewOfFile failed");
+ }
+
+ m_MappedPointer = MappedBase;
+ m_DataPtr = reinterpret_cast<uint8_t*>(MappedBase) + MappedOffsetDisplacement;
+
+ m_Flags |= kIsMaterialized;
+}
+
+bool
+IoBufferExtendedCore::GetFileReference(IoBufferFileReference& OutRef) const
+{
+ if (m_FileHandle == nullptr)
+ {
+ return false;
+ }
+
+ OutRef.FileHandle = m_FileHandle;
+ OutRef.FileChunkOffset = m_FileOffset;
+ OutRef.FileChunkSize = m_DataBytes;
+
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+IoBuffer::IoBuffer(size_t InSize) : m_Core(new IoBufferCore(InSize))
+{
+}
+
+IoBuffer::IoBuffer(size_t InSize, uint64_t InAlignment) : m_Core(new IoBufferCore(InSize, InAlignment))
+{
+}
+
+IoBuffer::IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t Size)
+{
+ if (Size == ~(0ull))
+ {
+ Size = std::clamp<size_t>(Size, 0, OuterBuffer.Size() - Offset);
+ }
+
+ ZEN_ASSERT(Offset <= OuterBuffer.Size());
+ ZEN_ASSERT((Offset + Size) <= OuterBuffer.Size());
+
+ if (IoBufferExtendedCore* Extended = OuterBuffer.m_Core->ExtendedCore())
+ {
+ m_Core = new IoBufferExtendedCore(Extended, Offset, Size);
+ }
+ else
+ {
+ m_Core = new IoBufferCore(OuterBuffer.m_Core, reinterpret_cast<const uint8_t*>(OuterBuffer.Data()) + Offset, Size);
+ }
+}
+
+IoBuffer::IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize)
+: m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ true))
+{
+}
+
+IoBuffer::IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize)
+: m_Core(new IoBufferExtendedCore(FileHandle, ChunkFileOffset, ChunkSize, /* owned */ false))
+{
+}
+
+bool
+IoBuffer::GetFileReference(IoBufferFileReference& OutRef) const
+{
+ if (IoBufferExtendedCore* ExtCore = m_Core->ExtendedCore())
+ {
+ if (ExtCore->GetFileReference(OutRef))
+ {
+ return true;
+ }
+ }
+
+ // Not a file reference
+
+ OutRef.FileHandle = 0;
+ OutRef.FileChunkOffset = ~0ull;
+ OutRef.FileChunkSize = 0;
+
+ return false;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+IoBuffer
+IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size)
+{
+ return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size);
+}
+
+IoBuffer
+IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t Size)
+{
+ CAtlFile DataFile;
+
+ HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
+
+ if (SUCCEEDED(hRes))
+ {
+ ULONGLONG FileSize;
+ DataFile.GetSize(FileSize);
+
+ if (Size == ~0ull)
+ {
+ Size = FileSize;
+ }
+ else
+ {
+ // Clamp size
+ if ((Offset + Size) > FileSize)
+ {
+ Size = FileSize - Offset;
+ }
+ }
+
+ return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size);
+ }
+
+ return {};
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+iobuffer_forcelink()
+{
+}
+
+TEST_CASE("IoBuffer")
+{
+ zen::IoBuffer buffer1;
+ zen::IoBuffer buffer2(16384);
+ zen::IoBuffer buffer3(buffer2, 0, buffer2.Size());
+}
+
+} // namespace zen
diff --git a/zencore/iohash.cpp b/zencore/iohash.cpp
new file mode 100644
index 000000000..afe2e54ba
--- /dev/null
+++ b/zencore/iohash.cpp
@@ -0,0 +1,73 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/iohash.h>
+
+#include <zencore/blake3.h>
+#include <zencore/string.h>
+
+#include <doctest/doctest.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+IoHash IoHash::Zero; // Initialized to all zeros
+
+IoHash
+IoHash::HashMemory(const void* data, size_t byteCount)
+{
+ BLAKE3 b3 = BLAKE3::HashMemory(data, byteCount);
+
+ IoHash io;
+ memcpy(io.Hash, b3.Hash, sizeof io.Hash);
+
+ return io;
+}
+
+IoHash
+IoHash::FromHexString(const char* string)
+{
+ return FromHexString({string, sizeof(IoHash::Hash) * 2});
+}
+
+IoHash
+IoHash::FromHexString(std::string_view string)
+{
+ ZEN_ASSERT(string.size() == 2 * sizeof(IoHash::Hash));
+
+ IoHash io;
+
+ ParseHexBytes(string.data(), string.size(), io.Hash);
+
+ return io;
+}
+
+const char*
+IoHash::ToHexString(char* outString /* 40 characters + NUL terminator */) const
+{
+ ToHexBytes(Hash, sizeof(IoHash), outString);
+ outString[2 * sizeof(IoHash)] = '\0';
+
+ return outString;
+}
+
+StringBuilderBase&
+IoHash::ToHexString(StringBuilderBase& outBuilder) const
+{
+ String_t Str;
+ ToHexString(Str);
+
+ outBuilder.AppendRange(Str, &Str[StringLength]);
+
+ return outBuilder;
+}
+
+std::string
+IoHash::ToHexString() const
+{
+ String_t Str;
+ ToHexString(Str);
+
+ return Str;
+}
+
+} // namespace zen
diff --git a/zencore/iothreadpool.cpp b/zencore/iothreadpool.cpp
new file mode 100644
index 000000000..4ed81d7a2
--- /dev/null
+++ b/zencore/iothreadpool.cpp
@@ -0,0 +1,36 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "iothreadpool.h"
+
+namespace zen {
+
+WinIoThreadPool::WinIoThreadPool(int InThreadCount)
+{
+ // Thread pool setup
+
+ m_ThreadPool = CreateThreadpool(NULL);
+
+ SetThreadpoolThreadMinimum(m_ThreadPool, InThreadCount);
+ SetThreadpoolThreadMaximum(m_ThreadPool, InThreadCount * 2);
+
+ InitializeThreadpoolEnvironment(&m_CallbackEnvironment);
+
+ m_CleanupGroup = CreateThreadpoolCleanupGroup();
+
+ SetThreadpoolCallbackPool(&m_CallbackEnvironment, m_ThreadPool);
+
+ SetThreadpoolCallbackCleanupGroup(&m_CallbackEnvironment, m_CleanupGroup, NULL);
+}
+
+WinIoThreadPool::~WinIoThreadPool()
+{
+ CloseThreadpool(m_ThreadPool);
+}
+
+void
+WinIoThreadPool::CreateIocp(HANDLE IoHandle, PTP_WIN32_IO_CALLBACK Callback, void* Context)
+{
+ m_ThreadPoolIo = CreateThreadpoolIo(IoHandle, Callback, Context, &m_CallbackEnvironment);
+}
+
+} // namespace zen
diff --git a/zencore/iothreadpool.h b/zencore/iothreadpool.h
new file mode 100644
index 000000000..f64868540
--- /dev/null
+++ b/zencore/iothreadpool.h
@@ -0,0 +1,31 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/windows.h>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Thread pool. Implemented in terms of Windows thread pool right now, will
+// need a cross-platform implementation eventually
+//
+
+class WinIoThreadPool
+{
+public:
+ WinIoThreadPool(int InThreadCount);
+ ~WinIoThreadPool();
+
+ void CreateIocp(HANDLE IoHandle, PTP_WIN32_IO_CALLBACK Callback, void* Context);
+ inline PTP_IO Iocp() const { return m_ThreadPoolIo; }
+
+private:
+ PTP_POOL m_ThreadPool = nullptr;
+ PTP_CLEANUP_GROUP m_CleanupGroup = nullptr;
+ PTP_IO m_ThreadPoolIo = nullptr;
+ TP_CALLBACK_ENVIRON m_CallbackEnvironment;
+};
+
+} // namespace zen
diff --git a/zencore/md5.cpp b/zencore/md5.cpp
new file mode 100644
index 000000000..228c0feff
--- /dev/null
+++ b/zencore/md5.cpp
@@ -0,0 +1,446 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/md5.h>
+#include <zencore/string.h>
+#include <zencore/zencore.h>
+
+#include <doctest/doctest.h>
+#include <string.h>
+
+// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN
+#ifndef _MSC_VER
+# include <endian.h>
+#endif
+
+/*
+ **********************************************************************
+ ** md5.h -- Header file for implementation of MD5 **
+ ** RSA Data Security, Inc. MD5 Message Digest Algorithm **
+ ** Created: 2/17/90 RLR **
+ ** Revised: 12/27/90 SRD,AJ,BSK,JT Reference C version **
+ ** Revised (for MD5): RLR 4/27/91 **
+ ** -- G modified to have y&~z instead of y&z **
+ ** -- FF, GG, HH modified to add in last register done **
+ ** -- Access pattern: round 2 works mod 5, round 3 works mod 3 **
+ ** -- distinct additive constant for each step **
+ ** -- round 4 added, working mod 7 **
+ **********************************************************************
+ */
+
+/*
+ **********************************************************************
+ ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. **
+ ** **
+ ** License to copy and use this software is granted provided that **
+ ** it is identified as the "RSA Data Security, Inc. MD5 Message **
+ ** Digest Algorithm" in all material mentioning or referencing this **
+ ** software or this function. **
+ ** **
+ ** License is also granted to make and use derivative works **
+ ** provided that such works are identified as "derived from the RSA **
+ ** Data Security, Inc. MD5 Message Digest Algorithm" in all **
+ ** material mentioning or referencing the derived work. **
+ ** **
+ ** RSA Data Security, Inc. makes no representations concerning **
+ ** either the merchantability of this software or the suitability **
+ ** of this software for any particular purpose. It is provided "as **
+ ** is" without express or implied warranty of any kind. **
+ ** **
+ ** These notices must be retained in any copies of any part of this **
+ ** documentation and/or software. **
+ **********************************************************************
+ */
+
+/* Data structure for MD5 (Message Digest) computation */
+struct MD5_CTX
+{
+ uint32_t i[2]; /* number of _bits_ handled mod 2^64 */
+ uint32_t buf[4]; /* scratch buffer */
+ unsigned char in[64]; /* input buffer */
+ unsigned char digest[16]; /* actual digest after MD5Final call */
+};
+
+void MD5Init();
+void MD5Update();
+void MD5Final();
+
+/*
+ **********************************************************************
+ ** End of md5.h **
+ ******************************* (cut) ********************************
+ */
+
+/*
+ **********************************************************************
+ ** md5.c **
+ ** RSA Data Security, Inc. MD5 Message Digest Algorithm **
+ ** Created: 2/17/90 RLR **
+ ** Revised: 1/91 SRD,AJ,BSK,JT Reference C Version **
+ **********************************************************************
+ */
+
+/*
+ **********************************************************************
+ ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. **
+ ** **
+ ** License to copy and use this software is granted provided that **
+ ** it is identified as the "RSA Data Security, Inc. MD5 Message **
+ ** Digest Algorithm" in all material mentioning or referencing this **
+ ** software or this function. **
+ ** **
+ ** License is also granted to make and use derivative works **
+ ** provided that such works are identified as "derived from the RSA **
+ ** Data Security, Inc. MD5 Message Digest Algorithm" in all **
+ ** material mentioning or referencing the derived work. **
+ ** **
+ ** RSA Data Security, Inc. makes no representations concerning **
+ ** either the merchantability of this software or the suitability **
+ ** of this software for any particular purpose. It is provided "as **
+ ** is" without express or implied warranty of any kind. **
+ ** **
+ ** These notices must be retained in any copies of any part of this **
+ ** documentation and/or software. **
+ **********************************************************************
+ */
+
+/* -- include the following line if the md5.h header file is separate -- */
+/* #include "md5.h" */
+
+/* forward declaration */
+static void Transform(uint32_t* buf, uint32_t* in);
+
+static unsigned char PADDING[64] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+/* F, G and H are basic MD5 functions: selection, majority, parity */
+#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
+#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
+#define H(x, y, z) ((x) ^ (y) ^ (z))
+#define I(x, y, z) ((y) ^ ((x) | (~z)))
+
+/* ROTATE_LEFT rotates x left n bits */
+#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
+
+/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4 */
+/* Rotation is separate from addition to prevent recomputation */
+#define FF(a, b, c, d, x, s, ac) \
+ { \
+ (a) += F((b), (c), (d)) + (x) + (uint32_t)(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+#define GG(a, b, c, d, x, s, ac) \
+ { \
+ (a) += G((b), (c), (d)) + (x) + (uint32_t)(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+#define HH(a, b, c, d, x, s, ac) \
+ { \
+ (a) += H((b), (c), (d)) + (x) + (uint32_t)(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+#define II(a, b, c, d, x, s, ac) \
+ { \
+ (a) += I((b), (c), (d)) + (x) + (uint32_t)(ac); \
+ (a) = ROTATE_LEFT((a), (s)); \
+ (a) += (b); \
+ }
+
+void
+MD5Init(MD5_CTX* mdContext)
+{
+ mdContext->i[0] = mdContext->i[1] = (uint32_t)0;
+
+ /* Load magic initialization constants.
+ */
+ mdContext->buf[0] = (uint32_t)0x67452301;
+ mdContext->buf[1] = (uint32_t)0xefcdab89;
+ mdContext->buf[2] = (uint32_t)0x98badcfe;
+ mdContext->buf[3] = (uint32_t)0x10325476;
+}
+
+void
+MD5Update(MD5_CTX* mdContext, unsigned char* inBuf, unsigned int inLen)
+{
+ uint32_t in[16];
+ int mdi;
+ unsigned int i, ii;
+
+ /* compute number of bytes mod 64 */
+ mdi = (int)((mdContext->i[0] >> 3) & 0x3F);
+
+ /* update number of bits */
+ if ((mdContext->i[0] + ((uint32_t)inLen << 3)) < mdContext->i[0])
+ mdContext->i[1]++;
+ mdContext->i[0] += ((uint32_t)inLen << 3);
+ mdContext->i[1] += ((uint32_t)inLen >> 29);
+
+ while (inLen--)
+ {
+ /* add new character to buffer, increment mdi */
+ mdContext->in[mdi++] = *inBuf++;
+
+ /* transform if necessary */
+ if (mdi == 0x40)
+ {
+ for (i = 0, ii = 0; i < 16; i++, ii += 4)
+ in[i] = (((uint32_t)mdContext->in[ii + 3]) << 24) | (((uint32_t)mdContext->in[ii + 2]) << 16) |
+ (((uint32_t)mdContext->in[ii + 1]) << 8) | ((uint32_t)mdContext->in[ii]);
+ Transform(mdContext->buf, in);
+ mdi = 0;
+ }
+ }
+}
+
+void
+MD5Final(MD5_CTX* mdContext)
+{
+ uint32_t in[16];
+ int mdi;
+ unsigned int i, ii;
+ unsigned int padLen;
+
+ /* save number of bits */
+ in[14] = mdContext->i[0];
+ in[15] = mdContext->i[1];
+
+ /* compute number of bytes mod 64 */
+ mdi = (int)((mdContext->i[0] >> 3) & 0x3F);
+
+ /* pad out to 56 mod 64 */
+ padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi);
+ MD5Update(mdContext, PADDING, padLen);
+
+ /* append length in bits and transform */
+ for (i = 0, ii = 0; i < 14; i++, ii += 4)
+ in[i] = (((uint32_t)mdContext->in[ii + 3]) << 24) | (((uint32_t)mdContext->in[ii + 2]) << 16) |
+ (((uint32_t)mdContext->in[ii + 1]) << 8) | ((uint32_t)mdContext->in[ii]);
+ Transform(mdContext->buf, in);
+
+ /* store buffer in digest */
+ for (i = 0, ii = 0; i < 4; i++, ii += 4)
+ {
+ mdContext->digest[ii] = (unsigned char)(mdContext->buf[i] & 0xFF);
+ mdContext->digest[ii + 1] = (unsigned char)((mdContext->buf[i] >> 8) & 0xFF);
+ mdContext->digest[ii + 2] = (unsigned char)((mdContext->buf[i] >> 16) & 0xFF);
+ mdContext->digest[ii + 3] = (unsigned char)((mdContext->buf[i] >> 24) & 0xFF);
+ }
+}
+
+/* Basic MD5 step. Transform buf based on in.
+ */
+static void
+Transform(uint32_t* buf, uint32_t* in)
+{
+ uint32_t a = buf[0], b = buf[1], c = buf[2], d = buf[3];
+
+ /* Round 1 */
+#define S11 7
+#define S12 12
+#define S13 17
+#define S14 22
+ FF(a, b, c, d, in[0], S11, 3614090360); /* 1 */
+ FF(d, a, b, c, in[1], S12, 3905402710); /* 2 */
+ FF(c, d, a, b, in[2], S13, 606105819); /* 3 */
+ FF(b, c, d, a, in[3], S14, 3250441966); /* 4 */
+ FF(a, b, c, d, in[4], S11, 4118548399); /* 5 */
+ FF(d, a, b, c, in[5], S12, 1200080426); /* 6 */
+ FF(c, d, a, b, in[6], S13, 2821735955); /* 7 */
+ FF(b, c, d, a, in[7], S14, 4249261313); /* 8 */
+ FF(a, b, c, d, in[8], S11, 1770035416); /* 9 */
+ FF(d, a, b, c, in[9], S12, 2336552879); /* 10 */
+ FF(c, d, a, b, in[10], S13, 4294925233); /* 11 */
+ FF(b, c, d, a, in[11], S14, 2304563134); /* 12 */
+ FF(a, b, c, d, in[12], S11, 1804603682); /* 13 */
+ FF(d, a, b, c, in[13], S12, 4254626195); /* 14 */
+ FF(c, d, a, b, in[14], S13, 2792965006); /* 15 */
+ FF(b, c, d, a, in[15], S14, 1236535329); /* 16 */
+
+ /* Round 2 */
+#define S21 5
+#define S22 9
+#define S23 14
+#define S24 20
+ GG(a, b, c, d, in[1], S21, 4129170786); /* 17 */
+ GG(d, a, b, c, in[6], S22, 3225465664); /* 18 */
+ GG(c, d, a, b, in[11], S23, 643717713); /* 19 */
+ GG(b, c, d, a, in[0], S24, 3921069994); /* 20 */
+ GG(a, b, c, d, in[5], S21, 3593408605); /* 21 */
+ GG(d, a, b, c, in[10], S22, 38016083); /* 22 */
+ GG(c, d, a, b, in[15], S23, 3634488961); /* 23 */
+ GG(b, c, d, a, in[4], S24, 3889429448); /* 24 */
+ GG(a, b, c, d, in[9], S21, 568446438); /* 25 */
+ GG(d, a, b, c, in[14], S22, 3275163606); /* 26 */
+ GG(c, d, a, b, in[3], S23, 4107603335); /* 27 */
+ GG(b, c, d, a, in[8], S24, 1163531501); /* 28 */
+ GG(a, b, c, d, in[13], S21, 2850285829); /* 29 */
+ GG(d, a, b, c, in[2], S22, 4243563512); /* 30 */
+ GG(c, d, a, b, in[7], S23, 1735328473); /* 31 */
+ GG(b, c, d, a, in[12], S24, 2368359562); /* 32 */
+
+ /* Round 3 */
+#define S31 4
+#define S32 11
+#define S33 16
+#define S34 23
+ HH(a, b, c, d, in[5], S31, 4294588738); /* 33 */
+ HH(d, a, b, c, in[8], S32, 2272392833); /* 34 */
+ HH(c, d, a, b, in[11], S33, 1839030562); /* 35 */
+ HH(b, c, d, a, in[14], S34, 4259657740); /* 36 */
+ HH(a, b, c, d, in[1], S31, 2763975236); /* 37 */
+ HH(d, a, b, c, in[4], S32, 1272893353); /* 38 */
+ HH(c, d, a, b, in[7], S33, 4139469664); /* 39 */
+ HH(b, c, d, a, in[10], S34, 3200236656); /* 40 */
+ HH(a, b, c, d, in[13], S31, 681279174); /* 41 */
+ HH(d, a, b, c, in[0], S32, 3936430074); /* 42 */
+ HH(c, d, a, b, in[3], S33, 3572445317); /* 43 */
+ HH(b, c, d, a, in[6], S34, 76029189); /* 44 */
+ HH(a, b, c, d, in[9], S31, 3654602809); /* 45 */
+ HH(d, a, b, c, in[12], S32, 3873151461); /* 46 */
+ HH(c, d, a, b, in[15], S33, 530742520); /* 47 */
+ HH(b, c, d, a, in[2], S34, 3299628645); /* 48 */
+
+ /* Round 4 */
+#define S41 6
+#define S42 10
+#define S43 15
+#define S44 21
+ II(a, b, c, d, in[0], S41, 4096336452); /* 49 */
+ II(d, a, b, c, in[7], S42, 1126891415); /* 50 */
+ II(c, d, a, b, in[14], S43, 2878612391); /* 51 */
+ II(b, c, d, a, in[5], S44, 4237533241); /* 52 */
+ II(a, b, c, d, in[12], S41, 1700485571); /* 53 */
+ II(d, a, b, c, in[3], S42, 2399980690); /* 54 */
+ II(c, d, a, b, in[10], S43, 4293915773); /* 55 */
+ II(b, c, d, a, in[1], S44, 2240044497); /* 56 */
+ II(a, b, c, d, in[8], S41, 1873313359); /* 57 */
+ II(d, a, b, c, in[15], S42, 4264355552); /* 58 */
+ II(c, d, a, b, in[6], S43, 2734768916); /* 59 */
+ II(b, c, d, a, in[13], S44, 1309151649); /* 60 */
+ II(a, b, c, d, in[4], S41, 4149444226); /* 61 */
+ II(d, a, b, c, in[11], S42, 3174756917); /* 62 */
+ II(c, d, a, b, in[2], S43, 718787259); /* 63 */
+ II(b, c, d, a, in[9], S44, 3951481745); /* 64 */
+
+ buf[0] += a;
+ buf[1] += b;
+ buf[2] += c;
+ buf[3] += d;
+}
+
+/*
+ **********************************************************************
+ ** End of md5.c **
+ ******************************* (cut) ********************************
+ */
+
+#undef FF
+#undef GG
+#undef HH
+#undef II
+#undef F
+#undef G
+#undef H
+#undef I
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+MD5 MD5::Zero; // Initialized to all zeroes
+
+//////////////////////////////////////////////////////////////////////////
+
+MD5Stream::MD5Stream()
+{
+ Reset();
+}
+
+void
+MD5Stream::Reset()
+{
+}
+
+MD5Stream&
+MD5Stream::Append(const void* Data, size_t ByteCount)
+{
+ ZEN_UNUSED(Data);
+ ZEN_UNUSED(ByteCount);
+
+ return *this;
+}
+
+MD5
+MD5Stream::GetHash()
+{
+ MD5 md5{};
+
+ return md5;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+MD5
+MD5::HashMemory(const void* data, size_t byteCount)
+{
+ return MD5Stream().Append(data, byteCount).GetHash();
+}
+
+MD5
+MD5::FromHexString(const char* string)
+{
+ MD5 md5;
+
+ ParseHexBytes(string, 40, md5.Hash);
+
+ return md5;
+}
+
+const char*
+MD5::ToHexString(char* outString /* 40 characters + NUL terminator */) const
+{
+ ToHexBytes(Hash, sizeof(MD5), outString);
+ outString[2 * sizeof(MD5)] = '\0';
+
+ return outString;
+}
+
+StringBuilderBase&
+MD5::ToHexString(StringBuilderBase& outBuilder) const
+{
+ char str[41];
+ ToHexString(str);
+
+ outBuilder.AppendRange(str, &str[40]);
+
+ return outBuilder;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+void
+md5_forcelink()
+{
+}
+
+doctest::String
+toString(const MD5& value)
+{
+ char md5text[2 * sizeof(MD5) + 1];
+ value.ToHexString(md5text);
+
+ return md5text;
+}
+
+TEST_CASE("MD5")
+{
+}
+
+} // namespace zen
diff --git a/zencore/memory.cpp b/zencore/memory.cpp
new file mode 100644
index 000000000..63d61f5e1
--- /dev/null
+++ b/zencore/memory.cpp
@@ -0,0 +1,165 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <malloc.h>
+#include <zencore/intmath.h>
+#include <zencore/memory.h>
+
+#include <doctest/doctest.h>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+MemoryArena::MemoryArena()
+{
+}
+
+MemoryArena::~MemoryArena()
+{
+}
+
+void*
+MemoryArena::Alloc(size_t size, size_t alignment)
+{
+ return _mm_malloc(size, alignment);
+}
+
+void
+MemoryArena::Free(void* ptr)
+{
+ if (ptr)
+ _mm_free(ptr);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void*
+Memory::Alloc(size_t size, size_t alignment)
+{
+ return _mm_malloc(size, alignment);
+}
+
+void
+Memory::Free(void* ptr)
+{
+ if (ptr)
+ _mm_free(ptr);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+ChunkingLinearAllocator::ChunkingLinearAllocator(uint64_t ChunkSize, uint64_t ChunkAlignment)
+: m_ChunkSize(ChunkSize)
+, m_ChunkAlignment(ChunkAlignment)
+{
+}
+
+ChunkingLinearAllocator::~ChunkingLinearAllocator()
+{
+ Reset();
+}
+
+void
+ChunkingLinearAllocator::Reset()
+{
+ for (void* ChunkEntry : m_ChunkList)
+ {
+ Memory::Free(ChunkEntry);
+ }
+ m_ChunkList.clear();
+
+ m_ChunkCursor = nullptr;
+ m_ChunkBytesRemain = 0;
+}
+
+void*
+ChunkingLinearAllocator::Alloc(size_t Size, size_t Alignment)
+{
+ ZEN_ASSERT_SLOW(zen::IsPow2(Alignment));
+
+ // This could be improved in a bunch of ways
+ //
+ // * We pessimistically allocate memory even though there may be enough memory available for a single allocation due to the way we take
+ // alignment into account below
+ // * The block allocation size could be chosen to minimize slack for the case when multiple oversize allocations are made rather than
+ // minimizing the number of chunks
+ // * ...
+
+ const uint64_t AllocationSize = zen::RoundUp(Size, Alignment);
+
+ if (m_ChunkBytesRemain < (AllocationSize + Alignment - 1))
+ {
+ const uint64_t ChunkSize = zen::RoundUp(zen::Max(m_ChunkSize, Size), m_ChunkSize);
+ void* ChunkPtr = Memory::Alloc(ChunkSize, m_ChunkAlignment);
+ m_ChunkCursor = reinterpret_cast<uint8_t*>(ChunkPtr);
+ m_ChunkBytesRemain = ChunkSize;
+ m_ChunkList.push_back(ChunkPtr);
+ }
+
+ const uint64_t AlignFixup = (Alignment - reinterpret_cast<uintptr_t>(m_ChunkCursor)) & (Alignment - 1);
+ void* ReturnPtr = m_ChunkCursor + AlignFixup;
+ const uint64_t Delta = AlignFixup + AllocationSize;
+
+ ZEN_ASSERT_SLOW(m_ChunkBytesRemain >= Delta);
+
+ m_ChunkCursor += Delta;
+ m_ChunkBytesRemain -= Delta;
+
+ ZEN_ASSERT_SLOW(IsPointerAligned(ReturnPtr, Alignment));
+
+ return ReturnPtr;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Unit tests
+//
+
+TEST_CASE("ChunkingLinearAllocator")
+{
+ ChunkingLinearAllocator Allocator(4096);
+
+ void* p1 = Allocator.Alloc(1, 1);
+ void* p2 = Allocator.Alloc(1, 1);
+
+ CHECK(p1 != p2);
+
+ void* p3 = Allocator.Alloc(1, 4);
+ CHECK(IsPointerAligned(p3, 4));
+
+ void* p3_2 = Allocator.Alloc(1, 4);
+ CHECK(IsPointerAligned(p3_2, 4));
+
+ void* p4 = Allocator.Alloc(1, 8);
+ CHECK(IsPointerAligned(p4, 8));
+
+ for (int i = 0; i < 100; ++i)
+ {
+ void* p0 = Allocator.Alloc(64);
+ ZEN_UNUSED(p0);
+ }
+}
+
+TEST_CASE("MemoryView")
+{
+ {
+ uint8_t Array1[16];
+ MemoryView View1 = MakeMemoryView(Array1);
+ CHECK(View1.GetSize() == 16);
+ }
+
+ {
+ uint32_t Array2[16];
+ MemoryView View2 = MakeMemoryView(Array2);
+ CHECK(View2.GetSize() == 64);
+ }
+
+ CHECK(MakeMemoryView<float>({1.0f, 1.2f}).GetSize() == 8);
+}
+
+void
+memory_forcelink()
+{
+}
+
+} // namespace zen
diff --git a/zencore/refcount.cpp b/zencore/refcount.cpp
new file mode 100644
index 000000000..943635552
--- /dev/null
+++ b/zencore/refcount.cpp
@@ -0,0 +1,96 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/refcount.h>
+
+#include <doctest/doctest.h>
+#include <functional>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+struct TestRefClass : public RefCounted
+{
+ ~TestRefClass()
+ {
+ if (OnDestroy)
+ OnDestroy();
+ }
+
+ using RefCounted::RefCount;
+
+ std::function<void()> OnDestroy;
+};
+
+void
+refcount_forcelink()
+{
+}
+
+TEST_CASE("RefPtr")
+{
+ RefPtr<TestRefClass> Ref;
+ Ref = new TestRefClass;
+
+ bool IsDestroyed = false;
+ Ref->OnDestroy = [&] { IsDestroyed = true; };
+
+ CHECK(IsDestroyed == false);
+ CHECK(Ref->RefCount() == 1);
+
+ RefPtr<TestRefClass> Ref2;
+ Ref2 = Ref;
+
+ CHECK(IsDestroyed == false);
+ CHECK(Ref->RefCount() == 2);
+
+ RefPtr<TestRefClass> Ref3;
+ Ref2 = Ref3;
+
+ CHECK(IsDestroyed == false);
+ CHECK(Ref->RefCount() == 1);
+ Ref = Ref3;
+
+ CHECK(IsDestroyed == true);
+}
+
+TEST_CASE("RefPtr on Stack allocated object")
+{
+ bool IsDestroyed = false;
+
+ {
+ TestRefClass StackRefClass;
+
+ StackRefClass.OnDestroy = [&] { IsDestroyed = true; };
+
+ CHECK(StackRefClass.RefCount() == 1); // Stack allocated objects should have +1 ref
+
+ RefPtr<TestRefClass> Ref{&StackRefClass};
+
+ CHECK(IsDestroyed == false);
+ CHECK(StackRefClass.RefCount() == 2);
+
+ RefPtr<TestRefClass> Ref2;
+ Ref2 = Ref;
+
+ CHECK(IsDestroyed == false);
+ CHECK(StackRefClass.RefCount() == 3);
+
+ RefPtr<TestRefClass> Ref3;
+ Ref2 = Ref3;
+
+ CHECK(IsDestroyed == false);
+ CHECK(StackRefClass.RefCount() == 2);
+
+ Ref = Ref3;
+ CHECK(IsDestroyed == false);
+ CHECK(StackRefClass.RefCount() == 1);
+ }
+
+ CHECK(IsDestroyed == true);
+}
+
+} // namespace zen
diff --git a/zencore/sha1.cpp b/zencore/sha1.cpp
new file mode 100644
index 000000000..3cc2f5cdf
--- /dev/null
+++ b/zencore/sha1.cpp
@@ -0,0 +1,439 @@
+// //////////////////////////////////////////////////////////
+// sha1.cpp
+// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved.
+// see http://create.stephan-brumme.com/disclaimer.html
+//
+
+#include <zencore/sha1.h>
+#include <zencore/string.h>
+#include <zencore/zencore.h>
+
+#include <doctest/doctest.h>
+#include <string.h>
+
+// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN
+#ifndef _MSC_VER
+# include <endian.h>
+#endif
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+SHA1 SHA1::Zero; // Initialized to all zeroes
+
+//////////////////////////////////////////////////////////////////////////
+
+SHA1Stream::SHA1Stream()
+{
+ Reset();
+}
+
+void
+SHA1Stream::Reset()
+{
+ m_NumBytes = 0;
+ m_BufferSize = 0;
+
+ // according to RFC 1321
+ m_Hash[0] = 0x67452301;
+ m_Hash[1] = 0xefcdab89;
+ m_Hash[2] = 0x98badcfe;
+ m_Hash[3] = 0x10325476;
+ m_Hash[4] = 0xc3d2e1f0;
+}
+
+namespace {
+ // mix functions for processBlock()
+ inline uint32_t f1(uint32_t b, uint32_t c, uint32_t d)
+ {
+ return d ^ (b & (c ^ d)); // original: f = (b & c) | ((~b) & d);
+ }
+
+ inline uint32_t f2(uint32_t b, uint32_t c, uint32_t d) { return b ^ c ^ d; }
+
+ inline uint32_t f3(uint32_t b, uint32_t c, uint32_t d) { return (b & c) | (b & d) | (c & d); }
+
+ inline uint32_t rotate(uint32_t a, uint32_t c) { return (a << c) | (a >> (32 - c)); }
+
+ inline uint32_t swap(uint32_t x)
+ {
+#if defined(__GNUC__) || defined(__clang__)
+ return __builtin_bswap32(x);
+#endif
+#ifdef MSC_VER
+ return _byteswap_ulong(x);
+#endif
+
+ return (x >> 24) | ((x >> 8) & 0x0000FF00) | ((x << 8) & 0x00FF0000) | (x << 24);
+ }
+} // namespace
+
+/// process 64 bytes
+void
+SHA1Stream::ProcessBlock(const void* data)
+{
+ // get last hash
+ uint32_t a = m_Hash[0];
+ uint32_t b = m_Hash[1];
+ uint32_t c = m_Hash[2];
+ uint32_t d = m_Hash[3];
+ uint32_t e = m_Hash[4];
+
+ // data represented as 16x 32-bit words
+ const uint32_t* input = (uint32_t*)data;
+ // convert to big endian
+ uint32_t words[80];
+ for (int i = 0; i < 16; i++)
+#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN)
+ words[i] = input[i];
+#else
+ words[i] = swap(input[i]);
+#endif
+
+ // extend to 80 words
+ for (int i = 16; i < 80; i++)
+ words[i] = rotate(words[i - 3] ^ words[i - 8] ^ words[i - 14] ^ words[i - 16], 1);
+
+ // first round
+ for (int i = 0; i < 4; i++)
+ {
+ int offset = 5 * i;
+ e += rotate(a, 5) + f1(b, c, d) + words[offset] + 0x5a827999;
+ b = rotate(b, 30);
+ d += rotate(e, 5) + f1(a, b, c) + words[offset + 1] + 0x5a827999;
+ a = rotate(a, 30);
+ c += rotate(d, 5) + f1(e, a, b) + words[offset + 2] + 0x5a827999;
+ e = rotate(e, 30);
+ b += rotate(c, 5) + f1(d, e, a) + words[offset + 3] + 0x5a827999;
+ d = rotate(d, 30);
+ a += rotate(b, 5) + f1(c, d, e) + words[offset + 4] + 0x5a827999;
+ c = rotate(c, 30);
+ }
+
+ // second round
+ for (int i = 4; i < 8; i++)
+ {
+ int offset = 5 * i;
+ e += rotate(a, 5) + f2(b, c, d) + words[offset] + 0x6ed9eba1;
+ b = rotate(b, 30);
+ d += rotate(e, 5) + f2(a, b, c) + words[offset + 1] + 0x6ed9eba1;
+ a = rotate(a, 30);
+ c += rotate(d, 5) + f2(e, a, b) + words[offset + 2] + 0x6ed9eba1;
+ e = rotate(e, 30);
+ b += rotate(c, 5) + f2(d, e, a) + words[offset + 3] + 0x6ed9eba1;
+ d = rotate(d, 30);
+ a += rotate(b, 5) + f2(c, d, e) + words[offset + 4] + 0x6ed9eba1;
+ c = rotate(c, 30);
+ }
+
+ // third round
+ for (int i = 8; i < 12; i++)
+ {
+ int offset = 5 * i;
+ e += rotate(a, 5) + f3(b, c, d) + words[offset] + 0x8f1bbcdc;
+ b = rotate(b, 30);
+ d += rotate(e, 5) + f3(a, b, c) + words[offset + 1] + 0x8f1bbcdc;
+ a = rotate(a, 30);
+ c += rotate(d, 5) + f3(e, a, b) + words[offset + 2] + 0x8f1bbcdc;
+ e = rotate(e, 30);
+ b += rotate(c, 5) + f3(d, e, a) + words[offset + 3] + 0x8f1bbcdc;
+ d = rotate(d, 30);
+ a += rotate(b, 5) + f3(c, d, e) + words[offset + 4] + 0x8f1bbcdc;
+ c = rotate(c, 30);
+ }
+
+ // fourth round
+ for (int i = 12; i < 16; i++)
+ {
+ int offset = 5 * i;
+ e += rotate(a, 5) + f2(b, c, d) + words[offset] + 0xca62c1d6;
+ b = rotate(b, 30);
+ d += rotate(e, 5) + f2(a, b, c) + words[offset + 1] + 0xca62c1d6;
+ a = rotate(a, 30);
+ c += rotate(d, 5) + f2(e, a, b) + words[offset + 2] + 0xca62c1d6;
+ e = rotate(e, 30);
+ b += rotate(c, 5) + f2(d, e, a) + words[offset + 3] + 0xca62c1d6;
+ d = rotate(d, 30);
+ a += rotate(b, 5) + f2(c, d, e) + words[offset + 4] + 0xca62c1d6;
+ c = rotate(c, 30);
+ }
+
+ // update hash
+ m_Hash[0] += a;
+ m_Hash[1] += b;
+ m_Hash[2] += c;
+ m_Hash[3] += d;
+ m_Hash[4] += e;
+}
+
+/// add arbitrary number of bytes
+SHA1Stream&
+SHA1Stream::Append(const void* data, size_t byteCount)
+{
+ const uint8_t* current = (const uint8_t*)data;
+
+ if (m_BufferSize > 0)
+ {
+ while (byteCount > 0 && m_BufferSize < BlockSize)
+ {
+ m_Buffer[m_BufferSize++] = *current++;
+ byteCount--;
+ }
+ }
+
+ // full buffer
+ if (m_BufferSize == BlockSize)
+ {
+ ProcessBlock((void*)m_Buffer);
+ m_NumBytes += BlockSize;
+ m_BufferSize = 0;
+ }
+
+ // no more data ?
+ if (byteCount == 0)
+ return *this;
+
+ // process full blocks
+ while (byteCount >= BlockSize)
+ {
+ ProcessBlock(current);
+ current += BlockSize;
+ m_NumBytes += BlockSize;
+ byteCount -= BlockSize;
+ }
+
+ // keep remaining bytes in buffer
+ while (byteCount > 0)
+ {
+ m_Buffer[m_BufferSize++] = *current++;
+ byteCount--;
+ }
+
+ return *this;
+}
+
+/// process final block, less than 64 bytes
+void
+SHA1Stream::ProcessBuffer()
+{
+ // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte
+
+ // - append "1" bit to message
+ // - append "0" bits until message length in bit mod 512 is 448
+ // - append length as 64 bit integer
+
+ // number of bits
+ size_t paddedLength = m_BufferSize * 8;
+
+ // plus one bit set to 1 (always appended)
+ paddedLength++;
+
+ // number of bits must be (numBits % 512) = 448
+ size_t lower11Bits = paddedLength & 511;
+ if (lower11Bits <= 448)
+ paddedLength += 448 - lower11Bits;
+ else
+ paddedLength += 512 + 448 - lower11Bits;
+ // convert from bits to bytes
+ paddedLength /= 8;
+
+ // only needed if additional data flows over into a second block
+ unsigned char extra[BlockSize];
+
+ // append a "1" bit, 128 => binary 10000000
+ if (m_BufferSize < BlockSize)
+ m_Buffer[m_BufferSize] = 128;
+ else
+ extra[0] = 128;
+
+ size_t i;
+ for (i = m_BufferSize + 1; i < BlockSize; i++)
+ m_Buffer[i] = 0;
+ for (; i < paddedLength; i++)
+ extra[i - BlockSize] = 0;
+
+ // add message length in bits as 64 bit number
+ uint64_t msgBits = 8 * (m_NumBytes + m_BufferSize);
+ // find right position
+ unsigned char* addLength;
+ if (paddedLength < BlockSize)
+ addLength = m_Buffer + paddedLength;
+ else
+ addLength = extra + paddedLength - BlockSize;
+
+ // must be big endian
+ *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF);
+ *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF);
+ *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF);
+ *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF);
+ *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF);
+ *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF);
+ *addLength++ = (unsigned char)((msgBits >> 8) & 0xFF);
+ *addLength = (unsigned char)(msgBits & 0xFF);
+
+ // process blocks
+ ProcessBlock(m_Buffer);
+ // flowed over into a second block ?
+ if (paddedLength > BlockSize)
+ ProcessBlock(extra);
+}
+
+/// return latest hash as bytes
+SHA1
+SHA1Stream::GetHash()
+{
+ SHA1 sha1;
+ // save old hash if buffer is partially filled
+ uint32_t oldHash[HashValues];
+ for (int i = 0; i < HashValues; i++)
+ oldHash[i] = m_Hash[i];
+
+ // process remaining bytes
+ ProcessBuffer();
+
+ unsigned char* current = sha1.Hash;
+ for (int i = 0; i < HashValues; i++)
+ {
+ *current++ = (m_Hash[i] >> 24) & 0xFF;
+ *current++ = (m_Hash[i] >> 16) & 0xFF;
+ *current++ = (m_Hash[i] >> 8) & 0xFF;
+ *current++ = m_Hash[i] & 0xFF;
+
+ // restore old hash
+ m_Hash[i] = oldHash[i];
+ }
+
+ return sha1;
+}
+
+/// compute SHA1 of a memory block
+SHA1
+SHA1Stream::Compute(const void* data, size_t byteCount)
+{
+ Reset();
+ Append(data, byteCount);
+ return GetHash();
+}
+
+SHA1
+SHA1::HashMemory(const void* data, size_t byteCount)
+{
+ return SHA1Stream().Append(data, byteCount).GetHash();
+}
+
+SHA1
+SHA1::FromHexString(const char* string)
+{
+ SHA1 sha1;
+
+ ParseHexBytes(string, 40, sha1.Hash);
+
+ return sha1;
+}
+
+const char*
+SHA1::ToHexString(char* outString /* 40 characters + NUL terminator */) const
+{
+ ToHexBytes(Hash, sizeof(SHA1), outString);
+ outString[2 * sizeof(SHA1)] = '\0';
+
+ return outString;
+}
+
+StringBuilderBase&
+SHA1::ToHexString(StringBuilderBase& outBuilder) const
+{
+ char str[41];
+ ToHexString(str);
+
+ outBuilder.AppendRange(str, &str[40]);
+
+ return outBuilder;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+void
+sha1_forcelink()
+{
+}
+
+doctest::String
+toString(const SHA1& value)
+{
+ char sha1text[2 * sizeof(SHA1) + 1];
+ value.ToHexString(sha1text);
+
+ return sha1text;
+}
+
+TEST_CASE("SHA1")
+{
+ uint8_t sha1_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55,
+ 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09};
+ SHA1 sha1z;
+ memcpy(sha1z.Hash, sha1_empty, sizeof sha1z.Hash);
+
+ SUBCASE("Empty string")
+ {
+ SHA1 sha1 = SHA1::HashMemory(nullptr, 0);
+
+ CHECK(sha1 == sha1z);
+ }
+
+ SUBCASE("Empty stream")
+ {
+ SHA1Stream sha1s;
+ sha1s.Append(nullptr, 0);
+ sha1s.Append(nullptr, 0);
+ sha1s.Append(nullptr, 0);
+ CHECK(sha1s.GetHash() == sha1z);
+ }
+
+ SUBCASE("SHA1 from string")
+ {
+ const SHA1 sha1empty = SHA1::FromHexString("da39a3ee5e6b4b0d3255bfef95601890afd80709");
+
+ CHECK(sha1z == sha1empty);
+ }
+
+ SUBCASE("SHA1 to string")
+ {
+ char sha1str[41];
+ sha1z.ToHexString(sha1str);
+
+ CHECK(StringEquals(sha1str, "da39a3ee5e6b4b0d3255bfef95601890afd80709"));
+ }
+
+ SUBCASE("Hash ABC")
+ {
+ const SHA1 sha1abc = SHA1::FromHexString("3c01bdbb26f358bab27f267924aa2c9a03fcfdb8");
+
+ SHA1Stream sha1s;
+
+ sha1s.Append("A", 1);
+ sha1s.Append("B", 1);
+ sha1s.Append("C", 1);
+ CHECK(sha1s.GetHash() == sha1abc);
+
+ sha1s.Reset();
+ sha1s.Append("AB", 2);
+ sha1s.Append("C", 1);
+ CHECK(sha1s.GetHash() == sha1abc);
+
+ sha1s.Reset();
+ sha1s.Append("ABC", 3);
+ CHECK(sha1s.GetHash() == sha1abc);
+
+ sha1s.Reset();
+ sha1s.Append("A", 1);
+ sha1s.Append("BC", 2);
+ CHECK(sha1s.GetHash() == sha1abc);
+ }
+}
+
+} // namespace zen
diff --git a/zencore/sharedbuffer.cpp b/zencore/sharedbuffer.cpp
new file mode 100644
index 000000000..bc991053d
--- /dev/null
+++ b/zencore/sharedbuffer.cpp
@@ -0,0 +1,110 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/sharedbuffer.h>
+
+#include <doctest/doctest.h>
+#include <memory.h>
+
+#include <atlfile.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+BufferOwner::~BufferOwner()
+{
+ if (m_IsOwned)
+ {
+ Memory::Free(m_Data);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+UniqueBuffer
+UniqueBuffer::Alloc(uint64_t Size)
+{
+ void* Buffer = Memory::Alloc(Size, 16);
+ BufferOwner* Owner = new BufferOwner(Buffer, Size, /* owned */ true);
+
+ return UniqueBuffer(Owner);
+}
+
+UniqueBuffer
+UniqueBuffer::MakeView(void* DataPtr, uint64_t Size)
+{
+ return UniqueBuffer(new BufferOwner(DataPtr, Size, /* owned */ false));
+}
+
+UniqueBuffer::UniqueBuffer(BufferOwner* Owner) : m_buffer(Owner)
+{
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+SharedBuffer::SharedBuffer(UniqueBuffer&& InBuffer) : m_buffer(std::move(InBuffer.m_buffer))
+{
+}
+
+void
+SharedBuffer::MakeOwned()
+{
+ if (IsOwned() || !m_buffer)
+ return;
+
+ const uint64_t Size = m_buffer->m_Size;
+ void* Buffer = Memory::Alloc(Size, 16);
+ auto NewOwner = new BufferOwner(Buffer, Size, /* owned */ true);
+
+ memcpy(Buffer, m_buffer->m_Data, Size);
+
+ m_buffer = NewOwner;
+}
+
+SharedBuffer
+SharedBuffer::MakeView(MemoryView View, SharedBuffer Buffer)
+{
+ // Todo: verify that view is within the shared buffer
+
+ return SharedBuffer(new BufferOwner(const_cast<void*>(View.GetData()), View.GetSize(), /* owned */ false, Buffer.m_buffer));
+}
+
+SharedBuffer
+SharedBuffer::MakeView(const void* Data, uint64_t Size)
+{
+ return SharedBuffer(new BufferOwner(const_cast<void*>(Data), Size, /* owned */ false));
+}
+
+SharedBuffer
+SharedBuffer::Clone()
+{
+ const uint64_t Size = GetSize();
+ void* Buffer = Memory::Alloc(Size, 16);
+ auto NewOwner = new BufferOwner(Buffer, Size, /* owned */ true);
+ memcpy(Buffer, m_buffer->m_Data, Size);
+
+ return SharedBuffer(NewOwner);
+}
+
+SharedBuffer
+SharedBuffer::Clone(MemoryView View)
+{
+ const uint64_t Size = View.GetSize();
+ void* Buffer = Memory::Alloc(Size, 16);
+ auto NewOwner = new BufferOwner(Buffer, Size, /* owned */ true);
+ memcpy(Buffer, View.GetData(), Size);
+
+ return SharedBuffer(NewOwner);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+sharedbuffer_forcelink()
+{
+}
+
+TEST_CASE("SharedBuffer")
+{
+}
+
+} // namespace zen
diff --git a/zencore/snapshot_manifest.cpp b/zencore/snapshot_manifest.cpp
new file mode 100644
index 000000000..7d0769d13
--- /dev/null
+++ b/zencore/snapshot_manifest.cpp
@@ -0,0 +1,281 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <doctest/doctest.h>
+#include <zencore/snapshot_manifest.h>
+#include <zencore/stream.h>
+#include <zencore/streamutil.h>
+#include <zencore/string.h>
+#include <ostream>
+
+#include <filesystem>
+
+#include <atlbase.h>
+
+// Used for getting My Documents for default snapshot dir
+#include <ShlObj.h>
+#pragma comment(lib, "shell32.lib")
+
+namespace zen {
+
+constexpr const char* magicString = "-=- ZEN_SNAP -=-";
+
+struct SerializedManifestHeader
+{
+ char Magic[16];
+
+ void init() { memcpy(Magic, magicString, sizeof Magic); }
+ bool verify() const { return memcmp(Magic, magicString, sizeof Magic) == 0; }
+};
+
+TextWriter&
+operator<<(TextWriter& Writer, const LeafNode& Leaf)
+{
+ Writer << "modTime: " << Leaf.FileModifiedTime << ", size: " << Leaf.FileSize << ", hash: " << Leaf.ChunkHash << ", name: " << Leaf.Name
+ << "\n";
+
+ return Writer;
+}
+
+BinaryWriter&
+operator<<(BinaryWriter& Writer, const LeafNode& Leaf)
+{
+ Writer << Leaf.FileModifiedTime << Leaf.FileSize << Leaf.ChunkHash << Leaf.Name;
+
+ return Writer;
+}
+
+BinaryReader&
+operator>>(BinaryReader& Reader, LeafNode& Leaf)
+{
+ Reader >> Leaf.FileModifiedTime >> Leaf.FileSize >> Leaf.ChunkHash >> Leaf.Name;
+
+ return Reader;
+}
+
+void
+TreeNode::Finalize()
+{
+ zen::BLAKE3Stream Blake3Stream;
+
+ for (auto& Node : Children)
+ {
+ Node.Finalize();
+ Blake3Stream.Append(Node.ChunkHash.Hash, sizeof Node.ChunkHash);
+ Blake3Stream.Append(Node.Name.data(), Node.Name.size() + 1);
+ }
+
+ for (auto& leaf : Leaves)
+ {
+ Blake3Stream.Append(leaf.ChunkHash.Hash, sizeof leaf.ChunkHash);
+ Blake3Stream.Append(leaf.Name.data(), leaf.Name.size() + 1);
+ }
+
+ this->ChunkHash = Blake3Stream.GetHash();
+}
+
+void
+TreeNode::VisitFiles(std::function<void(const LeafNode& node)> func)
+{
+ for (auto& Node : Children)
+ Node.VisitFiles(func);
+
+ for (auto& Leaf : Leaves)
+ func(Leaf);
+}
+
+void
+TreeNode::VisitModifyFiles(std::function<void(LeafNode& node)> func)
+{
+ for (auto& Node : Children)
+ Node.VisitModifyFiles(func);
+
+ for (auto& Leaf : Leaves)
+ func(Leaf);
+}
+
+IndentTextWriter&
+operator<<(IndentTextWriter& Writer, const TreeNode& Node)
+{
+ Writer << "hash: " << Node.ChunkHash << ", name: " << Node.Name << "\n";
+
+ if (!Node.Leaves.empty())
+ {
+ Writer << "files: "
+ << "\n";
+
+ IndentTextWriter::Scope _(Writer);
+
+ for (const LeafNode& Leaf : Node.Leaves)
+ Writer << Leaf;
+ }
+
+ if (!Node.Children.empty())
+ {
+ Writer << "children: "
+ << "\n";
+
+ IndentTextWriter::Scope _(Writer);
+
+ for (const TreeNode& Child : Node.Children)
+ {
+ Writer << Child;
+ }
+ }
+
+ return Writer;
+}
+
+BinaryWriter&
+operator<<(BinaryWriter& Writer, const TreeNode& Node)
+{
+ Writer << Node.ChunkHash << Node.Name;
+ Writer << uint32_t(Node.Children.size());
+
+ for (const TreeNode& child : Node.Children)
+ Writer << child;
+
+ Writer << uint32_t(Node.Leaves.size());
+
+ for (const LeafNode& Leaf : Node.Leaves)
+ Writer << Leaf;
+
+ return Writer;
+}
+
+BinaryReader&
+operator>>(BinaryReader& Reader, TreeNode& Node)
+{
+ Reader >> Node.ChunkHash >> Node.Name;
+
+ uint32_t ChildCount = 0;
+ Reader >> ChildCount;
+ Node.Children.resize(ChildCount);
+
+ for (TreeNode& Child : Node.Children)
+ Reader >> Child;
+
+ uint32_t LeafCount = 0;
+ Reader >> LeafCount;
+ Node.Leaves.resize(LeafCount);
+
+ for (LeafNode& Leaf : Node.Leaves)
+ Reader >> Leaf;
+
+ return Reader;
+}
+
+void
+SnapshotManifest::finalize()
+{
+ Root.Finalize();
+
+ zen::BLAKE3Stream Blake3Stream;
+
+ Blake3Stream.Append(Root.ChunkHash.Hash, sizeof Root.ChunkHash);
+ Blake3Stream.Append(Root.Name.data(), Root.Name.size() + 1);
+
+ this->ChunkHash = Blake3Stream.GetHash();
+}
+
+void
+WriteManifest(const SnapshotManifest& Manifest, OutStream& ToStream)
+{
+ BinaryWriter Out(ToStream);
+ SerializedManifestHeader Header;
+ Header.init();
+ Out.Write(&Header, sizeof Header);
+
+ Out << Manifest.ChunkHash << Manifest.Id << Manifest.Root;
+}
+
+void
+ReadManifest(SnapshotManifest& Manifest, InStream& FromStream)
+{
+ BinaryReader Reader(FromStream);
+ SerializedManifestHeader Header;
+ Reader.Read(&Header, sizeof Header);
+
+ Reader >> Manifest.ChunkHash >> Manifest.Id >> Manifest.Root;
+}
+
+void
+PrintManifest(const SnapshotManifest& Manifest, OutStream& ToStream)
+{
+ IndentTextWriter Writer(ToStream);
+
+ Writer << "hash: " << Manifest.ChunkHash << "\n";
+ Writer << "id: " << Manifest.Id << "\n";
+ Writer << "root: "
+ << "\n";
+ IndentTextWriter::Scope _(Writer);
+ Writer << Manifest.Root;
+}
+
+std::filesystem::path
+ManifestSpecToPath(const char* ManifestSpec)
+{
+ ExtendableWideStringBuilder<128> ManifestTargetFile;
+
+ if (ManifestSpec[0] == '#')
+ {
+ // Pick sensible default
+
+ WCHAR MyDocumentsDir[MAX_PATH];
+ HRESULT hRes = SHGetFolderPathW(NULL,
+ CSIDL_PERSONAL /* My Documents */,
+ NULL,
+ SHGFP_TYPE_CURRENT,
+ /* out */ MyDocumentsDir);
+
+ if (SUCCEEDED(hRes))
+ {
+ wcscat_s(MyDocumentsDir, L"\\zenfs\\Snapshots\\");
+
+ ManifestTargetFile.Append(MyDocumentsDir);
+ ManifestTargetFile.AppendAscii(ManifestSpec + 1);
+ }
+ }
+ else
+ {
+ ManifestTargetFile.AppendAscii(ManifestSpec);
+ }
+
+ std::filesystem::path ManifestPath{ManifestTargetFile.c_str()};
+
+ if (ManifestPath.extension() != L".zenfs")
+ {
+ ManifestPath.append(L".zenfs");
+ }
+
+ return ManifestPath;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+void
+snapshotmanifest_forcelink()
+{
+}
+
+TEST_CASE("Snapshot manifest")
+{
+ SnapshotManifest Manifest;
+
+ Manifest.Id = "test_manifest";
+ Manifest.ChunkHash = zen::BLAKE3::HashMemory("abcd", 4);
+
+ MemoryOutStream Outstream;
+ WriteManifest(Manifest, Outstream);
+
+ MemoryInStream Instream(Outstream.Data(), Outstream.Size());
+ SnapshotManifest Manifest2;
+ ReadManifest(/* out */ Manifest2, Instream);
+
+ CHECK(Manifest.Id == Manifest2.Id);
+ CHECK(Manifest.ChunkHash == Manifest2.ChunkHash);
+}
+
+} // namespace zen
diff --git a/zencore/stats.cpp b/zencore/stats.cpp
new file mode 100644
index 000000000..f8cdc8fbb
--- /dev/null
+++ b/zencore/stats.cpp
@@ -0,0 +1,73 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/stats.h"
+#include <doctest/doctest.h>
+#include <cmath>
+#include "zencore/timer.h"
+
+//
+// Derived from https://github.com/dln/medida/blob/master/src/medida/stats/ewma.cc
+//
+
+namespace zen {
+
+static constexpr int kTickInterval = 5; // In seconds
+static constexpr double kSecondsPerMinute = 60.0;
+static constexpr int kOneMinute = 1;
+static constexpr int kFiveMinutes = 5;
+static constexpr int kFifteenMinutes = 15;
+
+static double kM1_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kOneMinute);
+static double kM5_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFiveMinutes);
+static double kM15_ALPHA = 1.0 - std::exp(-kTickInterval / kSecondsPerMinute / kFifteenMinutes);
+
+static uint64_t CountPerTick = GetHifreqTimerFrequencySafe() * kTickInterval;
+static uint64_t CountPerSecond = GetHifreqTimerFrequencySafe();
+
+void
+EWMA::Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate)
+{
+ double InstantRate = double(Count) / Interval;
+
+ if (IsInitialUpdate)
+ {
+ m_rate = InstantRate;
+ }
+ else
+ {
+ m_rate += Alpha * (InstantRate - m_rate);
+ }
+}
+
+double
+EWMA::Rate() const
+{
+ return m_rate * CountPerSecond;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+TEST_CASE("Stats")
+{
+ SUBCASE("Simple")
+ {
+ EWMA ewma1;
+ ewma1.Tick(kM1_ALPHA, CountPerSecond, 5, true);
+
+ CHECK(ewma1.Rate() - 5 < 0.001);
+
+ for (int i = 0; i < 60; ++i)
+ ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false);
+
+ CHECK(ewma1.Rate() - 10 < 0.001);
+
+ ewma1.Tick(kM1_ALPHA, CountPerSecond, 10, false);
+ }
+}
+
+void
+stats_forcelink()
+{
+}
+
+} // namespace zen
diff --git a/zencore/stream.cpp b/zencore/stream.cpp
new file mode 100644
index 000000000..2cda0e123
--- /dev/null
+++ b/zencore/stream.cpp
@@ -0,0 +1,307 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <doctest/doctest.h>
+#include <stdarg.h>
+#include <zencore/memory.h>
+#include <zencore/stream.h>
+#include <algorithm>
+#include <exception>
+
+namespace zen {
+
+MemoryInStream::MemoryInStream(const void* buffer, size_t size)
+: m_Buffer(reinterpret_cast<const uint8_t*>(buffer), reinterpret_cast<const uint8_t*>(buffer) + size)
+{
+}
+
+void
+MemoryInStream::Read(void* buffer, size_t byteCount, uint64_t offset)
+{
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ const size_t needEnd = offset + byteCount;
+
+ if (needEnd > m_Buffer.size())
+ throw std::exception("read past end of file!"); // TODO: better exception
+
+ memcpy(buffer, m_Buffer.data() + offset, byteCount);
+}
+
+void
+MemoryOutStream::Write(const void* data, size_t byteCount, uint64_t offset)
+{
+ RwLock::ExclusiveLockScope _(m_Lock);
+
+ const size_t needEnd = offset + byteCount;
+
+ if (needEnd > m_Buffer.size())
+ m_Buffer.resize(needEnd);
+
+ memcpy(m_Buffer.data() + offset, data, byteCount);
+}
+
+void
+MemoryOutStream::Flush()
+{
+ // No-op
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+TextWriter::TextWriter(OutStream& stream) : m_Stream(&stream)
+{
+}
+
+TextWriter::~TextWriter() = default;
+
+void
+TextWriter::Write(const void* data, size_t byteCount)
+{
+ m_Stream->Write(data, byteCount, m_CurrentOffset);
+ m_CurrentOffset += byteCount;
+}
+
+TextWriter&
+operator<<(TextWriter& Writer, const char* value)
+{
+ if (value)
+ Writer.Write(value, strlen(value));
+ else
+ Writer.Write("(null)", 6);
+
+ return Writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, const std::string_view& value)
+{
+ writer.Write(value.data(), value.size());
+
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, bool value)
+{
+ if (value)
+ writer.Write("true", 4);
+ else
+ writer.Write("false", 5);
+
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, int8_t value)
+{
+ char buffer[16];
+ _itoa_s(value, buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, int16_t value)
+{
+ char buffer[16];
+ _itoa_s(value, buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, int32_t value)
+{
+ char buffer[16];
+ _itoa_s(value, buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, int64_t value)
+{
+ char buffer[32];
+ _i64toa_s(value, buffer, sizeof buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, uint8_t value)
+{
+ char buffer[16];
+ _ultoa_s(value, buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, uint16_t value)
+{
+ char buffer[16];
+ _ultoa_s(value, buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, uint32_t value)
+{
+ char buffer[16];
+ _ultoa_s(value, buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, uint64_t value)
+{
+ char buffer[32];
+ _ui64toa_s(value, buffer, sizeof buffer, 10);
+ writer << buffer;
+ return writer;
+}
+
+void
+TextWriter::Writef(const char* formatString, ...)
+{
+ va_list args;
+ va_start(args, formatString);
+
+ char* tempBuffer = nullptr;
+ char buffer[4096];
+ int rv = vsnprintf(buffer, sizeof buffer, formatString, args);
+
+ ZEN_ASSERT(rv >= 0);
+
+ if (rv > sizeof buffer)
+ {
+ // Need more room -- allocate temporary buffer
+
+ tempBuffer = (char*)Memory::Alloc(rv + 1, 8);
+
+ int rv2 = vsnprintf(tempBuffer, rv + 1, formatString, args);
+
+ ZEN_ASSERT(rv >= 0);
+ ZEN_ASSERT(rv2 <= rv);
+
+ rv = rv2;
+ }
+
+ m_Stream->Write(tempBuffer ? tempBuffer : buffer, rv, m_CurrentOffset);
+ m_CurrentOffset += rv;
+
+ if (tempBuffer)
+ Memory::Free(tempBuffer);
+
+ va_end(args);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+IndentTextWriter::IndentTextWriter(OutStream& stream) : TextWriter(stream)
+{
+}
+
+IndentTextWriter::~IndentTextWriter()
+{
+}
+
+void
+IndentTextWriter::Write(const void* data, size_t byteCount)
+{
+ const uint8_t* src = reinterpret_cast<const uint8_t*>(data);
+ int cur = m_LineCursor;
+
+ while (byteCount)
+ {
+ char c = *src++;
+
+ if (cur == 0)
+ {
+ const char indentSpaces[] =
+ " "
+ " ";
+
+ cur = std::min<int>(m_IndentAmount, sizeof indentSpaces - 1);
+ memcpy(m_LineBuffer, indentSpaces, cur);
+ }
+
+ m_LineBuffer[cur++] = c;
+ --byteCount;
+
+ if (c == '\n' || cur == sizeof m_LineBuffer)
+ {
+ TextWriter::Write(m_LineBuffer, cur);
+
+ cur = 0;
+ }
+ }
+
+ m_LineCursor = cur;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+void
+stream_forcelink()
+{
+}
+
+TEST_CASE("BinaryWriter and BinaryWriter")
+{
+ MemoryOutStream stream;
+ BinaryWriter writer(stream);
+
+ CHECK(writer.CurrentOffset() == 0);
+
+ writer.Write("foo!", 4);
+ CHECK(writer.CurrentOffset() == 4);
+
+ writer << uint8_t(42) << uint16_t(42) << uint32_t(42) << uint64_t(42);
+ writer << int8_t(42) << int16_t(42) << int32_t(42) << int64_t(42);
+
+ CHECK(writer.CurrentOffset() == (4 + 15 * 2));
+
+ // Read the data back
+
+ MemoryInStream instream(stream.Data(), stream.Size());
+ BinaryReader reader(instream);
+ CHECK(reader.CurrentOffset() == 0);
+
+ char buffer[4];
+ reader.Read(buffer, 4);
+ CHECK(reader.CurrentOffset() == 4);
+
+ CHECK(memcmp(buffer, "foo!", 4) == 0);
+
+ uint8_t ui8 = 0;
+ uint16_t ui16 = 0;
+ uint32_t ui32 = 0;
+ uint64_t ui64 = 0;
+ int8_t i8 = 0;
+ int16_t i16 = 0;
+ int32_t i32 = 0;
+ int64_t i64 = 0;
+
+ reader >> ui8 >> ui16 >> ui32 >> ui64;
+ reader >> i8 >> i16 >> i32 >> i64;
+
+ CHECK(reader.CurrentOffset() == (4 + 15 * 2));
+
+ CHECK(ui8 == 42);
+ CHECK(ui16 == 42);
+ CHECK(ui32 == 42);
+ CHECK(ui64 == 42);
+
+ CHECK(i8 == 42);
+ CHECK(i16 == 42);
+ CHECK(i32 == 42);
+ CHECK(i64 == 42);
+}
+
+} // namespace zen
diff --git a/zencore/streamutil.cpp b/zencore/streamutil.cpp
new file mode 100644
index 000000000..d3ed5ceaa
--- /dev/null
+++ b/zencore/streamutil.cpp
@@ -0,0 +1,104 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/streamutil.h>
+#include <zencore/string.h>
+
+namespace zen {
+
+BinaryWriter&
+operator<<(BinaryWriter& writer, const std::string_view& value)
+{
+ writer.Write(value.data(), value.size());
+ writer << uint8_t(0);
+
+ return writer;
+}
+
+BinaryReader&
+operator>>(BinaryReader& reader, std::string& value)
+{
+ for (;;)
+ {
+ uint8_t x;
+ reader.Read(&x, 1);
+
+ if (x == 0)
+ return reader;
+
+ value.push_back(char(x));
+ }
+}
+
+BinaryWriter&
+operator<<(BinaryWriter& writer, const std::wstring_view& value)
+{
+ // write as utf8
+
+ ExtendableStringBuilder<128> utf8;
+ WideToUtf8(value, utf8);
+
+ writer.Write(utf8.c_str(), utf8.Size() + 1);
+
+ return writer;
+}
+
+BinaryReader&
+operator>>(BinaryReader& reader, std::wstring& value)
+{
+ // read as utf8
+
+ std::string v8;
+ reader >> v8;
+
+ ExtendableWideStringBuilder<128> wstr;
+ Utf8ToWide(v8, wstr);
+
+ value = wstr.c_str();
+
+ return reader;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, const zen::SHA1& value)
+{
+ zen::SHA1::String_t buffer;
+ value.ToHexString(buffer);
+
+ writer.Write(buffer, zen::SHA1::StringLength);
+
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, const zen::BLAKE3& value)
+{
+ zen::BLAKE3::String_t buffer;
+ value.ToHexString(buffer);
+
+ writer.Write(buffer, zen::BLAKE3::StringLength);
+
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, const zen::IoHash& value)
+{
+ zen::IoHash::String_t buffer;
+ value.ToHexString(buffer);
+
+ writer.Write(buffer, zen::IoHash::StringLength);
+
+ return writer;
+}
+
+TextWriter&
+operator<<(TextWriter& writer, const std::wstring_view& value)
+{
+ ExtendableStringBuilder<128> v8;
+ WideToUtf8(value, v8);
+
+ writer.Write(v8.c_str(), v8.Size());
+ return writer;
+}
+
+} // namespace zen
diff --git a/zencore/string.cpp b/zencore/string.cpp
new file mode 100644
index 000000000..b6093ac2e
--- /dev/null
+++ b/zencore/string.cpp
@@ -0,0 +1,913 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <doctest/doctest.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <zencore/memory.h>
+#include <zencore/string.h>
+#include <exception>
+#include <ostream>
+
+#include <utf8.h>
+
+template<typename u16bit_iterator>
+void
+utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuilderBase& OutString)
+{
+ while (StartIt != EndIt)
+ {
+ uint32_t cp = utf8::internal::mask16(*StartIt++);
+ // Take care of surrogate pairs first
+ if (utf8::internal::is_lead_surrogate(cp))
+ {
+ uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++);
+ cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET;
+ }
+ OutString.AppendCodepoint(cp);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+bool
+ToString(std::span<char> Buffer, uint64_t Num)
+{
+ snprintf(Buffer.data(), Buffer.size(), "%I64u", Num);
+
+ return true;
+}
+bool
+ToString(std::span<char> Buffer, int64_t Num)
+{
+ snprintf(Buffer.data(), Buffer.size(), "%I64d", Num);
+
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+const char*
+FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch)
+{
+ const size_t PathLen = Path.size();
+
+ if (ExtensionToMatch)
+ {
+ size_t ExtLen = strlen(ExtensionToMatch);
+
+ if (ExtLen > PathLen)
+ return nullptr;
+
+ const char* PathExtension = Path.data() + PathLen - ExtLen;
+
+ if (StringEquals(PathExtension, ExtensionToMatch))
+ return PathExtension;
+
+ return nullptr;
+ }
+
+ if (PathLen == 0)
+ return nullptr;
+
+ // Look for extension introducer ('.')
+
+ for (size_t i = PathLen - 1; i >= 0; --i)
+ {
+ if (Path[i] == '.')
+ return Path.data() + i;
+ }
+
+ return nullptr;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+Utf8ToWide(const char8_t* Str8, WideStringBuilderBase& OutString)
+{
+ Utf8ToWide(std::u8string_view(Str8), OutString);
+}
+
+void
+Utf8ToWide(const std::string_view& Str8, WideStringBuilderBase& OutString)
+{
+ Utf8ToWide(std::u8string_view{reinterpret_cast<const char8_t*>(Str8.data()), Str8.size()}, OutString);
+}
+
+std::wstring
+Utf8ToWide(const std::string_view& Wstr)
+{
+ ExtendableWideStringBuilder<128> String;
+ Utf8ToWide(Wstr, String);
+
+ return String.c_str();
+}
+
+void
+Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString)
+{
+ const char* str = (const char*)Str8.data();
+ const size_t strLen = Str8.size();
+
+ const char* endStr = str + strLen;
+ size_t ByteCount = 0;
+ size_t CurrentOutChar = 0;
+
+ for (; str != endStr; ++str)
+ {
+ unsigned char Data = static_cast<unsigned char>(*str);
+
+ if (!(Data & 0x80))
+ {
+ // ASCII
+ OutString.Append(wchar_t(Data));
+ continue;
+ }
+ else if (!ByteCount)
+ {
+ // Start of multi-byte sequence. Figure out how
+ // many bytes we're going to consume
+
+ size_t Count = 0;
+
+ for (size_t Temp = Data; Temp & 0x80; Temp <<= 1)
+ ++Count;
+
+ ByteCount = Count - 1;
+ CurrentOutChar = Data & (0xff >> (Count + 1));
+ }
+ else
+ {
+ --ByteCount;
+
+ if ((Data & 0xc0) != 0x80)
+ {
+ break;
+ }
+
+ CurrentOutChar = (CurrentOutChar << 6) | (Data & 0x3f);
+
+ if (!ByteCount)
+ {
+ OutString.Append(wchar_t(CurrentOutChar));
+ CurrentOutChar = 0;
+ }
+ }
+ }
+}
+
+void
+WideToUtf8(const wchar_t* Wstr, StringBuilderBase& OutString)
+{
+ WideToUtf8(std::u16string_view{(char16_t*)Wstr}, OutString);
+}
+
+void
+WideToUtf8(const std::wstring_view& Wstr, StringBuilderBase& OutString)
+{
+ WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, OutString);
+}
+
+void
+WideToUtf8(const std::u16string_view& Wstr, StringBuilderBase& OutString)
+{
+ utf16to8_impl(begin(Wstr), end(Wstr), OutString);
+}
+
+std::string
+WideToUtf8(const wchar_t* Wstr)
+{
+ ExtendableStringBuilder<128> String;
+ WideToUtf8(std::u16string_view{(char16_t*)Wstr}, String);
+
+ return String.c_str();
+}
+
+std::string
+WideToUtf8(const std::wstring_view Wstr)
+{
+ ExtendableStringBuilder<128> String;
+ WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, String);
+
+ return String.c_str();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+enum NicenumFormat
+{
+ kNicenum1024 = 0, // Print kilo, mega, tera, peta, exa..
+ kNicenumBytes = 1, // Print single bytes ("13B"), kilo, mega, tera...
+ kNicenumTime = 2, // Print nanosecs, microsecs, millisecs, seconds...
+ kNicenumRaw = 3, // Print the raw number without any formatting
+ kNicenumRawTime = 4 // Same as RAW, but print dashes ('-') for zero.
+};
+
+namespace {
+ static const char* UnitStrings[3][7] = {
+ /* kNicenum1024 */ {"", "K", "M", "G", "T", "P", "E"},
+ /* kNicenumBytes */ {"B", "K", "M", "G", "T", "P", "E"},
+ /* kNicenumTime */ {"ns", "us", "ms", "s", "?", "?", "?"}};
+
+ static const int UnitsLen[] = {
+ /* kNicenum1024 */ 6,
+ /* kNicenumBytes */ 6,
+ /* kNicenumTime */ 3};
+
+ static const uint64_t KiloUnit[] = {
+ /* kNicenum1024 */ 1024,
+ /* kNicenumBytes */ 1024,
+ /* kNicenumTime */ 1000};
+} // namespace
+
+/*
+ * Convert a number to an appropriately human-readable output.
+ */
+int
+NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format)
+{
+ switch (Format)
+ {
+ case kNicenumRaw:
+ return snprintf(Buffer.data(), Buffer.size(), "%llu", (uint64_t)Num);
+
+ case kNicenumRawTime:
+ if (Num > 0)
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%llu", (uint64_t)Num);
+ }
+ else
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%s", "-");
+ }
+ break;
+
+ case kNicenum1024:
+ case kNicenumBytes:
+ case kNicenumTime:
+ default:
+ break;
+ }
+
+ // Bring into range and select unit
+
+ int Index = 0;
+ uint64_t n = Num;
+
+ {
+ const uint64_t Unit = KiloUnit[Format];
+ const int maxIndex = UnitsLen[Format];
+
+ while (n >= Unit && Index < maxIndex)
+ {
+ n /= Unit;
+ Index++;
+ }
+ }
+
+ const char* u = UnitStrings[Format][Index];
+
+ if ((Index == 0) || ((Num % (uint64_t)powl((int)KiloUnit[Format], Index)) == 0))
+ {
+ /*
+ * If this is an even multiple of the base, always display
+ * without any decimal precision.
+ */
+ return snprintf(Buffer.data(), Buffer.size(), "%llu%s", (uint64_t)n, u);
+ }
+ else
+ {
+ /*
+ * We want to choose a precision that reflects the best choice
+ * for fitting in 5 characters. This can get rather tricky when
+ * we have numbers that are very close to an order of magnitude.
+ * For example, when displaying 10239 (which is really 9.999K),
+ * we want only a single place of precision for 10.0K. We could
+ * develop some complex heuristics for this, but it's much
+ * easier just to try each combination in turn.
+ */
+
+ int StrLen = 0;
+
+ for (int i = 2; i >= 0; i--)
+ {
+ double Value = (double)Num / (uint64_t)powl((int)KiloUnit[Format], Index);
+
+ /*
+ * Don't print floating point values for time. Note,
+ * we use floor() instead of round() here, since
+ * round can result in undesirable results. For
+ * example, if "num" is in the range of
+ * 999500-999999, it will print out "1000us". This
+ * doesn't happen if we use floor().
+ */
+ if (Format == kNicenumTime)
+ {
+ StrLen = snprintf(Buffer.data(), Buffer.size(), "%d%s", (unsigned int)floor(Value), u);
+
+ if (StrLen <= 5)
+ break;
+ }
+ else
+ {
+ StrLen = snprintf(Buffer.data(), Buffer.size(), "%.*f%s", i, Value, u);
+
+ if (StrLen <= 5)
+ break;
+ }
+ }
+
+ return StrLen;
+ }
+}
+
+size_t
+NiceNumToBuffer(uint64_t Num, std::span<char> Buffer)
+{
+ return NiceNumGeneral(Num, Buffer, kNicenum1024);
+}
+
+size_t
+NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer)
+{
+ return NiceNumGeneral(Num, Buffer, kNicenumBytes);
+}
+
+size_t
+NiceByteRateToBuffer(uint64_t Num, uint64_t ElapsedMs, std::span<char> Buffer)
+{
+ size_t n = NiceNumGeneral(Num * 1000 / ElapsedMs, Buffer, kNicenumBytes);
+
+ Buffer[n++] = '/';
+ Buffer[n++] = 's';
+ Buffer[n++] = '\0';
+
+ return n;
+}
+
+size_t
+NiceLatencyNsToBuffer(uint64_t Nanos, std::span<char> Buffer)
+{
+ return NiceNumGeneral(Nanos, Buffer, kNicenumTime);
+}
+
+size_t
+NiceTimeSpanMsToBuffer(uint64_t Millis, std::span<char> Buffer)
+{
+ if (Millis < 1000)
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "ms", Millis);
+ }
+ else if (Millis < 10000)
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%.2fs", Millis / 1000.0);
+ }
+ else if (Millis < 60000)
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%.1fs", Millis / 1000.0);
+ }
+ else if (Millis < 60 * 60000)
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "m%02" PRIu64 "s", Millis / 60000, (Millis / 1000) % 60);
+ }
+ else
+ {
+ return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "h%02" PRIu64 "m", Millis / 3600000, (Millis / 60000) % 60);
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template<typename C>
+StringBuilderImpl<C>::~StringBuilderImpl()
+{
+ if (m_IsDynamic)
+ {
+ FreeBuffer(m_Base, m_End - m_Base);
+ }
+}
+
+template<typename C>
+void
+StringBuilderImpl<C>::Extend(size_t extraCapacity)
+{
+ if (!m_IsExtendable)
+ {
+ Fail("exceeded capacity");
+ }
+
+ const size_t oldCapacity = m_End - m_Base;
+ const size_t newCapacity = NextPow2(oldCapacity + extraCapacity);
+
+ C* newBase = (C*)AllocBuffer(newCapacity);
+
+ size_t pos = m_CurPos - m_Base;
+ memcpy(newBase, m_Base, pos * sizeof(C));
+
+ if (m_IsDynamic)
+ {
+ FreeBuffer(m_Base, oldCapacity);
+ }
+
+ m_Base = newBase;
+ m_CurPos = newBase + pos;
+ m_End = newBase + newCapacity;
+ m_IsDynamic = true;
+}
+
+template<typename C>
+void*
+StringBuilderImpl<C>::AllocBuffer(size_t byteCount)
+{
+ return Memory::Alloc(byteCount * sizeof(C));
+}
+
+template<typename C>
+void
+StringBuilderImpl<C>::FreeBuffer(void* buffer, size_t byteCount)
+{
+ ZEN_UNUSED(byteCount);
+
+ Memory::Free(buffer);
+}
+
+template<typename C>
+[[noreturn]] void
+StringBuilderImpl<C>::Fail(const char* reason)
+{
+ throw std::exception(reason);
+}
+
+// Instantiate templates once
+
+template class StringBuilderImpl<char>;
+template class StringBuilderImpl<wchar_t>;
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Unit tests
+//
+
+TEST_CASE("niceNum")
+{
+ char Buffer[16];
+
+ SUBCASE("raw")
+ {
+ NiceNumGeneral(1, Buffer, kNicenumRaw);
+ CHECK(StringEquals(Buffer, "1"));
+
+ NiceNumGeneral(10, Buffer, kNicenumRaw);
+ CHECK(StringEquals(Buffer, "10"));
+
+ NiceNumGeneral(100, Buffer, kNicenumRaw);
+ CHECK(StringEquals(Buffer, "100"));
+
+ NiceNumGeneral(1000, Buffer, kNicenumRaw);
+ CHECK(StringEquals(Buffer, "1000"));
+
+ NiceNumGeneral(10000, Buffer, kNicenumRaw);
+ CHECK(StringEquals(Buffer, "10000"));
+
+ NiceNumGeneral(100000, Buffer, kNicenumRaw);
+ CHECK(StringEquals(Buffer, "100000"));
+ }
+
+ SUBCASE("1024")
+ {
+ NiceNumGeneral(1, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1"));
+
+ NiceNumGeneral(10, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "10"));
+
+ NiceNumGeneral(100, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "100"));
+
+ NiceNumGeneral(1000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1000"));
+
+ NiceNumGeneral(10000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.77K"));
+
+ NiceNumGeneral(100000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "97.7K"));
+
+ NiceNumGeneral(1000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "977K"));
+
+ NiceNumGeneral(10000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.54M"));
+
+ NiceNumGeneral(100000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "95.4M"));
+
+ NiceNumGeneral(1000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "954M"));
+
+ NiceNumGeneral(10000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.31G"));
+
+ NiceNumGeneral(100000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "93.1G"));
+
+ NiceNumGeneral(1000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "931G"));
+
+ NiceNumGeneral(10000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.09T"));
+
+ NiceNumGeneral(100000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "90.9T"));
+
+ NiceNumGeneral(1000000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "909T"));
+
+ NiceNumGeneral(10000000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "8.88P"));
+
+ NiceNumGeneral(100000000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "88.8P"));
+
+ NiceNumGeneral(1000000000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "888P"));
+
+ NiceNumGeneral(10000000000000000000, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "8.67E"));
+
+ // pow2
+
+ NiceNumGeneral(0, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "0"));
+
+ NiceNumGeneral(1, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1"));
+
+ NiceNumGeneral(1024, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1K"));
+
+ NiceNumGeneral(1024 * 1024, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1M"));
+
+ NiceNumGeneral(1024 * 1024 * 1024, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1G"));
+
+ NiceNumGeneral(1024llu * 1024 * 1024 * 1024, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1T"));
+
+ NiceNumGeneral(1024llu * 1024 * 1024 * 1024 * 1024, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1P"));
+
+ NiceNumGeneral(1024llu * 1024 * 1024 * 1024 * 1024 * 1024, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1E"));
+
+ // pow2-1
+
+ NiceNumGeneral(1023, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "1023"));
+
+ NiceNumGeneral(2047, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "2.00K"));
+
+ NiceNumGeneral(9 * 1024 - 1, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.00K"));
+
+ NiceNumGeneral(10 * 1024 - 1, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "10.0K"));
+
+ NiceNumGeneral(10 * 1024 - 5, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "10.0K"));
+
+ NiceNumGeneral(10 * 1024 - 6, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.99K"));
+
+ NiceNumGeneral(10 * 1024 - 10, Buffer, kNicenum1024);
+ CHECK(StringEquals(Buffer, "9.99K"));
+ }
+
+ SUBCASE("time")
+ {
+ NiceNumGeneral(1, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "1ns"));
+
+ NiceNumGeneral(100, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "100ns"));
+
+ NiceNumGeneral(1000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "1us"));
+
+ NiceNumGeneral(10000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "10us"));
+
+ NiceNumGeneral(100000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "100us"));
+
+ NiceNumGeneral(1000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "1ms"));
+
+ NiceNumGeneral(10000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "10ms"));
+
+ NiceNumGeneral(100000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "100ms"));
+
+ NiceNumGeneral(1000000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "1s"));
+
+ NiceNumGeneral(10000000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "10s"));
+
+ NiceNumGeneral(100000000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "100s"));
+
+ NiceNumGeneral(1000000000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "1000s"));
+
+ NiceNumGeneral(10000000000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "10000s"));
+
+ NiceNumGeneral(100000000000000, Buffer, kNicenumTime);
+ CHECK(StringEquals(Buffer, "100000s"));
+ }
+
+ SUBCASE("bytes")
+ {
+ NiceNumGeneral(1, Buffer, kNicenumBytes);
+ CHECK(StringEquals(Buffer, "1B"));
+
+ NiceNumGeneral(10, Buffer, kNicenumBytes);
+ CHECK(StringEquals(Buffer, "10B"));
+
+ NiceNumGeneral(100, Buffer, kNicenumBytes);
+ CHECK(StringEquals(Buffer, "100B"));
+
+ NiceNumGeneral(1000, Buffer, kNicenumBytes);
+ CHECK(StringEquals(Buffer, "1000B"));
+
+ NiceNumGeneral(10000, Buffer, kNicenumBytes);
+ CHECK(StringEquals(Buffer, "9.77K"));
+ }
+
+ SUBCASE("byteRate")
+ {
+ NiceByteRateToBuffer(1, 1, Buffer);
+ CHECK(StringEquals(Buffer, "1000B/s"));
+
+ NiceByteRateToBuffer(1000, 1000, Buffer);
+ CHECK(StringEquals(Buffer, "1000B/s"));
+
+ NiceByteRateToBuffer(1024, 1, Buffer);
+ CHECK(StringEquals(Buffer, "1000K/s"));
+
+ NiceByteRateToBuffer(1024, 1000, Buffer);
+ CHECK(StringEquals(Buffer, "1K/s"));
+ }
+
+ SUBCASE("timespan")
+ {
+ NiceTimeSpanMsToBuffer(1, Buffer);
+ CHECK(StringEquals(Buffer, "1ms"));
+
+ NiceTimeSpanMsToBuffer(900, Buffer);
+ CHECK(StringEquals(Buffer, "900ms"));
+
+ NiceTimeSpanMsToBuffer(1000, Buffer);
+ CHECK(StringEquals(Buffer, "1.00s"));
+
+ NiceTimeSpanMsToBuffer(1900, Buffer);
+ CHECK(StringEquals(Buffer, "1.90s"));
+
+ NiceTimeSpanMsToBuffer(19000, Buffer);
+ CHECK(StringEquals(Buffer, "19.0s"));
+
+ NiceTimeSpanMsToBuffer(60000, Buffer);
+ CHECK(StringEquals(Buffer, "1m00s"));
+
+ NiceTimeSpanMsToBuffer(600000, Buffer);
+ CHECK(StringEquals(Buffer, "10m00s"));
+
+ NiceTimeSpanMsToBuffer(3600000, Buffer);
+ CHECK(StringEquals(Buffer, "1h00m"));
+
+ NiceTimeSpanMsToBuffer(36000000, Buffer);
+ CHECK(StringEquals(Buffer, "10h00m"));
+
+ NiceTimeSpanMsToBuffer(360000000, Buffer);
+ CHECK(StringEquals(Buffer, "100h00m"));
+ }
+}
+
+void
+string_forcelink()
+{
+}
+
+TEST_CASE("StringBuilder")
+{
+ StringBuilder<64> sb;
+
+ SUBCASE("Empty init")
+ {
+ const char* str = sb.c_str();
+
+ CHECK(StringLength(str) == 0);
+ }
+
+ SUBCASE("Append single character")
+ {
+ sb.Append('a');
+
+ const char* str = sb.c_str();
+ CHECK(StringLength(str) == 1);
+ CHECK(str[0] == 'a');
+
+ sb.Append('b');
+ str = sb.c_str();
+ CHECK(StringLength(str) == 2);
+ CHECK(str[0] == 'a');
+ CHECK(str[1] == 'b');
+ }
+
+ SUBCASE("Append string")
+ {
+ sb.Append("a");
+
+ const char* str = sb.c_str();
+ CHECK(StringLength(str) == 1);
+ CHECK(str[0] == 'a');
+
+ sb.Append("b");
+ str = sb.c_str();
+ CHECK(StringLength(str) == 2);
+ CHECK(str[0] == 'a');
+ CHECK(str[1] == 'b');
+
+ sb.Append("cdefghijklmnopqrstuvwxyz");
+ CHECK(sb.Size() == 26);
+
+ sb.Append("abcdefghijklmnopqrstuvwxyz");
+ CHECK(sb.Size() == 52);
+
+ sb.Append("abcdefghijk");
+ CHECK(sb.Size() == 63);
+ }
+}
+
+TEST_CASE("ExtendableStringBuilder")
+{
+ ExtendableStringBuilder<16> sb;
+
+ SUBCASE("Empty init")
+ {
+ const char* str = sb.c_str();
+
+ CHECK(StringLength(str) == 0);
+ }
+
+ SUBCASE("Short append")
+ {
+ sb.Append("abcd");
+ CHECK(sb.IsDynamic() == false);
+ }
+
+ SUBCASE("Short+long append")
+ {
+ sb.Append("abcd");
+ CHECK(sb.IsDynamic() == false);
+ // This should trigger a dynamic buffer allocation since the required
+ // capacity exceeds the internal fixed buffer.
+ sb.Append("abcdefghijklmnopqrstuvwxyz");
+ CHECK(sb.IsDynamic() == true);
+ CHECK(sb.Size() == 30);
+ CHECK(sb.Size() == StringLength(sb.c_str()));
+ }
+}
+
+TEST_CASE("WideStringBuilder")
+{
+ WideStringBuilder<64> sb;
+
+ SUBCASE("Empty init")
+ {
+ const wchar_t* str = sb.c_str();
+
+ CHECK(StringLength(str) == 0);
+ }
+
+ SUBCASE("Append single character")
+ {
+ sb.Append(L'a');
+
+ const wchar_t* str = sb.c_str();
+ CHECK(StringLength(str) == 1);
+ CHECK(str[0] == L'a');
+
+ sb.Append(L'b');
+ str = sb.c_str();
+ CHECK(StringLength(str) == 2);
+ CHECK(str[0] == L'a');
+ CHECK(str[1] == L'b');
+ }
+
+ SUBCASE("Append string")
+ {
+ sb.Append(L"a");
+
+ const wchar_t* str = sb.c_str();
+ CHECK(StringLength(str) == 1);
+ CHECK(str[0] == L'a');
+
+ sb.Append(L"b");
+ str = sb.c_str();
+ CHECK(StringLength(str) == 2);
+ CHECK(str[0] == L'a');
+ CHECK(str[1] == L'b');
+
+ sb.Append(L"cdefghijklmnopqrstuvwxyz");
+ CHECK(sb.Size() == 26);
+
+ sb.Append(L"abcdefghijklmnopqrstuvwxyz");
+ CHECK(sb.Size() == 52);
+
+ sb.Append(L"abcdefghijk");
+ CHECK(sb.Size() == 63);
+ }
+}
+
+TEST_CASE("ExtendableWideStringBuilder")
+{
+ ExtendableWideStringBuilder<16> sb;
+
+ SUBCASE("Empty init")
+ {
+ CHECK(sb.Size() == 0);
+
+ const wchar_t* str = sb.c_str();
+ CHECK(StringLength(str) == 0);
+ }
+
+ SUBCASE("Short append")
+ {
+ sb.Append(L"abcd");
+ CHECK(sb.IsDynamic() == false);
+ }
+
+ SUBCASE("Short+long append")
+ {
+ sb.Append(L"abcd");
+ CHECK(sb.IsDynamic() == false);
+ // This should trigger a dynamic buffer allocation since the required
+ // capacity exceeds the internal fixed buffer.
+ sb.Append(L"abcdefghijklmnopqrstuvwxyz");
+ CHECK(sb.IsDynamic() == true);
+ CHECK(sb.Size() == 30);
+ CHECK(sb.Size() == StringLength(sb.c_str()));
+ }
+}
+
+TEST_CASE("utf8")
+{
+ SUBCASE("utf8towide")
+ {
+ // TODO: add more extensive testing here - this covers a very small space
+
+ WideStringBuilder<32> wout;
+ Utf8ToWide(u8"abcdefghi", wout);
+ CHECK(StringEquals(L"abcdefghi", wout.c_str()));
+
+ wout.Reset();
+
+ Utf8ToWide(u8"abc���", wout);
+ CHECK(StringEquals(L"abc���", wout.c_str()));
+ }
+
+ SUBCASE("widetoutf8")
+ {
+ // TODO: add more extensive testing here - this covers a very small space
+
+ StringBuilder<32> out;
+
+ WideToUtf8(L"abcdefghi", out);
+ CHECK(StringEquals("abcdefghi", out.c_str()));
+
+ out.Reset();
+
+ WideToUtf8(L"abc���", out);
+ CHECK(StringEquals(u8"abc���", out.c_str()));
+ }
+}
+
+TEST_CASE("filepath")
+{
+ CHECK(FilepathFindExtension("foo\\bar\\baz.txt", ".txt") != nullptr);
+ CHECK(FilepathFindExtension("foo\\bar\\baz.txt", ".zap") == nullptr);
+
+ CHECK(FilepathFindExtension("foo\\bar\\baz.txt") != nullptr);
+ CHECK(FilepathFindExtension("foo\\bar\\baz.txt") == std::string_view(".txt"));
+
+ CHECK(FilepathFindExtension(".txt") == std::string_view(".txt"));
+}
+
+} // namespace zen
diff --git a/zencore/thread.cpp b/zencore/thread.cpp
new file mode 100644
index 000000000..80cf6f100
--- /dev/null
+++ b/zencore/thread.cpp
@@ -0,0 +1,192 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/thread.h>
+
+#include <zencore/except.h>
+#include <zencore/string.h>
+#include <zencore/windows.h>
+#include <thread>
+
+namespace zen {
+
+void
+RwLock::AcquireShared()
+{
+ AcquireSRWLockShared((PSRWLOCK)&m_Srw);
+}
+
+void
+RwLock::ReleaseShared()
+{
+ ReleaseSRWLockShared((PSRWLOCK)&m_Srw);
+}
+
+void
+RwLock::AcquireExclusive()
+{
+ AcquireSRWLockExclusive((PSRWLOCK)&m_Srw);
+}
+
+void
+RwLock::ReleaseExclusive()
+{
+ ReleaseSRWLockExclusive((PSRWLOCK)&m_Srw);
+}
+
+Event::Event()
+{
+ m_EventHandle = CreateEvent(nullptr, true, false, nullptr);
+}
+
+Event::~Event()
+{
+ CloseHandle(m_EventHandle);
+}
+
+void
+Event::Set()
+{
+ SetEvent(m_EventHandle);
+}
+
+void
+Event::Reset()
+{
+ ResetEvent(m_EventHandle);
+}
+
+bool
+Event::Wait(int TimeoutMs)
+{
+ const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs;
+
+ DWORD Result = WaitForSingleObject(m_EventHandle, Timeout);
+
+ if (Result == WAIT_FAILED)
+ {
+ throw WindowsException("Event wait failed");
+ }
+
+ return (Result == WAIT_OBJECT_0);
+}
+
+NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr)
+{
+ using namespace std::literals;
+
+ ExtendableStringBuilder<64> Name;
+ Name << "Local\\"sv;
+ Name << EventName;
+
+ m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str());
+}
+
+NamedEvent::NamedEvent(std::string_view EventName) : Event(nullptr)
+{
+ using namespace std::literals;
+
+ ExtendableStringBuilder<64> Name;
+ Name << "Local\\"sv;
+ Name << EventName;
+
+ m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str());
+}
+
+Process::Process() = default;
+
+void
+Process::Initialize(void* ProcessHandle)
+{
+ ZEN_ASSERT(m_ProcessHandle == nullptr);
+ // TODO: perform some debug verification here to verify it's a valid handle?
+ m_ProcessHandle = ProcessHandle;
+}
+
+Process::~Process()
+{
+ if (IsValid())
+ {
+ CloseHandle(m_ProcessHandle);
+ m_ProcessHandle = nullptr;
+ }
+}
+
+void
+Process::Initialize(int Pid)
+{
+ ZEN_ASSERT(m_ProcessHandle == nullptr);
+ m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, Pid);
+ m_Pid = Pid;
+}
+
+bool
+Process::IsRunning() const
+{
+ DWORD ExitCode = 0;
+ GetExitCodeProcess(m_ProcessHandle, &ExitCode);
+
+ return ExitCode == STILL_ACTIVE;
+}
+
+bool
+Process::IsValid() const
+{
+ return (m_ProcessHandle != nullptr) && (m_ProcessHandle != INVALID_HANDLE_VALUE);
+}
+
+void
+Process::Terminate(int ExitCode)
+{
+ if (IsRunning())
+ {
+ TerminateProcess(m_ProcessHandle, ExitCode);
+ }
+
+ DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE);
+
+ if (WaitResult != WAIT_OBJECT_0)
+ {
+ // What might go wrong here, and what is meaningful to act on?
+ }
+}
+
+bool
+Process::Wait(int TimeoutMs)
+{
+ const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs;
+
+ const DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, Timeout);
+
+ switch (WaitResult)
+ {
+ case WAIT_OBJECT_0:
+ return true;
+
+ case WAIT_TIMEOUT:
+ return false;
+
+ case WAIT_FAILED:
+ // What might go wrong here, and what is meaningful to act on?
+ throw WindowsException("Process::Wait failed");
+ }
+
+ return false;
+}
+
+void
+Sleep(int ms)
+{
+ ::Sleep(ms);
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+void
+thread_forcelink()
+{
+}
+
+} // namespace zen
diff --git a/zencore/timer.cpp b/zencore/timer.cpp
new file mode 100644
index 000000000..ee8e1cf9c
--- /dev/null
+++ b/zencore/timer.cpp
@@ -0,0 +1,67 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <doctest/doctest.h>
+#include <zencore/thread.h>
+#include <zencore/timer.h>
+#include <zencore/windows.h>
+
+namespace zen {
+
+uint64_t
+GetHifreqTimerValue()
+{
+ LARGE_INTEGER li;
+ QueryPerformanceCounter(&li);
+
+ return li.QuadPart;
+}
+
+uint64_t
+internalGetHifreqTimerFrequency()
+{
+ LARGE_INTEGER li;
+ QueryPerformanceFrequency(&li);
+
+ return li.QuadPart;
+}
+
+static uint64_t qpcFreq = internalGetHifreqTimerFrequency();
+
+uint64_t
+GetHifreqTimerFrequency()
+{
+ return qpcFreq;
+}
+
+uint64_t
+GetHifreqTimerFrequencySafe()
+{
+ if (!qpcFreq)
+ qpcFreq = internalGetHifreqTimerFrequency();
+
+ return qpcFreq;
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+void
+timer_forcelink()
+{
+}
+
+TEST_CASE("Timer")
+{
+ uint64_t s0 = GetHifreqTimerValue();
+ uint64_t t0 = GetCpuTimerValue();
+ Sleep(1000);
+ uint64_t s1 = GetHifreqTimerValue();
+ uint64_t t1 = GetCpuTimerValue();
+ // double r = double(t1 - t0) / (s1 - s0);
+ CHECK_NE(t0, t1);
+ CHECK_NE(s0, s1);
+}
+
+} // namespace zen
diff --git a/zencore/trace.cpp b/zencore/trace.cpp
new file mode 100644
index 000000000..8313b3b66
--- /dev/null
+++ b/zencore/trace.cpp
@@ -0,0 +1,51 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/trace.h"
+#include <doctest/doctest.h>
+#include <zencore/windows.h>
+
+namespace zen {
+
+void
+Tracer::Log(const TraceEvent& e)
+{
+ TraceBroadcast(e);
+}
+
+Tracer g_globalTracer;
+
+struct alignas(64) TraceHandlerList
+{
+ enum
+ {
+ kMaxHandlers = 7
+ };
+
+ uint8_t handlerCount = 0;
+ TraceHandler* handlers[kMaxHandlers];
+};
+
+static TraceHandlerList g_traceHandlers;
+
+void
+TraceBroadcast(const TraceEvent& e)
+{
+ for (size_t i = 0; i < g_traceHandlers.handlerCount; ++i)
+ {
+ g_traceHandlers.handlers[i]->Trace(e);
+ }
+}
+
+void
+trace_forcelink()
+{
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+TEST_CASE("Tracer")
+{
+ SUBCASE("Simple") { U_LOG_INFO("bajs"); }
+}
+
+} // namespace zen
diff --git a/zencore/uid.cpp b/zencore/uid.cpp
new file mode 100644
index 000000000..9506b305c
--- /dev/null
+++ b/zencore/uid.cpp
@@ -0,0 +1,196 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/uid.h>
+
+#include <zencore/string.h>
+
+#if _WIN32
+# include <zencore/windows.h>
+# include <bcrypt.h>
+# pragma comment(lib, "bcrypt.lib")
+#endif
+
+#include <atomic>
+#include <bit>
+#include <set>
+#include <unordered_map>
+
+#include <doctest/doctest.h>
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+template<typename T>
+T
+EndianSwap(T value)
+{
+ uint8_t dest[sizeof value];
+
+ memcpy(dest, &value, sizeof value);
+
+ for (int i = 0; i < sizeof(value); i++)
+ {
+ uint8_t& other = dest[sizeof(value) - i - 1];
+
+ uint8_t temp = dest[i];
+ dest[i] = other;
+ other = temp;
+ }
+
+ T ret;
+
+ memcpy(&ret, &value, sizeof value);
+
+ return ret;
+}
+
+#if _WIN32
+__forceinline uint16_t
+EndianSwap(uint16_t value)
+{
+ return _byteswap_ushort(value);
+}
+
+__forceinline uint32_t
+EndianSwap(uint32_t value)
+{
+ return _byteswap_ulong(value);
+}
+
+__forceinline uint64_t
+EndianSwap(uint64_t value)
+{
+ return _byteswap_uint64(value);
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace detail {
+ static bool OidInitialised;
+ static uint32_t RunId;
+ static std::atomic_uint32_t Serial;
+
+ // Number of 100 nanosecond units from 1/1/1601 to 1/1/1970 - used for Windows impl
+ constexpr int64_t kEpochBias = 116'444'736'000'000'000ull;
+} // namespace detail
+
+//////////////////////////////////////////////////////////////////////////
+
+const Oid Oid::Zero = {{0u, 0u, 0u}};
+const Oid Oid::Max = {{~0u, ~0u, ~0u}};
+
+void
+Oid::Initialize()
+{
+ using namespace detail;
+
+ if (OidInitialised)
+ return;
+
+ OidInitialised = true;
+
+#if _WIN32
+ char rng[8];
+ BCryptGenRandom(NULL, (PUCHAR)rng, sizeof rng, BCRYPT_USE_SYSTEM_PREFERRED_RNG);
+
+ memcpy(&RunId, &rng[0], sizeof(RunId));
+ memcpy((void*)&Serial, &rng[4], sizeof(Serial));
+#else
+# error Must implement Oid::Initialize
+#endif
+}
+
+const Oid&
+Oid::Generate()
+{
+ using namespace detail;
+
+ if (!OidInitialised)
+ {
+ Oid::Initialize();
+ }
+
+#if _WIN32
+ FILETIME filetime;
+
+ GetSystemTimeAsFileTime(&filetime); // Time is UTC
+
+ uint64_t filetime64;
+ memcpy(&filetime64, &filetime, sizeof filetime);
+
+ OidBits[0] = EndianSwap(uint32_t((filetime64 - kEpochBias) / 10'000'000l));
+ OidBits[1] = EndianSwap(uint32_t(Serial++));
+ OidBits[2] = RunId;
+#else
+# error Must implement Oid::Generate
+#endif
+
+ return *this;
+}
+
+Oid
+Oid::NewOid()
+{
+ return Oid().Generate();
+}
+
+Oid
+Oid::FromHexString(const std::string_view String)
+{
+ ZEN_ASSERT(String.size() == 2 * sizeof(Oid::OidBits));
+
+ Oid Id;
+
+ ParseHexBytes(String.data(), String.size(), reinterpret_cast<uint8_t*>(Id.OidBits));
+
+ return Id;
+}
+
+StringBuilderBase&
+Oid::ToString(StringBuilderBase& OutString) const
+{
+ char str[25];
+ ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), str);
+ str[2 * sizeof(Oid)] = '\0';
+
+ OutString.AppendRange(str, &str[25]);
+
+ return OutString;
+}
+
+TEST_CASE("Oid")
+{
+ SUBCASE("Basic")
+ {
+ Oid id1 = Oid::NewOid();
+
+ std::vector<Oid> ids;
+ std::set<Oid> idset;
+ std::unordered_map<Oid, int, Oid::Hasher> idmap;
+
+ const int Count = 1000;
+
+ for (int i = 0; i < Count; ++i)
+ {
+ Oid id;
+ id.Generate();
+
+ ids.emplace_back(id);
+ idset.insert(id);
+ idmap.insert({id, i});
+ }
+
+ CHECK(ids.size() == Count);
+ CHECK(idset.size() == Count); // All ids should be unique
+ CHECK(idmap.size() == Count); // Ditto
+ }
+}
+
+void
+uid_forcelink()
+{
+}
+
+} // namespace zen
diff --git a/zencore/xxhash.cpp b/zencore/xxhash.cpp
new file mode 100644
index 000000000..a20ee10bd
--- /dev/null
+++ b/zencore/xxhash.cpp
@@ -0,0 +1,50 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/xxhash.h>
+
+#include <zencore/string.h>
+
+#include <doctest/doctest.h>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+XXH3_128 XXH3_128::Zero; // Initialized to all zeros
+
+XXH3_128
+XXH3_128::FromHexString(const char* InString)
+{
+ return FromHexString({InString, sizeof(XXH3_128::Hash) * 2});
+}
+
+XXH3_128
+XXH3_128::FromHexString(std::string_view InString)
+{
+ ZEN_ASSERT(InString.size() == 2 * sizeof(XXH3_128::Hash));
+
+ XXH3_128 Xx;
+ ParseHexBytes(InString.data(), InString.size(), Xx.Hash);
+ return Xx;
+}
+
+const char*
+XXH3_128::ToHexString(char* OutString /* 40 characters + NUL terminator */) const
+{
+ ToHexBytes(Hash, sizeof(XXH3_128), OutString);
+ OutString[2 * sizeof(XXH3_128)] = '\0';
+
+ return OutString;
+}
+
+StringBuilderBase&
+XXH3_128::ToHexString(StringBuilderBase& OutBuilder) const
+{
+ String_t str;
+ ToHexString(str);
+
+ OutBuilder.AppendRange(str, &str[StringLength]);
+
+ return OutBuilder;
+}
+
+} // namespace zen
diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp
new file mode 100644
index 000000000..d4853b043
--- /dev/null
+++ b/zencore/zencore.cpp
@@ -0,0 +1,70 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/zencore.h>
+
+#include <zencore/windows.h>
+
+#include <zencore/blake3.h>
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/iobuffer.h>
+#include <zencore/memory.h>
+#include <zencore/refcount.h>
+#include <zencore/sha1.h>
+#include <zencore/snapshot_manifest.h>
+#include <zencore/stats.h>
+#include <zencore/stream.h>
+#include <zencore/string.h>
+#include <zencore/thread.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zencore/uid.h>
+
+bool
+IsPointerToStack(const void* ptr)
+{
+ ULONG_PTR low, high;
+ GetCurrentThreadStackLimits(&low, &high);
+
+ const uintptr_t intPtr = reinterpret_cast<uintptr_t>(ptr);
+
+ return (intPtr - low) < (high - low);
+}
+
+static int s_ApplicationExitCode = 0;
+static bool s_ApplicationExitRequested;
+
+bool
+IsApplicationExitRequested()
+{
+ return s_ApplicationExitRequested;
+}
+
+void
+RequestApplicationExit(int ExitCode)
+{
+ s_ApplicationExitCode = ExitCode;
+ s_ApplicationExitRequested = true;
+}
+
+void
+zencore_forcelinktests()
+{
+ zen::sha1_forcelink();
+ zen::blake3_forcelink();
+ zen::trace_forcelink();
+ zen::timer_forcelink();
+ zen::uid_forcelink();
+ zen::string_forcelink();
+ zen::thread_forcelink();
+ zen::stream_forcelink();
+ zen::refcount_forcelink();
+ zen::snapshotmanifest_forcelink();
+ zen::iobuffer_forcelink();
+ zen::stats_forcelink();
+ zen::uson_forcelink();
+ zen::usonbuilder_forcelink();
+ zen::usonpackage_forcelink();
+ zen::memory_forcelink();
+}
diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj
new file mode 100644
index 000000000..c68e922c5
--- /dev/null
+++ b/zencore/zencore.vcxproj
@@ -0,0 +1,188 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <VCProjectVersion>15.0</VCProjectVersion>
+ <ProjectGuid>{D75BF9AB-C61E-4FFF-AD59-1563430F05E2}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>zencore</RootNamespace>
+ <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v142</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v142</PlatformToolset>
+ <WholeProgramOptimization>false</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="Shared">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ <Import Project="..\zenfs_common.props" />
+ <Import Project="..\zen_base_debug.props" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ <Import Project="..\zenfs_common.props" />
+ <Import Project="..\zen_base_release.props" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <EnableMicrosoftCodeAnalysis>false</EnableMicrosoftCodeAnalysis>
+ <EnableClangTidyCodeAnalysis>true</EnableClangTidyCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <VcpkgEnableManifest>true</VcpkgEnableManifest>
+ <VcpkgUseStatic>true</VcpkgUseStatic>
+ </PropertyGroup>
+ <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <VcpkgEnableManifest>true</VcpkgEnableManifest>
+ <VcpkgUseStatic>true</VcpkgUseStatic>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>NotUsing</PrecompiledHeader>
+ <Optimization>Disabled</Optimization>
+ <SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
+ <AdditionalIncludeDirectories>.\include;..\3rdparty\utfcpp\source</AdditionalIncludeDirectories>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <LanguageStandard>stdcpplatest</LanguageStandard>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Windows</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ <Lib />
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <PrecompiledHeader>NotUsing</PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
+ <AdditionalIncludeDirectories>.\include;..\3rdparty\utfcpp\source</AdditionalIncludeDirectories>
+ <WholeProgramOptimization>false</WholeProgramOptimization>
+ <LanguageStandard>stdcpplatest</LanguageStandard>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Windows</SubSystem>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ <Lib>
+ <Verbose>true</Verbose>
+ </Lib>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClInclude Include="include\zencore\atomic.h" />
+ <ClInclude Include="include\zencore\blake3.h" />
+ <ClInclude Include="include\zencore\enumflags.h" />
+ <ClInclude Include="include\zencore\except.h" />
+ <ClInclude Include="include\zencore\compress.h" />
+ <ClInclude Include="include\zencore\filesystem.h" />
+ <ClInclude Include="include\zencore\fmtutils.h" />
+ <ClInclude Include="include\zencore\httpclient.h" />
+ <ClInclude Include="include\zencore\httpserver.h" />
+ <ClInclude Include="include\zencore\intmath.h" />
+ <ClInclude Include="include\zencore\iohash.h" />
+ <ClInclude Include="include\zencore\md5.h" />
+ <ClInclude Include="include\zencore\memory.h" />
+ <ClInclude Include="include\zencore\refcount.h" />
+ <ClInclude Include="include\zencore\scopeguard.h" />
+ <ClInclude Include="include\zencore\sha1.h" />
+ <ClInclude Include="include\zencore\iobuffer.h" />
+ <ClInclude Include="include\zencore\sharedbuffer.h" />
+ <ClInclude Include="include\zencore\snapshot_manifest.h" />
+ <ClInclude Include="include\zencore\stats.h" />
+ <ClInclude Include="include\zencore\stream.h" />
+ <ClInclude Include="include\zencore\streamutil.h" />
+ <ClInclude Include="include\zencore\string.h" />
+ <ClInclude Include="include\zencore\targetver.h" />
+ <ClInclude Include="include\zencore\thread.h" />
+ <ClInclude Include="include\zencore\timer.h" />
+ <ClInclude Include="include\zencore\trace.h" />
+ <ClInclude Include="include\zencore\uid.h" />
+ <ClInclude Include="include\zencore\compactbinary.h" />
+ <ClInclude Include="include\zencore\compactbinarybuilder.h" />
+ <ClInclude Include="include\zencore\compactbinarypackage.h" />
+ <ClInclude Include="include\zencore\compactbinaryvalidation.h" />
+ <ClInclude Include="include\zencore\varint.h" />
+ <ClInclude Include="include\zencore\windows.h" />
+ <ClInclude Include="include\zencore\xxhash.h" />
+ <ClInclude Include="include\zencore\zencore.h" />
+ <ClInclude Include="iothreadpool.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="blake3.cpp" />
+ <ClCompile Include="compress.cpp" />
+ <ClCompile Include="except.cpp" />
+ <ClCompile Include="filesystem.cpp" />
+ <ClCompile Include="httpclient.cpp" />
+ <ClCompile Include="httpserver.cpp" />
+ <ClCompile Include="iohash.cpp" />
+ <ClCompile Include="iothreadpool.cpp" />
+ <ClCompile Include="md5.cpp" />
+ <ClCompile Include="memory.cpp" />
+ <ClCompile Include="refcount.cpp" />
+ <ClCompile Include="sha1.cpp">
+ <Optimization Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">MaxSpeed</Optimization>
+ <InlineFunctionExpansion Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AnySuitable</InlineFunctionExpansion>
+ <IntrinsicFunctions Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</IntrinsicFunctions>
+ <BasicRuntimeChecks Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Default</BasicRuntimeChecks>
+ </ClCompile>
+ <ClCompile Include="iobuffer.cpp" />
+ <ClCompile Include="sharedbuffer.cpp" />
+ <ClCompile Include="snapshot_manifest.cpp" />
+ <ClCompile Include="stats.cpp" />
+ <ClCompile Include="stream.cpp" />
+ <ClCompile Include="streamutil.cpp" />
+ <ClCompile Include="string.cpp" />
+ <ClCompile Include="thread.cpp" />
+ <ClCompile Include="timer.cpp" />
+ <ClCompile Include="trace.cpp" />
+ <ClCompile Include="uid.cpp" />
+ <ClCompile Include="compactbinary.cpp" />
+ <ClCompile Include="compactbinarybuilder.cpp" />
+ <ClCompile Include="compactbinarypackage.cpp" />
+ <ClCompile Include="compactbinaryvalidation.cpp" />
+ <ClCompile Include="xxhash.cpp" />
+ <ClCompile Include="zencore.cpp" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project> \ No newline at end of file
diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters
new file mode 100644
index 000000000..c25f99e77
--- /dev/null
+++ b/zencore/zencore.vcxproj.filters
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <ClInclude Include="include\zencore\intmath.h" />
+ <ClInclude Include="include\zencore\scopeguard.h" />
+ <ClInclude Include="include\zencore\sha1.h" />
+ <ClInclude Include="include\zencore\snapshot_manifest.h" />
+ <ClInclude Include="include\zencore\targetver.h" />
+ <ClInclude Include="include\zencore\zencore.h" />
+ <ClInclude Include="include\zencore\compactbinary.h" />
+ <ClInclude Include="include\zencore\uid.h" />
+ <ClInclude Include="include\zencore\trace.h" />
+ <ClInclude Include="include\zencore\compress.h" />
+ <ClInclude Include="include\zencore\timer.h" />
+ <ClInclude Include="include\zencore\thread.h" />
+ <ClInclude Include="include\zencore\string.h" />
+ <ClInclude Include="include\zencore\streamutil.h" />
+ <ClInclude Include="include\zencore\stream.h" />
+ <ClInclude Include="include\zencore\stats.h" />
+ <ClInclude Include="include\zencore\blake3.h" />
+ <ClInclude Include="include\zencore\atomic.h" />
+ <ClInclude Include="include\zencore\enumflags.h" />
+ <ClInclude Include="include\zencore\except.h" />
+ <ClInclude Include="include\zencore\filesystem.h" />
+ <ClInclude Include="include\zencore\httpserver.h" />
+ <ClInclude Include="include\zencore\refcount.h" />
+ <ClInclude Include="include\zencore\memory.h" />
+ <ClInclude Include="include\zencore\windows.h" />
+ <ClInclude Include="include\zencore\iobuffer.h" />
+ <ClInclude Include="include\zencore\sharedbuffer.h" />
+ <ClInclude Include="include\zencore\iohash.h" />
+ <ClInclude Include="include\zencore\compactbinarybuilder.h" />
+ <ClInclude Include="include\zencore\compactbinarypackage.h" />
+ <ClInclude Include="include\zencore\compactbinaryvalidation.h" />
+ <ClInclude Include="include\zencore\httpclient.h" />
+ <ClInclude Include="include\zencore\md5.h" />
+ <ClInclude Include="include\zencore\fmtutils.h" />
+ <ClInclude Include="include\zencore\xxhash.h" />
+ <ClInclude Include="iothreadpool.h" />
+ <ClInclude Include="include\zencore\varint.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="snapshot_manifest.cpp" />
+ <ClCompile Include="sha1.cpp" />
+ <ClCompile Include="zencore.cpp" />
+ <ClCompile Include="compactbinary.cpp" />
+ <ClCompile Include="uid.cpp" />
+ <ClCompile Include="blake3.cpp" />
+ <ClCompile Include="filesystem.cpp" />
+ <ClCompile Include="httpserver.cpp" />
+ <ClCompile Include="memory.cpp" />
+ <ClCompile Include="refcount.cpp" />
+ <ClCompile Include="stats.cpp" />
+ <ClCompile Include="stream.cpp" />
+ <ClCompile Include="streamutil.cpp" />
+ <ClCompile Include="string.cpp" />
+ <ClCompile Include="thread.cpp" />
+ <ClCompile Include="timer.cpp" />
+ <ClCompile Include="trace.cpp" />
+ <ClCompile Include="iobuffer.cpp" />
+ <ClCompile Include="sharedbuffer.cpp" />
+ <ClCompile Include="iohash.cpp" />
+ <ClCompile Include="compactbinarybuilder.cpp" />
+ <ClCompile Include="compactbinarypackage.cpp" />
+ <ClCompile Include="compactbinaryvalidation.cpp" />
+ <ClCompile Include="httpclient.cpp" />
+ <ClCompile Include="md5.cpp" />
+ <ClCompile Include="except.cpp" />
+ <ClCompile Include="xxhash.cpp" />
+ <ClCompile Include="iothreadpool.cpp" />
+ <ClCompile Include="compress.cpp" />
+ </ItemGroup>
+ <ItemGroup>
+ <Filter Include="CAS">
+ <UniqueIdentifier>{af5266fa-37a5-494c-9116-b15a3e6edd29}</UniqueIdentifier>
+ </Filter>
+ </ItemGroup>
+</Project> \ No newline at end of file